periodically poll single-user servers

If they aren't running, unregister them
and remove them from the proxy so that future
logins are handled by the hub, and spawn new servers.

jupyterhub/app.py

@@ -111,8 +111,8 @@ class JupyterHubApp(Application):
         Useful for daemonizing jupyterhub.
         """
     )
-    proxy_check_interval = Integer(int(1e4), config=True,
-        help="Interval (in ms) at which to check if the proxy is running."
+    proxy_check_interval = Integer(10, config=True,
+        help="Interval (in seconds) at which to check if the proxy is running."
     )
     
     data_files_path = Unicode(DATA_FILES_PATH, config=True,
@@ -415,6 +415,15 @@ class JupyterHubApp(Application):
                 parts.append('running at %s' % user.server)
             return ' '.join(parts)
             
+        @gen.coroutine
+        def user_stopped(user):
+            status = yield user.spawner.poll()
+            self.log.warn("User %s server stopped with exit code: %s",
+                user.name, status,
+            )
+            yield self.proxy.delete_user(user)
+            yield user.stop()
+        
         for user in db.query(orm.User):
             if not user.state:
                 user_summaries.append(_user_summary(user))
@@ -425,6 +434,8 @@ class JupyterHubApp(Application):
             if status is None:
                 self.log.info("User %s still running", user.name)
                 user.spawner = spawner
+                spawner.add_poll_callback(user_stopped, user)
+                spawner.start_polling()
             else:
                 self.log.warn("Failed to load state for %s, assuming server is not running.", user.name)
                 # not running, state is invalid
@@ -650,7 +661,7 @@ class JupyterHubApp(Application):
             # only check / restart the proxy if we started it in the first place.
             # this means a restarted Hub cannot restart a Proxy that its
             # predecessor started.
-            pc = PeriodicCallback(self.check_proxy, self.proxy_check_interval)
+            pc = PeriodicCallback(self.check_proxy, 1e3 * self.proxy_check_interval)
             pc.start()
         
         # start the webserver

jupyterhub/handlers/base.py

@@ -169,8 +169,18 @@ class BaseHandler(RequestHandler):
             config=self.config,
         )
         yield self.proxy.add_user(user)
+        user.spawner.add_poll_callback(self.user_stopped, user)
         raise gen.Return(user)
     
+    @gen.coroutine
+    def user_stopped(self, user):
+        status = yield user.spawner.poll()
+        self.log.warn("User %s server stopped, with exit code: %s",
+            user.name, status,
+        )
+        yield self.proxy.delete_user(user)
+        yield user.stop()
+    
     @gen.coroutine
     def stop_single_user(self, user):
         yield self.proxy.delete_user(user)

jupyterhub/spawner.py

@@ -7,18 +7,16 @@ import errno
 import os
 import pwd
 import signal
-import time
 from subprocess import Popen
 
 from tornado import gen
-from tornado.ioloop import IOLoop
+from tornado.ioloop import IOLoop, PeriodicCallback
 
 from IPython.config import LoggingConfigurable
 from IPython.utils.traitlets import (
     Any, Bool, Dict, Enum, Instance, Integer, List, Unicode,
 )
 
-
 from .utils import random_port
 
 
@@ -38,6 +36,12 @@ class Spawner(LoggingConfigurable):
     hub = Any()
     api_token = Unicode()
     
+    poll_interval = Integer(30, config=True,
+        help="""Interval (in seconds) on which to poll the spawner."""
+    )
+    _callbacks = List()
+    _poll_callback = Any()
+    
     debug = Bool(False, config=True,
         help="Enable debug-logging of the single-user server"
     )
@@ -135,6 +139,60 @@ class Spawner(LoggingConfigurable):
         """
         raise NotImplementedError("Override in subclass. Must be a Tornado gen.coroutine.")
     
+    def add_poll_callback(self, callback, *args, **kwargs):
+        """add a callback to fire when the subprocess stops
+        
+        as noticed by periodic poll_and_notify()
+        """
+        if args or kwargs:
+            cb = callback
+            callback = lambda : cb(*args, **kwargs)
+        self._callbacks.append(callback)
+    
+    def stop_polling(self):
+        """stop the periodic poll"""
+        if self._poll_callback:
+            self._poll_callback.stop()
+            self._poll_callback = None
+        
+    def start_polling(self):
+        """Start polling periodically
+        
+        callbacks registered via `add_poll_callback` will fire
+        if/when the process stops.
+        
+        Explicit termination via the stop method will not trigger the callbacks.
+        """
+        if self.poll_interval <= 0:
+            self.log.debug("Not polling subprocess")
+            return
+        else:
+            self.log.debug("Polling subprocess every %is", self.poll_interval)
+        
+        self.stop_polling()
+        
+        self._poll_callback = PeriodicCallback(
+            self.poll_and_notify,
+            1e3 * self.poll_interval
+        )
+        self._poll_callback.start()
+
+    @gen.coroutine
+    def poll_and_notify(self):
+        """Used as a callback to periodically poll the process,
+        and notify any watchers
+        """
+        status = yield self.poll()
+        if status is None:
+            # still running, nothing to do here
+            return
+        
+        self.stop_polling()
+        
+        add_callback = IOLoop.current().add_callback
+        for callback in self._callbacks:
+            add_callback(callback)
+
 
 def set_user_setuid(username):
     """return a preexec_fn for setting the user (via setuid) of a spawned process"""
@@ -245,6 +303,7 @@ class LocalProcessSpawner(Spawner):
             preexec_fn=self.make_preexec_fn(self.user.name),
         )
         self.pid = self.proc.pid
+        self.start_polling()
     
     @gen.coroutine
     def poll(self):
@@ -284,6 +343,7 @@ class LocalProcessSpawner(Spawner):
         
         if `now`, skip waiting for clean shutdown
         """
+        self.stop_polling()
         if not now:
             # SIGINT to request clean shutdown
             self.log.debug("Interrupting %i", self.pid)