restart the proxy if it goes down

and populate its table from the database

jupyterhub/app.py

@@ -20,7 +20,7 @@ from jinja2 import Environment, FileSystemLoader
 
 import tornado.httpserver
 import tornado.options
-from tornado.ioloop import IOLoop
+from tornado.ioloop import IOLoop, PeriodicCallback
 from tornado.log import LogFormatter
 from tornado import gen, web
 
@@ -109,6 +109,9 @@ class JupyterHubApp(Application):
         Useful for daemonizing jupyterhub.
         """
     )
+    proxy_check_interval = Integer(int(1e4), config=True,
+        help="Interval (in ms) at which to check if the proxy is running."
+    )
     
     data_files_path = Unicode(DATA_FILES_PATH, config=True,
         help="The location of jupyter data files (e.g. /usr/local/share/jupyter)"
@@ -331,6 +334,7 @@ class JupyterHubApp(Application):
         self.db.add(self.proxy)
         self.db.commit()
     
+    @gen.coroutine
     def start_proxy(self):
         """Actually start the configurable-http-proxy"""
         env = os.environ.copy()
@@ -350,6 +354,35 @@ class JupyterHubApp(Application):
             cmd.extend(['--ssl-cert', self.ssl_cert])
         self.log.info("Starting proxy: %s", cmd)
         self.proxy_process = Popen(cmd, env=env)
+        def _check():
+            status = self.proxy_process.poll()
+            if status is not None:
+                e = RuntimeError("Proxy failed to start with exit code %i" % status)
+                # py2-compatible `raise e from None`
+                e.__cause__ = None
+                raise e
+        
+        for server in (self.proxy.public_server, self.proxy.api_server):
+            for i in range(10):
+                _check()
+                try:
+                    yield server.wait_up(1)
+                except TimeoutError:
+                    continue
+                else:
+                    break
+            yield server.wait_up(1)
+        self.log.debug("Proxy started and appears to be up")
+    
+    @gen.coroutine
+    def check_proxy(self):
+        if self.proxy_process.poll() is None:
+            return
+        self.log.error("Proxy stopped with exit code %i", self.proxy_process.poll())
+        yield self.start_proxy()
+        self.log.info("Setting up routes on new proxy")
+        yield self.proxy.add_all_users()
+        self.log.info("New proxy back up, and good to go")
     
     def init_tornado_settings(self):
         """Set up the tornado settings dict."""
@@ -460,13 +493,22 @@ class JupyterHubApp(Application):
         if self.generate_config:
             self.write_config_file()
             return
+        loop = IOLoop.current()
+        
         # start the proxy
-        self.start_proxy()
+        try:
+            loop.run_sync(self.start_proxy)
+        except Exception as e:
+            self.log.critical("Failed to start proxy", exc_info=True)
+            return
+        
+        pc = PeriodicCallback(self.check_proxy, self.proxy_check_interval)
+        pc.start()
+        
         # start the webserver
         http_server = tornado.httpserver.HTTPServer(self.tornado_application)
         http_server.listen(self.hub_port)
         
-        loop = IOLoop.current()
         try:
             loop.start()
         except KeyboardInterrupt:

jupyterhub/orm.py

@@ -3,6 +3,7 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+import errno
 import json
 import uuid
 
@@ -95,6 +96,19 @@ class Server(Base):
         """Wait for this server to come up"""
         yield wait_for_server(self.ip or 'localhost', self.port, timeout=timeout)
     
+    def is_up(self):
+        """Is the server accepting connections?"""
+        try:
+            socket.create_connection((self.ip or 'localhost', self.port))
+        except socket.error as e:
+            if e.errno == errno.ECONNREFUSED:
+                return True
+            else:
+                raise
+        else:
+            return True
+        
+
 
 class Proxy(Base):
     """A configurable-http-proxy instance.
@@ -146,6 +160,21 @@ class Proxy(Base):
         )
         r.raise_for_status()
 
+    @gen.coroutine
+    def add_all_users(self):
+        """Update the proxy table from the database.
+        
+        Used when loading up a new proxy.
+        """
+        db = inspect(self).session
+        futures = []
+        for user in db.query(User):
+            if (user.server):
+                futures.append(self.add_user(user))
+        # wait after submitting them all
+        for f in futures:
+            yield f
+
 
 class Hub(Base):
     """Bring it all together at the hub.