diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 3293bc15..c33ecc3f 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -20,7 +20,7 @@ from jinja2 import Environment, FileSystemLoader import tornado.httpserver import tornado.options -from tornado.ioloop import IOLoop +from tornado.ioloop import IOLoop, PeriodicCallback from tornado.log import LogFormatter from tornado import gen, web @@ -109,6 +109,9 @@ class JupyterHubApp(Application): Useful for daemonizing jupyterhub. """ ) + proxy_check_interval = Integer(int(1e4), config=True, + help="Interval (in ms) at which to check if the proxy is running." + ) data_files_path = Unicode(DATA_FILES_PATH, config=True, help="The location of jupyter data files (e.g. /usr/local/share/jupyter)" @@ -331,6 +334,7 @@ class JupyterHubApp(Application): self.db.add(self.proxy) self.db.commit() + @gen.coroutine def start_proxy(self): """Actually start the configurable-http-proxy""" env = os.environ.copy() @@ -350,6 +354,35 @@ class JupyterHubApp(Application): cmd.extend(['--ssl-cert', self.ssl_cert]) self.log.info("Starting proxy: %s", cmd) self.proxy_process = Popen(cmd, env=env) + def _check(): + status = self.proxy_process.poll() + if status is not None: + e = RuntimeError("Proxy failed to start with exit code %i" % status) + # py2-compatible `raise e from None` + e.__cause__ = None + raise e + + for server in (self.proxy.public_server, self.proxy.api_server): + for i in range(10): + _check() + try: + yield server.wait_up(1) + except TimeoutError: + continue + else: + break + yield server.wait_up(1) + self.log.debug("Proxy started and appears to be up") + + @gen.coroutine + def check_proxy(self): + if self.proxy_process.poll() is None: + return + self.log.error("Proxy stopped with exit code %i", self.proxy_process.poll()) + yield self.start_proxy() + self.log.info("Setting up routes on new proxy") + yield self.proxy.add_all_users() + self.log.info("New proxy back up, and good to go") def init_tornado_settings(self): """Set up the tornado settings dict.""" @@ -460,13 +493,22 @@ class JupyterHubApp(Application): if self.generate_config: self.write_config_file() return + loop = IOLoop.current() + # start the proxy - self.start_proxy() + try: + loop.run_sync(self.start_proxy) + except Exception as e: + self.log.critical("Failed to start proxy", exc_info=True) + return + + pc = PeriodicCallback(self.check_proxy, self.proxy_check_interval) + pc.start() + # start the webserver http_server = tornado.httpserver.HTTPServer(self.tornado_application) http_server.listen(self.hub_port) - loop = IOLoop.current() try: loop.start() except KeyboardInterrupt: diff --git a/jupyterhub/orm.py b/jupyterhub/orm.py index e313571b..658821c2 100644 --- a/jupyterhub/orm.py +++ b/jupyterhub/orm.py @@ -3,6 +3,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +import errno import json import uuid @@ -94,6 +95,19 @@ class Server(Base): def wait_up(self, timeout=10): """Wait for this server to come up""" yield wait_for_server(self.ip or 'localhost', self.port, timeout=timeout) + + def is_up(self): + """Is the server accepting connections?""" + try: + socket.create_connection((self.ip or 'localhost', self.port)) + except socket.error as e: + if e.errno == errno.ECONNREFUSED: + return True + else: + raise + else: + return True + class Proxy(Base): @@ -146,6 +160,21 @@ class Proxy(Base): ) r.raise_for_status() + @gen.coroutine + def add_all_users(self): + """Update the proxy table from the database. + + Used when loading up a new proxy. + """ + db = inspect(self).session + futures = [] + for user in db.query(User): + if (user.server): + futures.append(self.add_user(user)) + # wait after submitting them all + for f in futures: + yield f + class Hub(Base): """Bring it all together at the hub.