diff --git a/jupyterhub/handlers/base.py b/jupyterhub/handlers/base.py index 8c4cea88..68a54725 100644 --- a/jupyterhub/handlers/base.py +++ b/jupyterhub/handlers/base.py @@ -321,6 +321,7 @@ class BaseHandler(RequestHandler): tic = IOLoop.current().time() f = user.spawn(options) + user.proxy_pending = True @gen.coroutine def finish_user_spawn(f=None): @@ -335,8 +336,16 @@ class BaseHandler(RequestHandler): toc = IOLoop.current().time() self.log.info("User %s server took %.3f seconds to start", user.name, toc-tic) self.statsd.timing('spawner.success', (toc - tic) * 1000) - yield self.proxy.add_user(user) - user.spawner.add_poll_callback(self.user_stopped, user) + try: + yield self.proxy.add_user(user) + except Exception: + self.log.exception("Failed to add user %s to proxy!", user) + self.log.error("Stopping user %s to avoid inconsistent state") + yield user.stop() + else: + user.spawner.add_poll_callback(self.user_stopped, user) + finally: + user.proxy_pending = False try: yield gen.with_timeout(timedelta(seconds=self.slow_spawn_timeout), f) @@ -532,7 +541,7 @@ class UserSpawnHandler(BaseHandler): # logged in as correct user, spawn the server if current_user.spawner: - if current_user.spawn_pending: + if current_user.spawn_pending or current_user.proxy_pending: # spawn has started, but not finished self.statsd.incr('redirects.user_spawn_pending', 1) html = self.render_template("spawn_pending.html", user=current_user) diff --git a/jupyterhub/singleuser.py b/jupyterhub/singleuser.py index d52c0a12..95ca4028 100755 --- a/jupyterhub/singleuser.py +++ b/jupyterhub/singleuser.py @@ -4,9 +4,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -from distutils.version import LooseVersion as V import os -import re from textwrap import dedent from urllib.parse import urlparse @@ -349,10 +347,17 @@ class SingleUserNotebookApp(NotebookApp): - check version and warn on sufficient mismatch """ client = AsyncHTTPClient() - try: - resp = yield client.fetch(self.hub_api_url) - except Exception: - self.log.exception("Failed to connect to my Hub at %s. Is it running?", self.hub_api_url) + RETRIES = 5 + for i in range(1, RETRIES+1): + try: + resp = yield client.fetch(self.hub_api_url) + except Exception: + self.log.exception("Failed to connect to my Hub at %s (attempt %i/%i). Is it running?", + self.hub_api_url, i, RETRIES) + yield gen.sleep(min(2**i, 16)) + else: + break + else: self.exit(1) hub_version = resp.headers.get('X-JupyterHub-Version') diff --git a/jupyterhub/user.py b/jupyterhub/user.py index f23eaf33..34a06431 100644 --- a/jupyterhub/user.py +++ b/jupyterhub/user.py @@ -110,6 +110,7 @@ class User(HasTraits): spawner = None spawn_pending = False stop_pending = False + proxy_pending = False waiting_for_response = False @property @@ -158,8 +159,8 @@ class User(HasTraits): @property # FIX-ME CHECK IF STILL NEEDED def running(self): - """property for whether a user has a running server""" - if self.spawn_pending or self.stop_pending: + """property for whether a user has a fully running, accessible server""" + if self.spawn_pending or self.stop_pending or self.proxy_pending: return False # server is not running if spawn or stop is still pending if self.server is None: return False @@ -392,10 +393,13 @@ class User(HasTraits): self.db.delete(orm_token) self.db.commit() finally: - self.stop_pending = False # trigger post-spawner hook on authenticator auth = spawner.authenticator - if auth: - yield gen.maybe_future( - auth.post_spawn_stop(self, spawner) - ) + try: + if auth: + yield gen.maybe_future( + auth.post_spawn_stop(self, spawner) + ) + except Exception: + self.log.exception("Error in Authenticator.post_spawn_stop for %s", self) + self.stop_pending = False