From a79071bb33a9c693c3b24cf0a35d52f99325c158 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 14 Jul 2017 14:57:06 +0200 Subject: [PATCH 1/3] add User.proxy_pending flag for waiting for the proxy to be updated avoids User.running being True when the user's server has not yet been added to the proxy, causing potential redirect loops. --- jupyterhub/handlers/base.py | 15 ++++++++++++--- jupyterhub/user.py | 5 +++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/jupyterhub/handlers/base.py b/jupyterhub/handlers/base.py index 8c4cea88..68a54725 100644 --- a/jupyterhub/handlers/base.py +++ b/jupyterhub/handlers/base.py @@ -321,6 +321,7 @@ class BaseHandler(RequestHandler): tic = IOLoop.current().time() f = user.spawn(options) + user.proxy_pending = True @gen.coroutine def finish_user_spawn(f=None): @@ -335,8 +336,16 @@ class BaseHandler(RequestHandler): toc = IOLoop.current().time() self.log.info("User %s server took %.3f seconds to start", user.name, toc-tic) self.statsd.timing('spawner.success', (toc - tic) * 1000) - yield self.proxy.add_user(user) - user.spawner.add_poll_callback(self.user_stopped, user) + try: + yield self.proxy.add_user(user) + except Exception: + self.log.exception("Failed to add user %s to proxy!", user) + self.log.error("Stopping user %s to avoid inconsistent state") + yield user.stop() + else: + user.spawner.add_poll_callback(self.user_stopped, user) + finally: + user.proxy_pending = False try: yield gen.with_timeout(timedelta(seconds=self.slow_spawn_timeout), f) @@ -532,7 +541,7 @@ class UserSpawnHandler(BaseHandler): # logged in as correct user, spawn the server if current_user.spawner: - if current_user.spawn_pending: + if current_user.spawn_pending or current_user.proxy_pending: # spawn has started, but not finished self.statsd.incr('redirects.user_spawn_pending', 1) html = self.render_template("spawn_pending.html", user=current_user) diff --git a/jupyterhub/user.py b/jupyterhub/user.py index f23eaf33..223e674f 100644 --- a/jupyterhub/user.py +++ b/jupyterhub/user.py @@ -110,6 +110,7 @@ class User(HasTraits): spawner = None spawn_pending = False stop_pending = False + proxy_pending = False waiting_for_response = False @property @@ -158,8 +159,8 @@ class User(HasTraits): @property # FIX-ME CHECK IF STILL NEEDED def running(self): - """property for whether a user has a running server""" - if self.spawn_pending or self.stop_pending: + """property for whether a user has a fully running, accessible server""" + if self.spawn_pending or self.stop_pending or self.proxy_pending: return False # server is not running if spawn or stop is still pending if self.server is None: return False From c78d88707c059157c47546196ae2bf99088b0cd6 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 14 Jul 2017 15:34:34 +0200 Subject: [PATCH 2/3] fallback during initial hub connection --- jupyterhub/singleuser.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/jupyterhub/singleuser.py b/jupyterhub/singleuser.py index d52c0a12..95ca4028 100755 --- a/jupyterhub/singleuser.py +++ b/jupyterhub/singleuser.py @@ -4,9 +4,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -from distutils.version import LooseVersion as V import os -import re from textwrap import dedent from urllib.parse import urlparse @@ -349,10 +347,17 @@ class SingleUserNotebookApp(NotebookApp): - check version and warn on sufficient mismatch """ client = AsyncHTTPClient() - try: - resp = yield client.fetch(self.hub_api_url) - except Exception: - self.log.exception("Failed to connect to my Hub at %s. Is it running?", self.hub_api_url) + RETRIES = 5 + for i in range(1, RETRIES+1): + try: + resp = yield client.fetch(self.hub_api_url) + except Exception: + self.log.exception("Failed to connect to my Hub at %s (attempt %i/%i). Is it running?", + self.hub_api_url, i, RETRIES) + yield gen.sleep(min(2**i, 16)) + else: + break + else: self.exit(1) hub_version = resp.headers.get('X-JupyterHub-Version') From a6c2939bb4932bfd7dcaf52acf0319eb3dc2db0f Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 14 Jul 2017 15:53:38 +0200 Subject: [PATCH 3/3] delay stop_pending until the end of stop avoids stop_pending being False while there's still one yield to go --- jupyterhub/user.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/jupyterhub/user.py b/jupyterhub/user.py index 223e674f..34a06431 100644 --- a/jupyterhub/user.py +++ b/jupyterhub/user.py @@ -393,10 +393,13 @@ class User(HasTraits): self.db.delete(orm_token) self.db.commit() finally: - self.stop_pending = False # trigger post-spawner hook on authenticator auth = spawner.authenticator - if auth: - yield gen.maybe_future( - auth.post_spawn_stop(self, spawner) - ) + try: + if auth: + yield gen.maybe_future( + auth.post_spawn_stop(self, spawner) + ) + except Exception: + self.log.exception("Error in Authenticator.post_spawn_stop for %s", self) + self.stop_pending = False