Merge pull request #1221 from minrk/proxy-race

add User.proxy_pending

jupyterhub/handlers/base.py

@@ -321,6 +321,7 @@ class BaseHandler(RequestHandler):
         tic = IOLoop.current().time()
 
         f = user.spawn(options)
+        user.proxy_pending = True
 
         @gen.coroutine
         def finish_user_spawn(f=None):
@@ -335,8 +336,16 @@ class BaseHandler(RequestHandler):
             toc = IOLoop.current().time()
             self.log.info("User %s server took %.3f seconds to start", user.name, toc-tic)
             self.statsd.timing('spawner.success', (toc - tic) * 1000)
-            yield self.proxy.add_user(user)
-            user.spawner.add_poll_callback(self.user_stopped, user)
+            try:
+                yield self.proxy.add_user(user)
+            except Exception:
+                self.log.exception("Failed to add user %s to proxy!", user)
+                self.log.error("Stopping user %s to avoid inconsistent state")
+                yield user.stop()
+            else:
+                user.spawner.add_poll_callback(self.user_stopped, user)
+            finally:
+                user.proxy_pending = False
 
         try:
             yield gen.with_timeout(timedelta(seconds=self.slow_spawn_timeout), f)
@@ -532,7 +541,7 @@ class UserSpawnHandler(BaseHandler):
 
             # logged in as correct user, spawn the server
             if current_user.spawner:
-                if current_user.spawn_pending:
+                if current_user.spawn_pending or current_user.proxy_pending:
                     # spawn has started, but not finished
                     self.statsd.incr('redirects.user_spawn_pending', 1)
                     html = self.render_template("spawn_pending.html", user=current_user)

jupyterhub/singleuser.py

@@ -4,9 +4,7 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 
-from distutils.version import LooseVersion as V
 import os
-import re
 from textwrap import dedent
 from urllib.parse import urlparse
 
@@ -349,10 +347,17 @@ class SingleUserNotebookApp(NotebookApp):
         - check version and warn on sufficient mismatch
         """
         client = AsyncHTTPClient()
-        try:
-            resp = yield client.fetch(self.hub_api_url)
-        except Exception:
-            self.log.exception("Failed to connect to my Hub at %s. Is it running?", self.hub_api_url)
+        RETRIES = 5
+        for i in range(1, RETRIES+1):
+            try:
+                resp = yield client.fetch(self.hub_api_url)
+            except Exception:
+                self.log.exception("Failed to connect to my Hub at %s (attempt %i/%i). Is it running?",
+                self.hub_api_url, i, RETRIES)
+                yield gen.sleep(min(2**i, 16))
+            else:
+                break
+        else:
             self.exit(1)
         
         hub_version = resp.headers.get('X-JupyterHub-Version')

jupyterhub/user.py

@@ -110,6 +110,7 @@ class User(HasTraits):
     spawner = None
     spawn_pending = False
     stop_pending = False
+    proxy_pending = False
     waiting_for_response = False
 
     @property
@@ -158,8 +159,8 @@ class User(HasTraits):
 
     @property # FIX-ME CHECK IF STILL NEEDED
     def running(self):
-        """property for whether a user has a running server"""
-        if self.spawn_pending or self.stop_pending:
+        """property for whether a user has a fully running, accessible server"""
+        if self.spawn_pending or self.stop_pending or self.proxy_pending:
             return False  # server is not running if spawn or stop is still pending
         if self.server is None:
             return False
@@ -392,10 +393,13 @@ class User(HasTraits):
                     self.db.delete(orm_token)
             self.db.commit()
         finally:
-            self.stop_pending = False
             # trigger post-spawner hook on authenticator
             auth = spawner.authenticator
-            if auth:
-                yield gen.maybe_future(
-                    auth.post_spawn_stop(self, spawner)
-                )
+            try:
+                if auth:
+                    yield gen.maybe_future(
+                        auth.post_spawn_stop(self, spawner)
+                    )
+            except Exception:
+                self.log.exception("Error in Authenticator.post_spawn_stop for %s", self)
+            self.stop_pending = False