better handle servers failing to start

including more informative TimeoutError messages

jupyterhub/orm.py

@@ -54,6 +54,7 @@ class JSONDict(TypeDecorator):
 
 
 Base = declarative_base()
+Base.log = app_log
 
 
 class Server(Base):
@@ -121,7 +122,6 @@ class Proxy(Base):
     public_server = relationship(Server, primaryjoin=_public_server_id == Server.id)
     _api_server_id = Column(Integer, ForeignKey('servers.id'))
     api_server = relationship(Server, primaryjoin=_api_server_id == Server.id)
-    log = app_log
     
     def __repr__(self):
         if self.public_server:
@@ -191,7 +191,7 @@ class Proxy(Base):
     def get_routes(self, client=None):
         """Fetch the proxy's routes"""
         resp = yield self.api_request('', client=client)
-        raise gen.Return(json.loads(resp.body.decode('utf8', 'replace')))
+        return json.loads(resp.body.decode('utf8', 'replace'))
 
 
 class Hub(Base):
@@ -317,9 +317,21 @@ class User(Base):
         self.state = spawner.get_state()
         self.last_activity = datetime.utcnow()
         db.commit()
-        
+        try:
-        yield self.server.wait_up(http=True)
+            yield self.server.wait_up(http=True)
-        raise gen.Return(self)
+        except TimeoutError as e:
+            self.log.warn("{user}'s server never started at {url}, giving up.".format(
+                user=self.name, url=self.server.url,
+            ))
+            try:
+                yield self.stop()
+            except Exception:
+                self.log.error("Failed to cleanup {user}'s server that failed to start".format(
+                    user=self.name,
+                ), exc_info=True)
+            # raise original TimeoutError
+            raise e
+        return self
 
     @gen.coroutine
     def stop(self):

jupyterhub/utils.py

@@ -45,7 +45,9 @@ def wait_for_server(ip, port, timeout=10):
             yield gen.Task(loop.add_timeout, loop.time() + 0.1)
         else:
             return
-    raise TimeoutError
+    raise TimeoutError("Server at {ip}:{port} didn't respond in {timeout} seconds".format(
+        **locals()
+    ))
 
 @gen.coroutine
 def wait_for_http_server(url, timeout=10):
@@ -77,7 +79,9 @@ def wait_for_http_server(url, timeout=10):
         else:
             return
     
-    raise TimeoutError
+    raise TimeoutError("Server at {url} didn't respond in {timeout} seconds".format(
+        **locals()
+    ))
 
 
 # Decorators for authenticated Handlers
@@ -166,3 +170,4 @@ def compare_token(compare, token):
         return True
     return False
     
+