Check if a Spawner is running at the given URL on startup

avoids assuming partial spawns that may have resulted in running processes but not actually available servers
are treated as fully working servers.

jupyterhub/app.py

@@ -1687,6 +1687,40 @@ class JupyterHub(Application):
                         spawner._log_name)
                     status = -1
 
+            if status is None:
+                # poll claims it's running.
+                # Check if it's really there
+                url_in_db = spawner.server.url
+                url = await spawner.get_url()
+                if url != url_in_db:
+                    self.log.warning(
+                        "%s had invalid url %s. Updating to %s",
+                        spawner._log_name, url_in_db, url,
+                    )
+                    urlinfo = urlparse(url)
+                    spawner.server.protocol = urlinfo.scheme
+                    spawner.server.ip = urlinfo.hostname
+                    if urlinfo.port:
+                        spawner.server.port = urlinfo.port
+                    elif urlinfo.scheme == 'http':
+                        spawner.server.port = 80
+                    elif urlinfo.scheme == 'https':
+                        spawner.server.port = 443
+                    self.db.commit()
+
+                self.log.debug(
+                    "Verifying that %s is running at %s",
+                    spawner._log_name, url,
+                )
+                try:
+                    await user._wait_up(spawner)
+                except TimeoutError:
+                    self.log.error(
+                        "%s does not appear to be running at %s, shutting it down.",
+                        spawner._log_name, url,
+                    )
+                    status = -1
+
             if status is None:
                 self.log.info("%s still running", user.name)
                 spawner.add_poll_callback(user_stopped, user, name)

jupyterhub/user.py

@@ -531,7 +531,7 @@ class User:
                 self.settings['statsd'].incr('spawner.failure.error')
                 e.reason = 'error'
             try:
-                await self.stop()
+                await self.stop(spawner.server_name)
             except Exception:
                 self.log.error("Failed to cleanup {user}'s server that failed to start".format(
                     user=self.name,
@@ -550,6 +550,15 @@ class User:
         spawner.orm_spawner.state = spawner.get_state()
         db.commit()
         spawner._waiting_for_response = True
+        await self._wait_up(spawner)
+
+    async def _wait_up(self, spawner):
+        """Wait for a server to finish starting.
+
+        Shuts the server down if it doesn't respond within
+        spawner.http_timeout.
+        """
+        server = spawner.server
         key = self.settings.get('internal_ssl_key')
         cert = self.settings.get('internal_ssl_cert')
         ca = self.settings.get('internal_ssl_ca')
@@ -578,7 +587,7 @@ class User:
                 ))
                 self.settings['statsd'].incr('spawner.failure.http_error')
             try:
-                await self.stop()
+                await self.stop(spawner.server_name)
             except Exception:
                 self.log.error("Failed to cleanup {user}'s server that failed to start".format(
                     user=self.name,
@@ -594,7 +603,7 @@ class User:
         finally:
             spawner._waiting_for_response = False
             spawner._start_pending = False
-        return self
+        return spawner
 
     async def stop(self, server_name=''):
         """Stop the user's spawner