jupyterhub/jupyterhub/user.py

# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import json
import string
import warnings
from collections import defaultdict
from datetime import datetime, timedelta
from functools import lru_cache
from urllib.parse import quote, urlparse

from sqlalchemy import inspect
from tornado import gen, web
from tornado.httputil import urlencode
from tornado.log import app_log

from . import orm, roles, scopes
from ._version import __version__, _check_version
from .crypto import CryptKeeper, EncryptionUnavailable, InvalidToken, decrypt, encrypt
from .metrics import RUNNING_SERVERS, TOTAL_USERS
from .objects import Server
from .spawner import LocalProcessSpawner
from .utils import (
    AnyTimeoutError,
    make_ssl_context,
    maybe_future,
    url_escape_path,
    url_path_join,
)

# detailed messages about the most common failure-to-start errors,
# which manifest timeouts during start
start_timeout_message = """
Common causes of this timeout, and debugging tips:

1. Everything is working, but it took too long.
   To fix: increase `Spawner.start_timeout` configuration
   to a number of seconds that is enough for spawners to finish starting.
2. The server didn't finish starting,
   or it crashed due to a configuration issue.
   Check the single-user server's logs for hints at what needs fixing.
"""

http_timeout_message = """
Common causes of this timeout, and debugging tips:

1. The server didn't finish starting,
   or it crashed due to a configuration issue.
   Check the single-user server's logs for hints at what needs fixing.
2. The server started, but is not accessible at the specified URL.
   This may be a configuration issue specific to your chosen Spawner.
   Check the single-user server logs and resource to make sure the URL
   is correct and accessible from the Hub.
3. (unlikely) Everything is working, but the server took too long to respond.
   To fix: increase `Spawner.http_timeout` configuration
   to a number of seconds that is enough for servers to become responsive.
"""

# set of chars that are safe in dns labels
# (allow '.' because we don't mind multiple levels of subdomains)
_dns_safe = set(string.ascii_letters + string.digits + '-.')
# don't escape % because it's the escape char and we handle it separately
_dns_needs_replace = _dns_safe | {"%"}


@lru_cache()
def _dns_quote(name):
    """Escape a name for use in a dns label

    this is _NOT_ fully domain-safe, but works often enough for realistic usernames.
    Fully safe would be full IDNA encoding,
    PLUS escaping non-IDNA-legal ascii,
    PLUS some encoding of boundary conditions
    """
    # escape name for subdomain label
    label = quote(name, safe="").lower()
    # some characters are not handled by quote,
    # because they are legal in URLs but not domains,
    # specifically _ and ~ (starting in 3.7).
    # Escape these in the same way (%{hex_codepoint}).
    unique_chars = set(label)
    for c in unique_chars:
        if c not in _dns_needs_replace:
            label = label.replace(c, f"%{ord(c):x}")

    # underscore is our escape char -
    # it's not officially legal in hostnames,
    # but is valid in _domain_ names (?),
    # and always works in practice.
    # FIXME: We should consider switching to proper IDNA encoding
    # for 3.0.
    label = label.replace("%", "_")
    return label


class UserDict(dict):
    """Like defaultdict, but for users

    Users can be retrieved by:

    - integer database id
    - orm.User object
    - username str

    A User wrapper object is always returned.

    This dict contains at least all active users,
    but not necessarily all users in the database.

    Checking `key in userdict` returns whether
    an item is already in the cache,
    *not* whether it is in the database.

    .. versionchanged:: 1.2
        ``'username' in userdict`` pattern is now supported
    """

    def __init__(self, db_factory, settings):
        self.db_factory = db_factory
        self.settings = settings
        super().__init__()

    @property
    def db(self):
        return self.db_factory()

    def from_orm(self, orm_user):
        return User(orm_user, self.settings)

    def add(self, orm_user):
        """Add a user to the UserDict"""
        if orm_user.id not in self:
            self[orm_user.id] = self.from_orm(orm_user)
        return self[orm_user.id]

    def __contains__(self, key):
        """key in userdict checks presence in the cache

        it does not check if the user is in the database
        """
        if isinstance(key, (User, orm.User)):
            key = key.id
        elif isinstance(key, str):
            # username lookup, O(N)
            for user in self.values():
                if user.name == key:
                    key = user.id
                    break
        return super().__contains__(key)

    def __getitem__(self, key):
        """UserDict allows retrieval of user by any of:

        - User object
        - orm.User object
        - username (str)
        - orm.User.id int (actual key used in underlying dict)
        """
        if isinstance(key, User):
            key = key.id
        elif isinstance(key, str):
            orm_user = self.db.query(orm.User).filter(orm.User.name == key).first()
            if orm_user is None:
                raise KeyError("No such user: %s" % key)
            else:
                key = orm_user.id
        if isinstance(key, orm.User):
            # users[orm_user] returns User(orm_user)
            orm_user = key
            if orm_user.id not in self:
                user = self[orm_user.id] = User(orm_user, self.settings)
                return user
            user = super().__getitem__(orm_user.id)
            user.db = self.db
            return user
        elif isinstance(key, int):
            id = key
            if id not in self:
                orm_user = self.db.query(orm.User).filter(orm.User.id == id).first()
                if orm_user is None:
                    raise KeyError("No such user: %s" % id)
                user = self.add(orm_user)
            else:
                user = super().__getitem__(id)
            return user
        else:
            raise KeyError(repr(key))

    def get(self, key, default=None):
        """Retrieve a User object if it can be found, else default

        Lookup can be by User object, id, or name

        .. versionchanged:: 1.2
            ``get()`` accesses the database instead of just the cache by integer id,
            so is equivalent to catching KeyErrors on attempted lookup.
        """
        try:
            return self[key]
        except KeyError:
            return default

    def __delitem__(self, key):
        user = self[key]
        for orm_spawner in user.orm_user._orm_spawners:
            if orm_spawner in self.db:
                self.db.expunge(orm_spawner)
        if user.orm_user in self.db:
            self.db.expunge(user.orm_user)
        super().__delitem__(user.id)

    def delete(self, key):
        """Delete a user from the cache and the database"""
        user = self[key]
        user_id = user.id
        self.db.delete(user)
        self.db.commit()
        # delete from dict after commit
        TOTAL_USERS.dec()
        del self[user_id]

    def count_active_users(self):
        """Count the number of user servers that are active/pending/ready

        Returns dict with counts of active/pending/ready servers
        """
        counts = defaultdict(int)
        for user in self.values():
            for spawner in user.spawners.values():
                pending = spawner.pending
                if pending:
                    counts['pending'] += 1
                    counts[pending + '_pending'] += 1
                if spawner.active:
                    counts['active'] += 1
                if spawner.ready:
                    counts['ready'] += 1

        return counts


class _SpawnerDict(dict):
    def __init__(self, spawner_factory):
        self.spawner_factory = spawner_factory

    def __getitem__(self, key):
        if key not in self:
            self[key] = self.spawner_factory(key)
        return super().__getitem__(key)


class User:
    """High-level wrapper around an orm.User object"""

    # declare instance attributes
    db = None
    orm_user = None
    log = app_log
    settings = None
    _auth_refreshed = None

    def __init__(self, orm_user, settings=None, db=None):
        self.db = db or inspect(orm_user).session
        self.settings = settings or {}
        self.orm_user = orm_user

        self.allow_named_servers = self.settings.get('allow_named_servers', False)

        self.base_url = self.prefix = (
            url_path_join(self.settings.get('base_url', '/'), 'user', self.escaped_name)
            + '/'
        )

        self.spawners = _SpawnerDict(self._new_spawner)

        # ensure default spawner exists in the database
        if '' not in self.orm_user.orm_spawners:
            self._new_orm_spawner('')

    @property
    def authenticator(self):
        return self.settings.get('authenticator', None)

    @property
    def spawner_class(self):
        return self.settings.get('spawner_class', LocalProcessSpawner)

    def get_spawner(self, server_name="", replace_failed=False):
        """Get a spawner by name

        replace_failed governs whether a failed spawner should be replaced
        or returned (default: returned).

        .. versionadded:: 2.2
        """
        spawner = self.spawners[server_name]
        if replace_failed and spawner._failed:
            self.log.debug(f"Discarding failed spawner {spawner._log_name}")
            # remove failed spawner, create a new one
            self.spawners.pop(server_name)
            spawner = self.spawners[server_name]
        return spawner

    def sync_groups(self, group_names):
        """Synchronize groups with database"""

        current_groups = {g.name for g in self.orm_user.groups}
        new_groups = set(group_names)
        if current_groups == new_groups:
            # no change, nothing to do
            return

        # log group changes
        added_groups = new_groups.difference(current_groups)
        removed_groups = current_groups.difference(group_names)
        if added_groups:
            self.log.info(f"Adding user {self.name} to group(s): {added_groups}")
        if removed_groups:
            self.log.info(f"Removing user {self.name} from group(s): {removed_groups}")

        if group_names:
            groups = (
                self.db.query(orm.Group).filter(orm.Group.name.in_(new_groups)).all()
            )
            existing_groups = {g.name for g in groups}
            for group_name in added_groups:
                if group_name not in existing_groups:
                    # create groups that don't exist yet
                    self.log.info(
                        f"Creating new group {group_name} for user {self.name}"
                    )
                    group = orm.Group(name=group_name)
                    self.db.add(group)
                    groups.append(group)
            self.orm_user.groups = groups
        else:
            self.orm_user.groups = []
        self.db.commit()

    async def save_auth_state(self, auth_state):
        """Encrypt and store auth_state"""
        if auth_state is None:
            self.encrypted_auth_state = None
        else:
            self.encrypted_auth_state = await encrypt(auth_state)
        self.db.commit()

    async def get_auth_state(self):
        """Retrieve and decrypt auth_state for the user"""
        encrypted = self.encrypted_auth_state
        if encrypted is None:
            return None
        try:
            auth_state = await decrypt(encrypted)
        except (ValueError, InvalidToken, EncryptionUnavailable) as e:
            self.log.warning(
                "Failed to retrieve encrypted auth_state for %s because %s",
                self.name,
                e,
            )
            return
        # loading auth_state
        if auth_state:
            # Crypt has multiple keys, store again with new key for rotation.
            if len(CryptKeeper.instance().keys) > 1:
                await self.save_auth_state(auth_state)
        return auth_state

    async def delete_spawners(self):
        """Call spawner cleanup methods

        Allows the spawner to cleanup persistent resources
        """
        for name in self.orm_user.orm_spawners.keys():
            await self._delete_spawner(name)

    async def _delete_spawner(self, name_or_spawner):
        """Delete a single spawner"""
        # always ensure full Spawner
        # this may instantiate the Spawner if it wasn't already running,
        # just to delete it
        if isinstance(name_or_spawner, str):
            spawner = self.spawners[name_or_spawner]
        else:
            spawner = name_or_spawner

        if spawner.active:
            raise RuntimeError(
                f"Spawner {spawner._log_name} is active and cannot be deleted."
            )
        try:
            await maybe_future(spawner.delete_forever())
        except Exception as e:
            self.log.exception(
                f"Error cleaning up persistent resources on {spawner._log_name}"
            )

    def all_spawners(self, include_default=True):
        """Generator yielding all my spawners

        including those that are not running.

        Spawners that aren't running will be low-level orm.Spawner objects,
        while those that are will be higher-level Spawner wrapper objects.
        """

        for name, orm_spawner in sorted(self.orm_user.orm_spawners.items()):
            if name == '' and not include_default:
                continue
            if name and not self.allow_named_servers:
                continue
            if name in self.spawners:
                # yield wrapper if it exists (server may be active)
                yield self.spawners[name]
            else:
                # otherwise, yield low-level ORM object (server is not active)
                yield orm_spawner

    def _new_orm_spawner(self, server_name):
        """Create the low-level orm Spawner object"""
        orm_spawner = orm.Spawner(name=server_name)
        self.db.add(orm_spawner)
        orm_spawner.user = self.orm_user
        self.db.commit()
        assert server_name in self.orm_spawners
        return orm_spawner

    def _new_spawner(self, server_name, spawner_class=None, **kwargs):
        """Create a new spawner"""
        if spawner_class is None:
            spawner_class = self.spawner_class
        self.log.debug("Creating %s for %s:%s", spawner_class, self.name, server_name)

        orm_spawner = self.orm_spawners.get(server_name)
        if orm_spawner is None:
            orm_spawner = self._new_orm_spawner(server_name)
        if server_name == '' and self.state:
            # migrate user.state to spawner.state
            orm_spawner.state = self.state
            self.state = None

        # use fully quoted name for client_id because it will be used in cookie-name
        # self.escaped_name may contain @ which is legal in URLs but not cookie keys
        client_id = 'jupyterhub-user-%s' % quote(self.name)
        if server_name:
            client_id = f'{client_id}-{quote(server_name)}'

        trusted_alt_names = []
        trusted_alt_names.extend(self.settings.get('trusted_alt_names', []))
        if self.settings.get('subdomain_host'):
            trusted_alt_names.append('DNS:' + self.domain)

        spawn_kwargs = dict(
            user=self,
            orm_spawner=orm_spawner,
            hub=self.settings.get('hub'),
            authenticator=self.authenticator,
            config=self.settings.get('config'),
            proxy_spec=url_path_join(
                self.proxy_spec, url_escape_path(server_name), '/'
            ),
            _deprecated_db_session=self.db,
            oauth_client_id=client_id,
            cookie_options=self.settings.get('cookie_options', {}),
            cookie_host_prefix_enabled=self.settings.get(
                "cookie_host_prefix_enabled", False
            ),
            trusted_alt_names=trusted_alt_names,
            user_options=orm_spawner.user_options or {},
        )

        if self.settings.get('internal_ssl'):
            ssl_kwargs = dict(
                internal_ssl=self.settings.get('internal_ssl'),
                internal_trust_bundles=self.settings.get('internal_trust_bundles'),
                internal_certs_location=self.settings.get('internal_certs_location'),
            )
            spawn_kwargs.update(ssl_kwargs)

        # update with kwargs. Mainly for testing.
        spawn_kwargs.update(kwargs)
        spawner = spawner_class(**spawn_kwargs)
        spawner.load_state(orm_spawner.state or {})
        return spawner

    # singleton property, self.spawner maps onto spawner with empty server_name
    @property
    def spawner(self):
        return self.spawners['']

    @spawner.setter
    def spawner(self, spawner):
        self.spawners[''] = spawner

    # pass get/setattr to ORM user
    def __getattr__(self, attr):
        if hasattr(self.orm_user, attr):
            return getattr(self.orm_user, attr)
        else:
            raise AttributeError(attr)

    def __setattr__(self, attr, value):
        if not attr.startswith('_') and self.orm_user and hasattr(self.orm_user, attr):
            setattr(self.orm_user, attr, value)
        else:
            super().__setattr__(attr, value)

    def __repr__(self):
        return repr(self.orm_user)

    @property
    def running(self):
        """property for whether the user's default server is running"""
        if not self.spawners:
            return False
        return self.spawner.ready

    @property
    def active(self):
        """True if any server is active"""
        if not self.spawners:
            return False
        return any(s.active for s in self.spawners.values())

    @property
    def spawn_pending(self):
        warnings.warn(
            "User.spawn_pending is deprecated in JupyterHub 0.8. Use Spawner.pending",
            DeprecationWarning,
        )
        return self.spawner.pending == 'spawn'

    @property
    def stop_pending(self):
        warnings.warn(
            "User.stop_pending is deprecated in JupyterHub 0.8. Use Spawner.pending",
            DeprecationWarning,
        )
        return self.spawner.pending == 'stop'

    @property
    def server(self):
        return self.spawner.server

    @property
    def escaped_name(self):
        """My name, escaped for use in URLs, cookies, etc."""
        return url_escape_path(self.name)

    @property
    def json_escaped_name(self):
        """The user name, escaped for use in javascript inserts, etc."""
        return json.dumps(self.name)[1:-1]

    @property
    def proxy_spec(self):
        """The proxy routespec for my default server"""
        if self.settings.get('subdomain_host'):
            return url_path_join(self.domain, self.base_url, '/')
        else:
            return url_path_join(self.base_url, '/')

    @property
    def domain(self):
        """Get the domain for my server."""

        return _dns_quote(self.name) + '.' + self.settings['domain']

    @property
    def host(self):
        """Get the *host* for my server (proto://domain[:port])"""
        # FIXME: escaped_name probably isn't escaped enough in general for a domain fragment
        parsed = urlparse(self.settings['subdomain_host'])
        h = f'{parsed.scheme}://{self.domain}'
        if parsed.port:
            h += ':%i' % parsed.port
        return h

    @property
    def url(self):
        """My URL

        Full name.domain/path if using subdomains, otherwise just my /base/url
        """
        if self.settings.get('subdomain_host'):
            return f'{self.host}{self.base_url}'
        else:
            return self.base_url

    def server_url(self, server_name=''):
        """Get the url for a server with a given name"""
        if not server_name:
            return self.url
        else:
            return url_path_join(self.url, url_escape_path(server_name), "/")

    def progress_url(self, server_name=''):
        """API URL for progress endpoint for a server with a given name"""
        url_parts = [self.settings['hub'].base_url, 'api/users', self.escaped_name]
        if server_name:
            url_parts.extend(['servers', url_escape_path(server_name), 'progress'])
        else:
            url_parts.extend(['server/progress'])
        return url_path_join(*url_parts)

    async def refresh_auth(self, handler):
        """Refresh authentication if needed

        Checks authentication expiry and refresh it if needed.
        See Spawner.

        If the auth is expired and cannot be refreshed
        without forcing a new login, a few things can happen:

        1. if this is a normal user spawn,
           the user should be redirected to login
           and back to spawn after login.
        2. if this is a spawn via API or other user,
           spawn will fail until the user logs in again.

        Args:
            handler (RequestHandler):
                The handler for the request triggering the spawn.
                May be None
        """
        authenticator = self.authenticator
        if authenticator is None or not authenticator.refresh_pre_spawn:
            # nothing to do
            return

        # refresh auth
        auth_user = await handler.refresh_auth(self, force=True)

        if auth_user:
            # auth refreshed, all done
            return

        # if we got to here, auth is expired and couldn't be refreshed
        self.log.error(
            "Auth expired for %s; cannot spawn until they login again", self.name
        )
        # auth expired, cannot spawn without a fresh login
        # it's the current user *and* spawn via GET, trigger login redirect
        if handler.request.method == 'GET' and handler.current_user is self:
            self.log.info("Redirecting %s to login to refresh auth", self.name)
            url = self.get_login_url()
            next_url = self.request.uri
            sep = '&' if '?' in url else '?'
            url += sep + urlencode(dict(next=next_url))
            self.redirect(url)
            raise web.Finish()
        else:
            # spawn via POST or on behalf of another user.
            # nothing we can do here but fail
            raise web.HTTPError(400, f"{self.name}'s authentication has expired")

    async def spawn(self, server_name='', options=None, handler=None):
        """Start the user's spawner

        depending from the value of JupyterHub.allow_named_servers

        if False:
        JupyterHub expects only one single-server per user
        url of the server will be /user/:name

        if True:
        JupyterHub expects more than one single-server per user
        url of the server will be /user/:name/:server_name
        """
        db = self.db

        if handler:
            await self.refresh_auth(handler)

        base_url = url_path_join(self.base_url, url_escape_path(server_name)) + '/'

        orm_server = orm.Server(base_url=base_url)
        db.add(orm_server)
        note = "Server at %s" % base_url
        db.commit()

        spawner = self.get_spawner(server_name, replace_failed=True)
        spawner.server = server = Server(orm_server=orm_server)
        assert spawner.orm_spawner.server is orm_server

        requested_scopes = spawner.server_token_scopes
        if callable(requested_scopes):
            requested_scopes = await maybe_future(requested_scopes(spawner))
        if not requested_scopes:
            # nothing requested, default to 'server' role
            requested_scopes = orm.Role.find(db, "server").scopes
        requested_scopes = set(requested_scopes)
        # resolve !server filter, which won't resolve elsewhere,
        # because this token is not owned by the server's own oauth client
        server_filter = f"={self.name}/{server_name}"
        requested_scopes = {
            scope + server_filter if scope.endswith("!server") else scope
            for scope in requested_scopes
        }
        # ensure activity scope is requested, since activity doesn't work without
        activity_scope = "users:activity!user"
        if not {activity_scope, "users:activity", "inherit"}.intersection(
            requested_scopes
        ):
            self.log.warning(
                f"Adding required scope {activity_scope} to server token, missing from Spawner.server_token_scopes. Please make sure to add it!"
            )
            requested_scopes |= {activity_scope}

        have_scopes = roles.roles_to_scopes(roles.get_roles_for(self.orm_user))
        have_scopes |= {"inherit"}
        jupyterhub_client = (
            db.query(orm.OAuthClient)
            .filter_by(
                identifier="jupyterhub",
            )
            .one()
        )

        resolved_scopes, excluded_scopes = scopes._resolve_requested_scopes(
            requested_scopes, have_scopes, self.orm_user, jupyterhub_client, db
        )
        if excluded_scopes:
            # what level should this be?
            # for admins-get-more use case, this is going to happen for most users
            # but for misconfiguration, folks will want to know!
            self.log.debug(
                "Not assigning requested scopes for %s: requested=%s, assigned=%s, excluded=%s",
                spawner._log_name,
                requested_scopes,
                resolved_scopes,
                excluded_scopes,
            )

        api_token = self.new_api_token(note=note, scopes=resolved_scopes)

        # pass requesting handler to the spawner
        # e.g. for processing GET params
        spawner.handler = handler

        # Passing user_options to the spawner
        if options is None:
            # options unspecified, load from db which should have the previous value
            options = spawner.orm_spawner.user_options or {}
        else:
            # options specified, save for use as future defaults
            spawner.orm_spawner.user_options = options
            db.commit()

        spawner.user_options = options
        # we are starting a new server, make sure it doesn't restore state
        spawner.clear_state()

        # create API and OAuth tokens
        spawner.api_token = api_token
        spawner.admin_access = self.settings.get('admin_access', False)
        client_id = spawner.oauth_client_id
        oauth_provider = self.settings.get('oauth_provider')
        if oauth_provider:
            allowed_scopes = await spawner._get_oauth_client_allowed_scopes()
            oauth_client = oauth_provider.add_client(
                client_id,
                api_token,
                url_path_join(self.url, url_escape_path(server_name), 'oauth_callback'),
                allowed_scopes=allowed_scopes,
                description="Server at %s"
                % (url_path_join(self.base_url, server_name) + '/'),
            )
            spawner.orm_spawner.oauth_client = oauth_client
        db.commit()

        # trigger pre-spawn hook on authenticator
        authenticator = self.authenticator
        try:
            spawner._start_pending = True

            if authenticator:
                # pre_spawn_start can throw errors that can lead to a redirect loop
                # if left uncaught (see https://github.com/jupyterhub/jupyterhub/issues/2683)
                await maybe_future(authenticator.pre_spawn_start(self, spawner))

            # trigger auth_state hook
            auth_state = await self.get_auth_state()
            await spawner.run_auth_state_hook(auth_state)

            # update spawner start time, and activity for both spawner and user
            self.last_activity = spawner.orm_spawner.started = (
                spawner.orm_spawner.last_activity
            ) = datetime.utcnow()
            db.commit()
            # wait for spawner.start to return
            # run optional preparation work to bootstrap the notebook
            await maybe_future(spawner.run_pre_spawn_hook())
            if self.settings.get('internal_ssl'):
                self.log.debug("Creating internal SSL certs for %s", spawner._log_name)
                hub_paths = await maybe_future(spawner.create_certs())
                spawner.cert_paths = await maybe_future(spawner.move_certs(hub_paths))
            self.log.debug("Calling Spawner.start for %s", spawner._log_name)
            f = maybe_future(spawner.start())
            # commit any changes in spawner.start (always commit db changes before yield)
            db.commit()
            url = await gen.with_timeout(timedelta(seconds=spawner.start_timeout), f)
            if url:
                # get ip, port info from return value of start()
                if isinstance(url, str):
                    # >= 0.9 can return a full URL string
                    pass
                else:
                    # >= 0.7 returns (ip, port)
                    proto = 'https' if self.settings['internal_ssl'] else 'http'

                    # check if spawner returned an IPv6 address
                    if ':' in url[0]:
                        url = '%s://[%s]:%i' % ((proto,) + url)
                    else:
                        url = '%s://%s:%i' % ((proto,) + url)
                urlinfo = urlparse(url)
                server.proto = urlinfo.scheme
                server.ip = urlinfo.hostname
                port = urlinfo.port
                if not port:
                    if urlinfo.scheme == 'https':
                        port = 443
                    else:
                        port = 80
                server.port = port
                db.commit()
            else:
                # prior to 0.7, spawners had to store this info in user.server themselves.
                # Handle < 0.7 behavior with a warning, assuming info was stored in db by the Spawner.
                self.log.warning(
                    "DEPRECATION: Spawner.start should return a url or (ip, port) tuple in JupyterHub >= 0.9"
                )
            if spawner.api_token and spawner.api_token != api_token:
                # Spawner re-used an API token, discard the unused api_token
                orm_token = orm.APIToken.find(self.db, api_token)
                if orm_token is not None:
                    self.db.delete(orm_token)
                    self.db.commit()
                # check if the re-used API token is valid
                found = orm.APIToken.find(self.db, spawner.api_token)
                if found:
                    if found.user is not self.orm_user:
                        self.log.error(
                            "%s's server is using %s's token! Revoking this token.",
                            self.name,
                            (found.user or found.service).name,
                        )
                        self.db.delete(found)
                        self.db.commit()
                        raise ValueError("Invalid token for %s!" % self.name)
                else:
                    # Spawner.api_token has changed, but isn't in the db.
                    # What happened? Maybe something unclean in a resumed container.
                    self.log.warning(
                        "%s's server specified its own API token that's not in the database",
                        self.name,
                    )
                    # use generated=False because we don't trust this token
                    # to have been generated properly
                    self.new_api_token(
                        spawner.api_token,
                        generated=False,
                        note="retrieved from spawner %s" % server_name,
                        scopes=resolved_scopes,
                    )
                # update OAuth client secret with updated API token
                if oauth_provider:
                    oauth_provider.add_client(
                        client_id,
                        spawner.api_token,
                        url_path_join(
                            self.url, url_escape_path(server_name), 'oauth_callback'
                        ),
                    )
                    db.commit()

        except Exception as e:
            if isinstance(e, AnyTimeoutError):
                self.log.warning(
                    f"{self.name}'s server failed to start"
                    f" in {spawner.start_timeout} seconds, giving up."
                    f"\n{start_timeout_message}"
                )
                e.reason = 'timeout'
                self.settings['statsd'].incr('spawner.failure.timeout')
            else:
                self.log.exception(
                    "Unhandled error starting {user}'s server: {error}".format(
                        user=self.name, error=e
                    )
                )
                self.settings['statsd'].incr('spawner.failure.error')
                e.reason = 'error'
            try:
                await self.stop(spawner.name)
            except Exception:
                self.log.exception(
                    "Failed to cleanup {user}'s server that failed to start".format(
                        user=self.name
                    ),
                    exc_info=True,
                )
            # raise original exception
            spawner._start_pending = False
            raise e
        finally:
            # clear reference to handler after start finishes
            spawner.handler = None
        spawner.start_polling()

        # store state
        if self.state is None:
            self.state = {}
        spawner.orm_spawner.state = spawner.get_state()
        db.commit()
        spawner._waiting_for_response = True
        await self._wait_up(spawner)

    async def _wait_up(self, spawner):
        """Wait for a server to finish starting.

        Shuts the server down if it doesn't respond within
        spawner.http_timeout.
        """
        server = spawner.server
        key = self.settings.get('internal_ssl_key')
        cert = self.settings.get('internal_ssl_cert')
        ca = self.settings.get('internal_ssl_ca')
        ssl_context = make_ssl_context(key, cert, cafile=ca)
        try:
            resp = await server.wait_up(
                http=True, timeout=spawner.http_timeout, ssl_context=ssl_context
            )
        except Exception as e:
            if isinstance(e, AnyTimeoutError):
                self.log.warning(
                    f"{self.name}'s server never showed up at {server.url}"
                    f" after {spawner.http_timeout} seconds. Giving up."
                    f"\n{http_timeout_message}"
                )
                e.reason = 'timeout'
                self.settings['statsd'].incr('spawner.failure.http_timeout')
            else:
                e.reason = 'error'
                self.log.exception(
                    "Unhandled error waiting for {user}'s server to show up at {url}: {error}".format(
                        user=self.name, url=server.url, error=e
                    )
                )
                self.settings['statsd'].incr('spawner.failure.http_error')
            try:
                await self.stop(spawner.name)
            except Exception:
                self.log.exception(
                    "Failed to cleanup {user}'s server that failed to start".format(
                        user=self.name
                    ),
                    exc_info=True,
                )
            # raise original TimeoutError
            raise e
        else:
            server_version = resp.headers.get('X-JupyterHub-Version')
            _check_version(__version__, server_version, self.log)
            # record the Spawner version for better error messages
            # if it doesn't work
            spawner._jupyterhub_version = server_version
        finally:
            spawner._waiting_for_response = False
            spawner._start_pending = False
        return spawner

    async def stop(self, server_name=''):
        """Stop the user's spawner

        and cleanup after it.
        """
        spawner = self.spawners[server_name]
        spawner._spawn_pending = False
        spawner._start_pending = False
        spawner._check_pending = False
        spawner.stop_polling()
        spawner._stop_pending = True

        self.log.debug("Stopping %s", spawner._log_name)

        try:
            api_token = spawner.api_token
            status = await spawner.poll()
            if status is None:
                await spawner.stop()
            self.last_activity = spawner.orm_spawner.last_activity = datetime.utcnow()
            # remove server entry from db
            spawner.server = None
            if not spawner.will_resume:
                # find and remove the API token and oauth client if the spawner isn't
                # going to re-use it next time
                orm_token = orm.APIToken.find(self.db, api_token)
                if orm_token:
                    self.db.delete(orm_token)
                # remove oauth client as well
                for oauth_client in self.db.query(orm.OAuthClient).filter_by(
                    identifier=spawner.oauth_client_id,
                ):
                    self.log.debug("Deleting oauth client %s", oauth_client.identifier)
                    self.db.delete(oauth_client)
            self.db.commit()
            self.log.debug("Finished stopping %s", spawner._log_name)
            RUNNING_SERVERS.dec()
        finally:
            spawner.server = None
            spawner.orm_spawner.started = None
            self.db.commit()
            # trigger post-stop hook
            try:
                await maybe_future(spawner.run_post_stop_hook())
            except:
                self.log.exception("Error in Spawner.post_stop_hook for %s", self)
            spawner.clear_state()
            spawner.orm_spawner.state = spawner.get_state()
            self.db.commit()

            # trigger post-spawner hook on authenticator
            auth = spawner.authenticator
            try:
                if auth:
                    await maybe_future(auth.post_spawn_stop(self, spawner))
            except Exception:
                self.log.exception(
                    "Error in Authenticator.post_spawn_stop for %s", self
                )
            spawner._stop_pending = False
            if not (
                spawner._spawn_future
                and (
                    not spawner._spawn_future.done()
                    or spawner._spawn_future.exception()
                )
            ):
                # pop Spawner *unless* it's stopping due to an error
                # because some pages serve latest-spawn error messages
                self.spawners.pop(server_name)