release 1.2.2

Backport PR #3284 on branch 1.2.x (Changelog for 1.2.2)

.circleci/config.yml

@@ -19,52 +19,11 @@ jobs:
           name: smoke test jupyterhub
           command: |
             docker run --rm -it jupyterhub/jupyterhub jupyterhub --help
-
-  docs:
-    # This is the base environment that Circle will use
-    docker:
-      - image: circleci/python:3.6-stretch
-    steps:
-      # Get our data and merge with upstream
-      - run: sudo apt-get update
-      - checkout
-      # Update our path
-      - run: echo "export PATH=~/.local/bin:$PATH" >> $BASH_ENV
-      # Restore cached files to speed things up
-      - restore_cache:
-          keys:
-            - cache-pip
-      # Install the packages needed to build our documentation
       - run:
-          name: Install NodeJS
+          name: verify static files
           command: |
-            # From https://github.com/nodesource/distributions/blob/master/README.md#debinstall
-            curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash -
-            sudo apt-get install -y nodejs
+            docker run --rm -it -v $PWD/dockerfiles:/io jupyterhub/jupyterhub python3 /io/test.py
 
-      - run:
-          name: Install dependencies
-          command: |
-            python3 -m pip install --user -r dev-requirements.txt
-            python3 -m pip install --user -r docs/requirements.txt
-            sudo npm install -g configurable-http-proxy
-            sudo python3 -m pip install --editable .
-
-      # Cache some files for a speedup in subsequent builds
-      - save_cache:
-          key: cache-pip
-          paths:
-            - ~/.cache/pip
-      # Build the docs
-      - run:
-          name: Build docs to store
-          command: |
-            cd docs
-            make html
-      # Tell Circle to store the documentation output in a folder that we can access later
-      - store_artifacts:
-          path: docs/build/html/
-          destination: html
 
 # Tell CircleCI to use this workflow when it builds the site
 workflows:
@@ -72,4 +31,3 @@ workflows:
   default:
     jobs:
       - build
-      - docs

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,39 +0,0 @@
----
-name: Issue report
-about: Create a report to help us improve
-
----
-
-<!---
-Hi! Thanks for using JupyterHub.
-
-If you are reporting an issue with JupyterHub, please use the GitHub search feature to check if your issue has been asked already. If it has, please add your comments to the existing issue.
-
-Some tips:
-- Running `jupyter troubleshoot` from the command line, if possible, and posting
-its output would also be helpful.
-- Running JupyterHub in `--debug` mode (`jupyterhub --debug`) can also be helpful for troubleshooting.
---->
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-<!---Add description here--->
-
-**To Reproduce**
-<!---
-Please share the steps to reproduce the behavior:
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
---->
-
-**Expected behavior**
-<!---
-A clear and concise description of what you expected to happen.
---->
-
-**Compute Information**
- - Operating System
- - JupyterHub Version [e.g. 22]

.github/ISSUE_TEMPLATE/installation-and-configuration-issues.md

@@ -1,22 +0,0 @@
----
-name: Installation and configuration questions
-about: Installation and configuration assistance
-
----
-
-<!---
-If you are reading this message, you have probably already searched the existing
-GitHub issues for JupyterHub. If you haven't tried a search, we encourage you to do so.
-
-If you are unsure where to ask your question (Jupyter, JupyterHub, JupyterLab, etc.),
-please ask on our [Discourse Q&A channel](https://discourse.jupyter.org/c/questions).
-
-If you have a quick question about JupyterHub installation or configuratation, you
-may ask on the [JupyterHub gitter channel](https://gitter.im/jupyterhub/jupyterhub).
-
-:sunny: Please be patient. We are volunteers and will address your question when we are able. :sunny:
-
-If after trying the above steps, you still have an in-depth installation or
-configuration question, such as a possible bug, please file an issue below and include
-any relevant details.
---->

.github/workflows/test.yml

@@ -0,0 +1,206 @@
+# This is a GitHub workflow defining a set of jobs with a set of steps.
+# ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions
+#
+name: Run tests
+
+# Trigger the workflow's on all PRs but only on pushed tags or commits to
+# main/master branch to avoid PRs developed in a GitHub fork's dedicated branch
+# to trigger.
+on:
+  pull_request:
+  push:
+
+defaults:
+  run:
+    # Declare bash be used by default in this workflow's "run" steps.
+    #
+    # NOTE: bash will by default run with:
+    #   --noprofile: Ignore ~/.profile etc.
+    #   --norc:      Ignore ~/.bashrc etc.
+    #   -e:          Exit directly on errors
+    #   -o pipefail: Don't mask errors from a command piped into another command
+    shell: bash
+
+env:
+  # UTF-8 content may be interpreted as ascii and causes errors without this.
+  LANG: C.UTF-8
+
+jobs:
+  # Run "pre-commit run --all-files"
+  pre-commit:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 2
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      # ref: https://github.com/pre-commit/action
+      - uses: pre-commit/action@v2.0.0
+      - name: Help message if pre-commit fail
+        if: ${{ failure() }}
+        run: |
+          echo "You can install pre-commit hooks to automatically run formatting"
+          echo "on each commit with:"
+          echo "    pre-commit install"
+          echo "or you can run by hand on staged files with"
+          echo "    pre-commit run"
+          echo "or after-the-fact on already committed files with"
+          echo "    pre-commit run --all-files"
+
+
+  # Run "pytest jupyterhub/tests" in various configurations
+  pytest:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 10
+
+    strategy:
+      # Keep running even if one variation of the job fail
+      fail-fast: false
+      matrix:
+        # We run this job multiple times with different parameterization
+        # specified below, these parameters have no meaning on their own and
+        # gain meaning on how job steps use them.
+        #
+        # subdomain:
+        #   Tests everything when JupyterHub is configured to add routes for
+        #   users with dedicated subdomains like user1.jupyter.example.com
+        #   rather than jupyter.example.com/user/user1.
+        #
+        # db: [mysql/postgres]
+        #   Tests everything when JupyterHub works against a dedicated mysql or
+        #   postgresql server.
+        #
+        # jupyter_server:
+        #   Tests everything when the user instances are started with
+        #   jupyter_server instead of notebook.
+        #
+        # main_dependencies:
+        #   Tests everything when the we use the latest available dependencies
+        #   from: ipytraitlets.
+        #
+        # NOTE: Since only the value of these parameters are presented in the
+        #       GitHub UI when the workflow run, we avoid using true/false as
+        #       values by instead duplicating the name to signal true.
+        include:
+          - python: "3.6"
+            subdomain: subdomain
+          - python: "3.7"
+            db: mysql
+          - python: "3.8"
+            db: postgres
+          - python: "3.8"
+            jupyter_server: jupyter_server
+          - python: "3.9"
+            main_dependencies: main_dependencies
+
+    steps:
+      # NOTE: In GitHub workflows, environment variables are set by writing
+      #       assignment statements to a file. They will be set in the following
+      #       steps as if would used `export MY_ENV=my-value`.
+      - name: Configure environment variables
+        run: |
+          if [ "${{ matrix.subdomain }}" != "" ]; then
+              echo "JUPYTERHUB_TEST_SUBDOMAIN_HOST=http://localhost.jovyan.org:8000" >> $GITHUB_ENV
+          fi
+          if [ "${{ matrix.db }}" == "mysql" ]; then
+              echo "MYSQL_HOST=127.0.0.1" >> $GITHUB_ENV
+              echo "JUPYTERHUB_TEST_DB_URL=mysql+mysqlconnector://root@127.0.0.1:3306/jupyterhub" >> $GITHUB_ENV
+          fi
+          if [ "${{ matrix.db }}" == "postgres" ]; then
+              echo "PGHOST=127.0.0.1" >> $GITHUB_ENV
+              echo "PGUSER=test_user" >> $GITHUB_ENV
+              echo "PGPASSWORD=hub[test/:?" >> $GITHUB_ENV
+              echo "JUPYTERHUB_TEST_DB_URL=postgresql://test_user:hub%5Btest%2F%3A%3F@127.0.0.1:5432/jupyterhub" >> $GITHUB_ENV
+          fi
+          if [ "${{ matrix.jupyter_server }}" != "" ]; then
+              echo "JUPYTERHUB_SINGLEUSER_APP=jupyterhub.tests.mockserverapp.MockServerApp" >> $GITHUB_ENV
+          fi
+      - uses: actions/checkout@v2
+
+      # NOTE: actions/setup-node@v1 make use of a cache within the GitHub base
+      #       environment and setup in a fraction of a second.
+      - name: Install Node v14
+        uses: actions/setup-node@v1
+        with:
+          node-version: "14"
+      - name: Install Node dependencies
+        run: |
+          npm install
+          npm install -g configurable-http-proxy
+          npm list
+
+      # NOTE: actions/setup-python@v2 make use of a cache within the GitHub base
+      #       environment and setup in a fraction of a second.
+      - name: Install Python ${{ matrix.python }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install Python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install --upgrade . -r dev-requirements.txt
+
+          if [ "${{ matrix.main_dependencies }}" != "" ]; then
+              pip install git+https://github.com/ipython/traitlets#egg=traitlets --force
+          fi
+          if [ "${{ matrix.jupyter_server }}" != "" ]; then
+              pip uninstall notebook --yes
+              pip install jupyter_server
+          fi
+          if [ "${{ matrix.db }}" == "mysql" ]; then
+              pip install mysql-connector-python
+          fi
+          if [ "${{ matrix.db }}" == "postgres" ]; then
+              pip install psycopg2-binary
+          fi
+
+          pip freeze
+
+      # NOTE: If you need to debug this DB setup step, consider the following.
+      #
+      # 1. mysql/postgressql are database servers we start as docker containers,
+      #    and we use clients named mysql/psql.
+      #
+      # 2. When we start a database server we need to pass environment variables
+      #    explicitly as part of the `docker run` command. These environment
+      #    variables are named differently from the similarly named environment
+      #    variables used by the clients.
+      #
+      #    - mysql    server ref: https://hub.docker.com/_/mysql/
+      #    - mysql    client ref: https://dev.mysql.com/doc/refman/5.7/en/environment-variables.html
+      #    - postgres server ref: https://hub.docker.com/_/postgres/
+      #    - psql     client ref: https://www.postgresql.org/docs/9.5/libpq-envars.html
+      #
+      # 3. When we connect, they should use 127.0.0.1 rather than the
+      #    default way of connecting which leads to errors like below both for
+      #    mysql and postgresql unless we set MYSQL_HOST/PGHOST to 127.0.0.1.
+      #
+      #    - ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock' (2)
+      #
+      - name: Start a database server (${{ matrix.db }})
+        if: ${{ matrix.db }}
+        run: |
+          if [ "${{ matrix.db }}" == "mysql" ]; then
+              sudo apt-get update
+              sudo apt-get install -y mysql-client
+              DB=mysql bash ci/docker-db.sh
+              DB=mysql bash ci/init-db.sh
+          fi
+          if [ "${{ matrix.db }}" == "postgres" ]; then
+              sudo apt-get update
+              sudo apt-get install -y postgresql-client
+              DB=postgres bash ci/docker-db.sh
+              DB=postgres bash ci/init-db.sh
+          fi
+
+      - name: Run pytest
+        # FIXME: --color=yes explicitly set because:
+        #        https://github.com/actions/runner/issues/241
+        run: |
+          pytest -v --maxfail=2 --color=yes --cov=jupyterhub jupyterhub/tests
+      - name: Submit codecov report
+        run: |
+          codecov

.gitignore

@@ -24,5 +24,7 @@ MANIFEST
 .coverage.*
 htmlcov
 .idea/
+.vscode/
 .pytest_cache
 pip-wheel-metadata
+docs/source/reference/metrics.rst

.pre-commit-config.yaml

@@ -1,10 +1,9 @@
 repos:
 - repo: https://github.com/asottile/reorder_python_imports
-  rev: v1.8.0
+  rev: v1.9.0
   hooks:
   - id: reorder-python-imports
-    language_version: python3.6
-- repo: https://github.com/ambv/black
+- repo: https://github.com/psf/black
   rev: 19.10b0
   hooks:
   - id: black

.travis.yml

@@ -1,94 +0,0 @@
-dist: bionic
-language: python
-cache:
-  - pip
-env:
-  global:
-    - MYSQL_HOST=127.0.0.1
-    - MYSQL_TCP_PORT=13306
-
-# request additional services for the jobs to access
-services:
-  - postgresql
-  - docker
-
-# install dependencies for running pytest (but not linting)
-before_install:
-  - set -e
-  - nvm install 6; nvm use 6
-  - npm install
-  - npm install -g configurable-http-proxy
-  - |
-    # setup database
-    if [[ $JUPYTERHUB_TEST_DB_URL == mysql* ]]; then
-      unset MYSQL_UNIX_PORT
-      DB=mysql bash ci/docker-db.sh
-      DB=mysql bash ci/init-db.sh
-      # FIXME: mysql-connector-python 8.0.16 incorrectly decodes bytes to str
-      # ref: https://bugs.mysql.com/bug.php?id=94944
-      pip install 'mysql-connector-python==8.0.11'
-    elif [[ $JUPYTERHUB_TEST_DB_URL == postgresql* ]]; then
-      psql -c "CREATE USER $PGUSER WITH PASSWORD '$PGPASSWORD';" -U postgres
-      DB=postgres bash ci/init-db.sh
-      pip install psycopg2-binary
-    fi
-
-# install general dependencies
-install:
-  - pip install --upgrade pip
-  - pip install --upgrade --pre -r dev-requirements.txt .
-  - pip freeze
-
-# run tests
-script:
-  - pytest -v --maxfail=2 --cov=jupyterhub jupyterhub/tests
-
-# collect test coverage information
-after_success:
-  - codecov
-
-# list the jobs
-jobs:
-  include:
-    - name: autoformatting check
-      python: 3.6
-      # NOTE: It does not suffice to override to: null, [], or [""]. Travis will
-      #       fall back to the default if we do.
-      before_install: echo "Do nothing before install."
-      script:
-        - pre-commit run --all-files
-      after_success: echo "Do nothing after success."
-      after_failure:
-        - |
-          echo "You can install pre-commit hooks to automatically run formatting"
-          echo "on each commit with:"
-          echo "    pre-commit install"
-          echo "or you can run by hand on staged files with"
-          echo "    pre-commit run"
-          echo "or after-the-fact on already committed files with"
-          echo "    pre-commit run --all-files"
-    # When we run pytest, we want to run it with python>=3.5 as well as with
-    # various configurations. We increment the python version at the same time
-    # as we test new configurations in order to reduce the number of test jobs.
-    - name: python:3.5 + dist:xenial
-      python: 3.5
-      dist: xenial
-    - name: python:3.6 + subdomain
-      python: 3.6
-      env: JUPYTERHUB_TEST_SUBDOMAIN_HOST=http://localhost.jovyan.org:8000
-    - name: python:3.7 + mysql
-      python: 3.7
-      env:
-        - JUPYTERHUB_TEST_DB_URL=mysql+mysqlconnector://root@127.0.0.1:$MYSQL_TCP_PORT/jupyterhub
-    - name: python:3.8 + postgresql
-      python: 3.8
-      env:
-        - PGUSER=jupyterhub
-        - PGPASSWORD=hub[test/:?
-        # The password in url below is url-encoded with: urllib.parse.quote($PGPASSWORD, safe='')
-        - JUPYTERHUB_TEST_DB_URL=postgresql://jupyterhub:hub%5Btest%2F%3A%3F@127.0.0.1/jupyterhub
-    - name: python:nightly
-      python: nightly
-  allow_failures:
-    - name: python:nightly
-  fast_finish: true

CONTRIBUTING.md

@@ -8,6 +8,12 @@ for a friendly and welcoming collaborative environment.
 
 ## Setting up a development environment
 
+<!--
+https://jupyterhub.readthedocs.io/en/stable/contributing/setup.html
+contains a lot of the same information. Should we merge the docs and
+just have this page link to that one?
+-->
+
 JupyterHub requires Python >= 3.5 and nodejs.
 
 As a Python project, a development install of JupyterHub follows standard practices for the basics (steps 1-2).
@@ -60,7 +66,7 @@ pre-commit run
 
 which should run any autoformatting on your code
 and tell you about any errors it couldn't fix automatically.
-You may also install [black integration](https://github.com/ambv/black#editor-integration)
+You may also install [black integration](https://github.com/psf/black#editor-integration)
 into your text editor to format code automatically.
 
 If you have already committed files before setting up the pre-commit
@@ -128,4 +134,4 @@ To read more about fixtures check out the
 [pytest docs](https://docs.pytest.org/en/latest/fixture.html)
 for how to use the existing fixtures, and how to create new ones.
 
-When in doubt, feel free to ask.
+When in doubt, feel free to [ask](https://gitter.im/jupyterhub/jupyterhub).

Dockerfile

@@ -21,8 +21,7 @@
 # your jupyterhub_config.py will be added automatically
 # from your docker directory.
 
-# https://github.com/tianon/docker-brew-ubuntu-core/commit/d4313e13366d24a97bd178db4450f63e221803f1
-ARG BASE_IMAGE=ubuntu:bionic-20191029@sha256:6e9f67fa63b0323e9a1e587fd71c561ba48a034504fb804fd26fd8800039835d
+ARG BASE_IMAGE=ubuntu:focal-20200729@sha256:6f2fb2f9fb5582f8b587837afd6ea8f37d8d1d9e41168c90f410a6ef15fa8ce5
 FROM $BASE_IMAGE AS builder
 
 USER root
@@ -41,19 +40,18 @@ RUN apt-get update \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*
 
-# copy only what we need to avoid unnecessary rebuilds
-COPY package.json \
-     pyproject.toml \
-     README.md \
-     requirements.txt \
-     setup.py \
-     /src/jupyterhub/
-COPY jupyterhub/ /src/jupyterhub/jupyterhub
-COPY share/ /src/jupyterhub/share
-
-WORKDIR /src/jupyterhub
 RUN python3 -m pip install --upgrade setuptools pip wheel
-RUN python3 -m pip wheel -v --wheel-dir wheelhouse .
+
+# copy everything except whats in .dockerignore, its a
+# compromise between needing to rebuild and maintaining
+# what needs to be part of the build
+COPY . /src/jupyterhub/
+WORKDIR /src/jupyterhub
+
+# Build client component packages (they will be copied into ./share and
+# packaged with the built wheel.)
+RUN python3 setup.py bdist_wheel
+RUN python3 -m pip wheel --wheel-dir wheelhouse dist/*.whl
 
 
 FROM $BASE_IMAGE
@@ -90,7 +88,6 @@ RUN npm install -g configurable-http-proxy@^4.2.0 \
 
 # install the wheels we built in the first stage
 COPY --from=builder /src/jupyterhub/wheelhouse /tmp/wheelhouse
-COPY --from=builder /src/jupyterhub/share /src/jupyterhub/share
 RUN python3 -m pip install --no-cache /tmp/wheelhouse/*
 
 RUN mkdir -p /srv/jupyterhub/

README.md

@@ -13,7 +13,7 @@
 [![Latest PyPI version](https://img.shields.io/pypi/v/jupyterhub?logo=pypi)](https://pypi.python.org/pypi/jupyterhub)
 [![Latest conda-forge version](https://img.shields.io/conda/vn/conda-forge/jupyterhub?logo=conda-forge)](https://www.npmjs.com/package/jupyterhub)
 [![Documentation build status](https://img.shields.io/readthedocs/jupyterhub?logo=read-the-docs)](https://jupyterhub.readthedocs.org/en/latest/)
-[![TravisCI build status](https://img.shields.io/travis/jupyterhub/jupyterhub/master?logo=travis)](https://travis-ci.org/jupyterhub/jupyterhub)
+[![TravisCI build status](https://img.shields.io/travis/com/jupyterhub/jupyterhub?logo=travis)](https://travis-ci.com/jupyterhub/jupyterhub)
 [![DockerHub build status](https://img.shields.io/docker/build/jupyterhub/jupyterhub?logo=docker&label=build)](https://hub.docker.com/r/jupyterhub/jupyterhub/tags)
 [![CircleCI build status](https://img.shields.io/circleci/build/github/jupyterhub/jupyterhub?logo=circleci)](https://circleci.com/gh/jupyterhub/jupyterhub)<!-- CircleCI Token: b5b65862eb2617b9a8d39e79340b0a6b816da8cc -->
 [![Test coverage of code](https://codecov.io/gh/jupyterhub/jupyterhub/branch/master/graph/badge.svg)](https://codecov.io/gh/jupyterhub/jupyterhub)
@@ -74,6 +74,7 @@ for administration of the Hub and its users.
     The `nodejs-legacy` package installs the `node` executable and is currently
     required for npm to work on Debian/Ubuntu.
 
+- If using the default PAM Authenticator, a [pluggable authentication module (PAM)](https://en.wikipedia.org/wiki/Pluggable_authentication_module).
 - TLS certificate and key for HTTPS communication
 - Domain name
 

ci/docker-db.sh

@@ -1,36 +1,55 @@
 #!/usr/bin/env bash
-# source this file to setup postgres and mysql
-# for local testing (as similar as possible to docker)
+# The goal of this script is to start a database server as a docker container.
+#
+# Required environment variables:
+# - DB: The database server to start, either "postgres" or "mysql".
+#
+# - PGUSER/PGPASSWORD: For the creation of a postgresql user with associated
+#   password.
 
 set -eu
 
-export MYSQL_HOST=127.0.0.1
-export MYSQL_TCP_PORT=${MYSQL_TCP_PORT:-13306}
-export PGHOST=127.0.0.1
-NAME="hub-test-$DB"
-DOCKER_RUN="docker run -d --name $NAME"
+# Stop and remove any existing database container
+DOCKER_CONTAINER="hub-test-$DB"
+docker rm -f "$DOCKER_CONTAINER" 2>/dev/null || true
 
-docker rm -f "$NAME" 2>/dev/null || true
-
-case "$DB" in
-"mysql")
-  RUN_ARGS="-e MYSQL_ALLOW_EMPTY_PASSWORD=1 -p $MYSQL_TCP_PORT:3306 mysql:5.7"
-  CHECK="mysql --host $MYSQL_HOST --port $MYSQL_TCP_PORT --user root -e \q"
-  ;;
-"postgres")
-  RUN_ARGS="-p 5432:5432 postgres:9.5"
-  CHECK="psql --user postgres -c \q"
-  ;;
-*)
+# Prepare environment variables to startup and await readiness of either a mysql
+# or postgresql server.
+if [[ "$DB" == "mysql" ]]; then
+    # Environment variables can influence both the mysql server in the docker
+    # container and the mysql client.
+    #
+    # ref server: https://hub.docker.com/_/mysql/
+    # ref client: https://dev.mysql.com/doc/refman/5.7/en/setting-environment-variables.html
+    #
+    DOCKER_RUN_ARGS="-p 3306:3306 --env MYSQL_ALLOW_EMPTY_PASSWORD=1 mysql:5.7"
+    READINESS_CHECK="mysql --user root --execute \q"
+elif [[ "$DB" == "postgres" ]]; then
+    # Environment variables can influence both the postgresql server in the
+    # docker container and the postgresql client (psql).
+    #
+    # ref server: https://hub.docker.com/_/postgres/
+    # ref client: https://www.postgresql.org/docs/9.5/libpq-envars.html
+    #
+    # POSTGRES_USER / POSTGRES_PASSWORD will create a user on startup of the
+    # postgres server, but PGUSER and PGPASSWORD are the environment variables
+    # used by the postgresql client psql, so we configure the user based on how
+    # we want to connect.
+    #
+    DOCKER_RUN_ARGS="-p 5432:5432 --env "POSTGRES_USER=${PGUSER}" --env "POSTGRES_PASSWORD=${PGPASSWORD}" postgres:9.5"
+    READINESS_CHECK="psql --command \q"
+else
     echo '$DB must be mysql or postgres'
     exit 1
-esac
+fi
 
-$DOCKER_RUN $RUN_ARGS
+# Start the database server
+docker run --detach --name "$DOCKER_CONTAINER" $DOCKER_RUN_ARGS
 
+# Wait for the database server to start
 echo -n "waiting for $DB "
 for i in {1..60}; do
-  if $CHECK; then
+    if $READINESS_CHECK; then
         echo 'done'
         break
     else
@@ -38,22 +57,4 @@ for i in {1..60}; do
         sleep 1
     fi
 done
-$CHECK
-
-case "$DB" in
-"mysql")
-  ;;
-"postgres")
-  # create the user
-  psql --user postgres -c "CREATE USER $PGUSER WITH PASSWORD '$PGPASSWORD';"
-  ;;
-*)
-esac
-
-echo -e "
-Set these environment variables:
-
-    export MYSQL_HOST=127.0.0.1
-    export MYSQL_TCP_PORT=$MYSQL_TCP_PORT
-    export PGHOST=127.0.0.1
-"
+$READINESS_CHECK

ci/init-db.sh

@@ -1,27 +1,26 @@
 #!/usr/bin/env bash
-# initialize jupyterhub databases for testing
+# The goal of this script is to initialize a running database server with clean
+# databases for use during tests.
+#
+# Required environment variables:
+# - DB: The database server to start, either "postgres" or "mysql".
 
 set -eu
 
-MYSQL="mysql --user root --host $MYSQL_HOST --port $MYSQL_TCP_PORT -e "
-PSQL="psql --user postgres -c "
-
-case "$DB" in
-"mysql")
-  EXTRA_CREATE='CHARACTER SET utf8 COLLATE utf8_general_ci'
-  SQL="$MYSQL"
-  ;;
-"postgres")
-  SQL="$PSQL"
-  ;;
-*)
+# Prepare env vars SQL_CLIENT and EXTRA_CREATE_DATABASE_ARGS
+if [[ "$DB" == "mysql" ]]; then
+    SQL_CLIENT="mysql --user root --execute "
+    EXTRA_CREATE_DATABASE_ARGS='CHARACTER SET utf8 COLLATE utf8_general_ci'
+elif [[ "$DB" == "postgres" ]]; then
+    SQL_CLIENT="psql --command "
+else
     echo '$DB must be mysql or postgres'
     exit 1
-esac
+fi
 
+# Configure a set of databases in the database server for upgrade tests
 set -x
-
 for SUFFIX in '' _upgrade_072 _upgrade_081 _upgrade_094; do
-    $SQL "DROP DATABASE jupyterhub${SUFFIX};" 2>/dev/null || true
-    $SQL "CREATE DATABASE jupyterhub${SUFFIX} ${EXTRA_CREATE:-};"
+    $SQL_CLIENT "DROP DATABASE jupyterhub${SUFFIX};" 2>/dev/null || true
+    $SQL_CLIENT "CREATE DATABASE jupyterhub${SUFFIX} ${EXTRA_CREATE_DATABASE_ARGS:-};"
 done

demo-image/Dockerfile

@@ -0,0 +1,16 @@
+# Demo JupyterHub Docker image
+#
+# This should only be used for demo or testing and not as a base image to build on.
+#
+# It includes the notebook package and it uses the DummyAuthenticator and the SimpleLocalProcessSpawner.
+ARG BASE_IMAGE=jupyterhub/jupyterhub-onbuild
+FROM ${BASE_IMAGE}
+
+# Install the notebook package
+RUN python3 -m pip install notebook
+
+# Create a demo user
+RUN useradd --create-home demo
+RUN chown demo .
+
+USER demo

demo-image/README.md

@@ -0,0 +1,25 @@
+## Demo Dockerfile
+
+This is a demo JupyterHub Docker image to help you get a quick overview of what
+JupyterHub is and how it works.
+
+It uses the SimpleLocalProcessSpawner to spawn new user servers and
+DummyAuthenticator for authentication.
+The DummyAuthenticator allows you to log in with any username & password and the
+SimpleLocalProcessSpawner allows starting servers without having to create a
+local user for each JupyterHub user.
+
+### Important!
+
+This should only be used for demo or testing purposes!
+It shouldn't be used as a base image to build on.
+
+### Try it
+1. `cd` to the root of your jupyterhub repo.
+
+2. Build the demo image with `docker build -t jupyterhub-demo demo-image`. 
+
+3. Run the demo image with `docker run -d -p 8000:8000 jupyterhub-demo`.
+
+4. Visit http://localhost:8000 and login with any username and password
+5. Happy demo-ing :tada:!

demo-image/jupyterhub_config.py

@@ -0,0 +1,7 @@
+# Configuration file for jupyterhub-demo
+
+c = get_config()
+
+# Use DummyAuthenticator and SimpleSpawner
+c.JupyterHub.spawner_class = "simple"
+c.JupyterHub.authenticator_class = "dummy"

dockerfiles/test.py

@@ -0,0 +1,9 @@
+import os
+
+from jupyterhub._data import DATA_FILES_PATH
+
+print(f"DATA_FILES_PATH={DATA_FILES_PATH}")
+
+for sub_path in ("templates", "static/components", "static/css/style.min.css"):
+    path = os.path.join(DATA_FILES_PATH, sub_path)
+    assert os.path.exists(path), path

docs/Makefile

@@ -48,6 +48,7 @@ help:
 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
 	@echo "  coverage   to run coverage check of the documentation (if enabled)"
 	@echo "  spelling   to run spell check on documentation"
+	@echo "  metrics    to generate documentation for metrics by inspecting the source code"
 
 clean:
 	rm -rf $(BUILDDIR)/*
@@ -60,7 +61,12 @@ rest-api: source/_static/rest-api/index.html
 source/_static/rest-api/index.html: rest-api.yml node_modules
 	npm run rest-api
 
-html: rest-api
+metrics: source/reference/metrics.rst
+
+source/reference/metrics.rst: generate-metrics.py
+	python3 generate-metrics.py
+
+html: rest-api metrics
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."

docs/environment.yml

@@ -1,20 +0,0 @@
-# ReadTheDocs uses the `environment.yaml` so make sure to update that as well
-# if you change the dependencies of JupyterHub in the various `requirements.txt`
-name: jhub_docs
-channels:
-  - conda-forge
-dependencies:
-- pip
-- nodejs
-- python=3.6
-- alembic
-- jinja2
-- pamela
-- recommonmark==0.6.0
-- requests
-- sqlalchemy>=1
-- tornado>=5.0
-- traitlets>=4.1
-- sphinx>=1.7
-- pip:
-  - -r requirements.txt

docs/generate-metrics.py

@@ -0,0 +1,57 @@
+import os
+from os.path import join
+
+from pytablewriter import RstSimpleTableWriter
+from pytablewriter.style import Style
+
+import jupyterhub.metrics
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+
+
+class Generator:
+    @classmethod
+    def create_writer(cls, table_name, headers, values):
+        writer = RstSimpleTableWriter()
+        writer.table_name = table_name
+        writer.headers = headers
+        writer.value_matrix = values
+        writer.margin = 1
+        [writer.set_style(header, Style(align="center")) for header in headers]
+        return writer
+
+    def _parse_metrics(self):
+        table_rows = []
+        for name in dir(jupyterhub.metrics):
+            obj = getattr(jupyterhub.metrics, name)
+            if obj.__class__.__module__.startswith('prometheus_client.'):
+                for metric in obj.describe():
+                    table_rows.append([metric.type, metric.name, metric.documentation])
+        return table_rows
+
+    def prometheus_metrics(self):
+        generated_directory = f"{HERE}/source/reference"
+        if not os.path.exists(generated_directory):
+            os.makedirs(generated_directory)
+
+        filename = f"{generated_directory}/metrics.rst"
+        table_name = ""
+        headers = ["Type", "Name", "Description"]
+        values = self._parse_metrics()
+        writer = self.create_writer(table_name, headers, values)
+
+        title = "List of Prometheus Metrics"
+        underline = "============================"
+        content = f"{title}\n{underline}\n{writer.dumps()}"
+        with open(filename, 'w') as f:
+            f.write(content)
+        print(f"Generated {filename}.")
+
+
+def main():
+    doc_generator = Generator()
+    doc_generator.prometheus_metrics()
+
+
+if __name__ == "__main__":
+    main()

docs/requirements.txt

@@ -1,10 +1,12 @@
-# ReadTheDocs uses the `environment.yaml` so make sure to update that as well
-# if you change this file
 -r ../requirements.txt
+
 alabaster_jupyterhub
-autodoc-traits
-git+https://github.com/pandas-dev/pandas-sphinx-theme.git@master
-recommonmark==0.5.0
+# Temporary fix of #3021. Revert back to released autodoc-traits when
+# 0.1.0 released.
+https://github.com/jupyterhub/autodoc-traits/archive/75885ee24636efbfebfceed1043459715049cd84.zip
+pydata-sphinx-theme
+pytablewriter>=0.56
+recommonmark>=0.6
 sphinx-copybutton
 sphinx-jsonschema
 sphinx>=1.7

docs/rest-api.yml

@@ -3,7 +3,7 @@ swagger: '2.0'
 info:
   title: JupyterHub
   description: The REST API for JupyterHub
-  version: 0.9.0dev
+  version: 1.2.0dev
   license:
     name: BSD-3-Clause
 schemes:
@@ -248,10 +248,13 @@ paths:
             when spawning via the API instead of spawn form.
             The structure of the options
             will depend on the Spawner's configuration.
+            The body itself will be available as `user_options` for the
+            Spawner.
           in: body
           required: false
           schema:
             type: object
+
       responses:
         '201':
           description: The user's notebook server has started
@@ -280,7 +283,10 @@ paths:
           required: true
           type: string
         - name: server_name
-          description: name given to a named-server
+          description: |
+            name given to a named-server. 
+            
+            Note that depending on your JupyterHub infrastructure there are chracterter size limitation to `server_name`. Default spawner with K8s pod will not allow Jupyter Notebooks to be spawned with a name that contains more than 253 characters (keep in mind that the pod will be spawned with extra characters to identify the user and hub).
           in: path
           required: true
           type: string
@@ -312,14 +318,18 @@ paths:
           in: path
           required: true
           type: string
-        - name: remove
-          description: |
-            Whether to fully remove the server, rather than just stop it.
-            Removing a server deletes things like the state of the stopped server.
+        - name: body
           in: body
           required: false
           schema:
+            type: object
+            properties:
+              remove:
                 type: boolean
+                description: |
+                  Whether to fully remove the server, rather than just stop it.
+                  Removing a server deletes things like the state of the stopped server.
+                  Default: false.
       responses:
         '204':
           description: The user's notebook named-server has stopped
@@ -795,6 +805,9 @@ definitions:
       state:
         type: object
         description: Arbitrary internal state from this server's spawner.  Only available on the hub's users list or get-user-by-name method, and only if a hub admin.  None otherwise.
+      user_options:
+        type: object
+        description: User specified options for the user's spawned instance of a single-user server.
   Group:
     type: object
     properties:

docs/source/_static/custom.css

@@ -1,106 +1,4 @@
-div#helm-chart-schema h2,
-div#helm-chart-schema h3,
-div#helm-chart-schema h4,
-div#helm-chart-schema h5,
-div#helm-chart-schema h6 {
-    font-family: courier new;
-}
-
-h3, h3 ~ * {
-  margin-left: 3% !important;
-}
-
-h4, h4 ~ * {
-  margin-left: 6% !important;
-}
-
-h5, h5 ~ * {
-  margin-left: 9% !important;
-}
-
-h6, h6 ~ * {
-  margin-left: 12% !important;
-}
-
-h7, h7 ~ * {
-  margin-left: 15% !important;
-}
-
-img.logo {
-  width:100%
-}
-
-.right-next {
-    float: right;
-    max-width: 45%;
-    overflow: auto;
-    text-overflow: ellipsis;
-    white-space: nowrap;
-}
-
-.right-next::after{
-    content: ' »';
-}
-
-.left-prev {
-    float: left;
-    max-width: 45%;
-    overflow: auto;
-    text-overflow: ellipsis;
-    white-space: nowrap;
-}
-
-.left-prev::before{
-    content: '« ';
-}
-
-.prev-next-bottom {
-  margin-top: 3em;
-}
-
-.prev-next-top {
-  margin-bottom: 1em;
-}
-
-/* Sidebar TOC and headers */
-
-div.sphinxsidebarwrapper div {
-    margin-bottom: .8em;
-}
-div.sphinxsidebar h3 {
-    font-size: 1.3em;
-    padding-top: 0px;
-    font-weight: 800;
-    margin-left: 0px !important;
-}
-
-div.sphinxsidebar p.caption {
-    font-size: 1.2em;
-    margin-bottom: 0px;
-    margin-left: 0px !important;
-    font-weight: 900;
-    color: #767676;
-}
-
-div.sphinxsidebar ul {
-    font-size: .8em;
-    margin-top: 0px;
-    padding-left: 3%;
-    margin-left: 0px !important;
-}
-
-div.relations ul {
-    font-size: 1em;
-    margin-left: 0px !important;
-}
-
-div#searchbox form {
-    margin-left: 0px !important;
-}
-
-/* body elements */
-.toctree-wrapper span.caption-text {
-    color: #767676;
-    font-style: italic;
-    font-weight: 300;
+/* Added to avoid logo being too squeezed */
+.navbar-brand {
+  height: 4rem !important;
 }

docs/source/_templates/navigation.html

@@ -1,16 +0,0 @@
-{# Custom template for navigation.html
-
-   alabaster theme does not provide blocks for titles to
-   be overridden so this custom theme handles title and
-   toctree for sidebar
-#}
-<h3>{{ _('Table of Contents') }}</h3>
-{{ toctree(includehidden=theme_sidebar_includehidden, collapse=theme_sidebar_collapse) }}
-{% if theme_extra_nav_links %}
-<hr />
-<ul>
-    {% for text, uri in theme_extra_nav_links.items() %}
-    <li class="toctree-l1"><a href="{{ uri }}">{{ text }}</a></li>
-    {% endfor %}
-</ul>
-{% endif %}

docs/source/_templates/relations.html

@@ -1,17 +0,0 @@
-{# Custom template for relations.html
-
-   alabaster theme does not provide previous/next page by default
-#}
-<div class="relations">
-<h3>Navigation</h3>
-<ul>
-  <li><a href="{{ pathto(master_doc) }}">Documentation Home</a><ul>
-  {%- if prev %}
-  <li><a href="{{ prev.link|e }}" title="Previous">Previous topic</a></li>
-  {%- endif %}
-  {%- if next %}
-  <li><a href="{{ next.link|e }}" title="Next">Next topic</a></li>
-  {%- endif %}
-  </ul>
-</ul>
-</div>

docs/source/conf.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 #
 import os
-import shlex
 import sys
 
 # Set paths
@@ -20,10 +19,9 @@ extensions = [
     'autodoc_traits',
     'sphinx_copybutton',
     'sphinx-jsonschema',
+    'recommonmark',
 ]
 
-templates_path = ['_templates']
-
 # The master toctree document.
 master_doc = 'index'
 
@@ -59,22 +57,74 @@ default_role = 'literal'
 import recommonmark
 from recommonmark.transform import AutoStructify
 
+# -- Config -------------------------------------------------------------
+from jupyterhub.app import JupyterHub
+from docutils import nodes
+from sphinx.directives.other import SphinxDirective
+from contextlib import redirect_stdout
+from io import StringIO
+
+# create a temp instance of JupyterHub just to get the output of the generate-config
+# and help --all commands.
+jupyterhub_app = JupyterHub()
+
+
+class ConfigDirective(SphinxDirective):
+    """Generate the configuration file output for use in the documentation."""
+
+    has_content = False
+    required_arguments = 0
+    optional_arguments = 0
+    final_argument_whitespace = False
+    option_spec = {}
+
+    def run(self):
+        # The generated configuration file for this version
+        generated_config = jupyterhub_app.generate_config_file()
+        # post-process output
+        home_dir = os.environ['HOME']
+        generated_config = generated_config.replace(home_dir, '$HOME', 1)
+        par = nodes.literal_block(text=generated_config)
+        return [par]
+
+
+class HelpAllDirective(SphinxDirective):
+    """Print the output of jupyterhub help --all for use in the documentation."""
+
+    has_content = False
+    required_arguments = 0
+    optional_arguments = 0
+    final_argument_whitespace = False
+    option_spec = {}
+
+    def run(self):
+        # The output of the help command for this version
+        buffer = StringIO()
+        with redirect_stdout(buffer):
+            jupyterhub_app.print_help('--help-all')
+        all_help = buffer.getvalue()
+        # post-process output
+        home_dir = os.environ['HOME']
+        all_help = all_help.replace(home_dir, '$HOME', 1)
+        par = nodes.literal_block(text=all_help)
+        return [par]
+
 
 def setup(app):
     app.add_config_value('recommonmark_config', {'enable_eval_rst': True}, True)
-    app.add_stylesheet('custom.css')
+    app.add_css_file('custom.css')
     app.add_transform(AutoStructify)
+    app.add_directive('jupyterhub-generate-config', ConfigDirective)
+    app.add_directive('jupyterhub-help-all', HelpAllDirective)
 
 
-source_parsers = {'.md': 'recommonmark.parser.CommonMarkParser'}
-
 source_suffix = ['.rst', '.md']
 # source_encoding = 'utf-8-sig'
 
 # -- Options for HTML output ----------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.
-html_theme = 'pandas_sphinx_theme'
+html_theme = 'pydata_sphinx_theme'
 
 html_logo = '_static/images/logo/logo.png'
 html_favicon = '_static/images/logo/favicon.ico'
@@ -166,10 +216,10 @@ intersphinx_mapping = {'https://docs.python.org/3/': None}
 on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
 if on_rtd:
     # readthedocs.org uses their theme by default, so no need to specify it
-    # build rest-api, since RTD doesn't run make
+    # build both metrics and rest-api, since RTD doesn't run make
     from subprocess import check_call as sh
 
-    sh(['make', 'rest-api'], cwd=docs)
+    sh(['make', 'metrics', 'rest-api'], cwd=docs)
 
 # -- Spell checking -------------------------------------------------------
 

docs/source/contributing/roadmap.md

@@ -83,7 +83,6 @@ these will be moved at a future review of the roadmap.
     - (prometheus?) API for resource monitoring
     - tracking activity on single-user servers instead of the proxy
     - notes and activity tracking per API token
-    - UI for managing named servers
 
 
 ### Later

docs/source/contributing/setup.rst

@@ -45,6 +45,12 @@ When developing JupyterHub, you need to make changes to the code & see
 their effects quickly. You need to do a developer install to make that
 happen.
 
+.. note:: This guide does not attempt to dictate *how* development
+   environements should be isolated since that is a personal preference and can
+   be achieved in many ways, for example `tox`, `conda`, `docker`, etc. See this
+   `forum thread <https://discourse.jupyter.org/t/thoughts-on-using-tox/3497>`_ for
+   a more detailed discussion.
+
 1. Clone the `JupyterHub git repository <https://github.com/jupyterhub/jupyterhub>`_
    to your computer.
 
@@ -93,7 +99,14 @@ happen.
       python3 -m pip install -r dev-requirements.txt
       python3 -m pip install -r requirements.txt
 
-5. Install the development version of JupyterHub. This lets you edit
+5. Setup a database.
+
+   The default database engine is ``sqlite`` so if you are just trying
+   to get up and running quickly for local development that should be
+   available via `python <https://docs.python.org/3.5/library/sqlite3.html>`__.
+   See :doc:`/reference/database` for details on other supported databases.
+
+6. Install the development version of JupyterHub. This lets you edit
    JupyterHub code in a text editor & restart the JupyterHub process to
    see your code changes immediately.
 
@@ -101,13 +114,13 @@ happen.
 
       python3 -m pip install --editable .
 
-6. You are now ready to start JupyterHub!
+7. You are now ready to start JupyterHub!
 
    .. code:: bash
 
       jupyterhub
 
-7. You can access JupyterHub from your browser at
+8. You can access JupyterHub from your browser at
    ``http://localhost:8000`` now.
 
 Happy developing!

docs/source/contributing/tests.rst

@@ -64,5 +64,5 @@ Troubleshooting Test Failures
 All the tests are failing
 -------------------------
 
-Make sure you have completed all the steps in :ref:`contributing/setup` sucessfully, and
+Make sure you have completed all the steps in :ref:`contributing/setup` successfully, and
 can launch ``jupyterhub`` from the terminal.

docs/source/getting-started/authenticators-users-basics.md

@@ -4,23 +4,23 @@ The default Authenticator uses [PAM][] to authenticate system users with
 their username and password. With the default Authenticator, any user
 with an account and password on the system will be allowed to login.
 
-## Create a whitelist of users
+## Create a set of allowed users
 
-You can restrict which users are allowed to login with a whitelist,
-`Authenticator.whitelist`:
+You can restrict which users are allowed to login with a set,
+`Authenticator.allowed_users`:
 
 
 ```python
-c.Authenticator.whitelist = {'mal', 'zoe', 'inara', 'kaylee'}
+c.Authenticator.allowed_users = {'mal', 'zoe', 'inara', 'kaylee'}
 ```
 
-Users in the whitelist are added to the Hub database when the Hub is
+Users in the `allowed_users` set are added to the Hub database when the Hub is
 started.
 
 ## Configure admins (`admin_users`)
 
 Admin users of JupyterHub, `admin_users`, can add and remove users from
-the user `whitelist`. `admin_users` can take actions on other users'
+the user `allowed_users` set. `admin_users` can take actions on other users'
 behalf, such as stopping and restarting their servers.
 
 A set of initial admin users, `admin_users` can configured be as follows:
@@ -28,7 +28,7 @@ A set of initial admin users, `admin_users` can configured be as follows:
 ```python
 c.Authenticator.admin_users = {'mal', 'zoe'}
 ```
-Users in the admin list are automatically added to the user `whitelist`,
+Users in the admin set are automatically added to the user `allowed_users` set,
 if they are not already present.
 
 Each authenticator may have different ways of determining whether a user is an
@@ -53,12 +53,12 @@ sure your users know if admin_access is enabled.**
 
 Users can be added to and removed from the Hub via either the admin
 panel or the REST API. When a user is **added**, the user will be
-automatically added to the whitelist and database. Restarting the Hub
-will not require manually updating the whitelist in your config file,
+automatically added to the allowed users set and database. Restarting the Hub
+will not require manually updating the allowed users set in your config file,
 as the users will be loaded from the database.
 
 After starting the Hub once, it is not sufficient to **remove** a user
-from the whitelist in your config file. You must also remove the user
+from the allowed users set in your config file. You must also remove the user
 from the Hub's database, either by deleting the user from JupyterHub's
 admin page, or you can clear the `jupyterhub.sqlite` database and start
 fresh.

docs/source/getting-started/faq.md

@@ -0,0 +1,36 @@
+# Frequently asked questions
+
+
+### How do I share links to notebooks?
+
+In short, where you see `/user/name/notebooks/foo.ipynb` use `/hub/user-redirect/notebooks/foo.ipynb` (replace `/user/name` with `/hub/user-redirect`).
+
+Sharing links to notebooks is a common activity,
+and can look different based on what you mean.
+Your first instinct might be to copy the URL you see in the browser,
+e.g. `hub.jupyter.org/user/yourname/notebooks/coolthing.ipynb`.
+However, let's break down what this URL means:
+
+`hub.jupyter.org/user/yourname/` is the URL prefix handled by *your server*,
+which means that sharing this URL is asking the person you share the link with
+to come to *your server* and look at the exact same file.
+In most circumstances, this is forbidden by permissions because the person you share with does not have access to your server.
+What actually happens when someone visits this URL will depend on whether your server is running and other factors.
+
+But what is our actual goal?
+A typical situation is that you have some shared or common filesystem,
+such that the same path corresponds to the same document
+(either the exact same document or another copy of it).
+Typically, what folks want when they do sharing like this
+is for each visitor to open the same file *on their own server*,
+so Breq would open `/user/breq/notebooks/foo.ipynb` and
+Seivarden would open `/user/seivarden/notebooks/foo.ipynb`, etc.
+
+JupyterHub has a special URL that does exactly this!
+It's called `/hub/user-redirect/...` and after the visitor logs in,
+So if you replace `/user/yourname` in your URL bar
+with `/hub/user-redirect` any visitor should get the same
+URL on their own server, rather than visiting yours.
+
+In JupyterLab 2.0, this should also be the result of the "Copy Shareable Link"
+action in the file browser.

docs/source/getting-started/index.rst

@@ -15,4 +15,5 @@ own JupyterHub.
    authenticators-users-basics
    spawners-basics
    services-basics
+   faq
    institutional-faq

docs/source/getting-started/institutional-faq.md

@@ -21,7 +21,7 @@ Here is a quick breakdown of these three tools:
 * **The Jupyter Notebook** is a document specification (the `.ipynb`) file that interweaves
   narrative text with code cells and their outputs. It is also a graphical interface
   that allows users to edit these documents. There are also several other graphical interfaces
-  that allow users to edit the `.ipynb` format (nteract, Jupyer Lab, Google Colab, Kaggle, etc).
+  that allow users to edit the `.ipynb` format (nteract, Jupyter Lab, Google Colab, Kaggle, etc).
 * **JupyterLab** is a flexible and extendible user interface for interactive computing. It
   has several extensions that are tailored for using Jupyter Notebooks, as well as extensions
   for other parts of the data science stack.

docs/source/getting-started/security-basics.rst

@@ -80,6 +80,49 @@ To achieve this, simply omit the configuration settings
 ``c.JupyterHub.ssl_key`` and ``c.JupyterHub.ssl_cert``
 (setting them to ``None`` does not have the same effect, and is an error).
 
+.. _authentication-token:
+
+Proxy authentication token
+--------------------------
+
+The Hub authenticates its requests to the Proxy using a secret token that
+the Hub and Proxy agree upon. Note that this applies to the default
+``ConfigurableHTTPProxy`` implementation. Not all proxy implementations
+use an auth token.
+
+The value of this token should be a random string (for example, generated by
+``openssl rand -hex 32``). You can store it in the configuration file or an
+environment variable
+
+Generating and storing token in the configuration file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can set the value in the configuration file, ``jupyterhub_config.py``:
+
+.. code-block:: python
+
+    c.ConfigurableHTTPProxy.api_token = 'abc123...' # any random string
+
+Generating and storing as an environment variable
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can pass this value of the proxy authentication token to the Hub and Proxy
+using the ``CONFIGPROXY_AUTH_TOKEN`` environment variable:
+
+.. code-block:: bash
+
+    export CONFIGPROXY_AUTH_TOKEN=$(openssl rand -hex 32)
+
+This environment variable needs to be visible to the Hub and Proxy.
+
+Default if token is not set
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you don't set the Proxy authentication token, the Hub will generate a random
+key itself, which means that any time you restart the Hub you **must also
+restart the Proxy**. If the proxy is a subprocess of the Hub, this should happen
+automatically (this is the default configuration).
+
 .. _cookie-secret:
 
 Cookie secret
@@ -146,41 +189,73 @@ itself, ``jupyterhub_config.py``, as a binary string:
    If the cookie secret value changes for the Hub, all single-user notebook
    servers must also be restarted.
 
+.. _cookies:
 
-.. _authentication-token:
+Cookies used by JupyterHub authentication
+-----------------------------------------
 
-Proxy authentication token
---------------------------
+The following cookies are used by the Hub for handling user authentication.
 
-The Hub authenticates its requests to the Proxy using a secret token that
-the Hub and Proxy agree upon. The value of this string should be a random
-string (for example, generated by ``openssl rand -hex 32``).
+This section was created based on this post_ from Discourse.
 
-Generating and storing token in the configuration file
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. _post: https://discourse.jupyter.org/t/how-to-force-re-login-for-users/1998/6
 
-Or you can set the value in the configuration file, ``jupyterhub_config.py``:
+jupyterhub-hub-login
+~~~~~~~~~~~~~~~~~~~~
 
-.. code-block:: python
+This is the login token used when visiting Hub-served pages that are
+protected by authentication such as the main home, the spawn form, etc.
+If this cookie is set, then the user is logged in.
 
-    c.JupyterHub.proxy_auth_token = '0bc02bede919e99a26de1e2a7a5aadfaf6228de836ec39a05a6c6942831d8fe5'
+Resetting the Hub cookie secret effectively revokes this cookie.
 
-Generating and storing as an environment variable
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+This cookie is restricted to the path ``/hub/``.
 
-You can pass this value of the proxy authentication token to the Hub and Proxy
-using the ``CONFIGPROXY_AUTH_TOKEN`` environment variable:
+jupyterhub-user-<username>
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. code-block:: bash
+This is the cookie used for authenticating with a single-user server.
+It is set by the single-user server after OAuth with the Hub.
 
-    export CONFIGPROXY_AUTH_TOKEN=$(openssl rand -hex 32)
+Effectively the same as ``jupyterhub-hub-login``, but for the
+single-user server instead of the Hub. It contains an OAuth access token,
+which is checked with the Hub to authenticate the browser.
 
-This environment variable needs to be visible to the Hub and Proxy.
+Each OAuth access token is associated with a session id (see ``jupyterhub-session-id`` section
+below).
 
-Default if token is not set
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+To avoid hitting the Hub on every request, the authentication response
+is cached. And to avoid a stale cache the cache key is comprised of both
+the token and session id.
 
-If you don't set the Proxy authentication token, the Hub will generate a random
-key itself, which means that any time you restart the Hub you **must also
-restart the Proxy**. If the proxy is a subprocess of the Hub, this should happen
-automatically (this is the default configuration).
+Resetting the Hub cookie secret effectively revokes this cookie.
+
+This cookie is restricted to the path ``/user/<username>``, so that
+only the user’s server receives it.
+
+jupyterhub-session-id
+~~~~~~~~~~~~~~~~~~~~~
+
+This is a random string, meaningless in itself, and the only cookie
+shared by the Hub and single-user servers.
+
+Its sole purpose is to coordinate logout of the multiple OAuth cookies.
+
+This cookie is set to ``/`` so all endpoints can receive it, or clear it, etc.
+
+jupyterhub-user-<username>-oauth-state
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A short-lived cookie, used solely to store and validate OAuth state.
+It is only set while OAuth between the single-user server and the Hub
+is processing.
+
+If you use your browser development tools, you should see this cookie
+for a very brief moment before your are logged in,
+with an expiration date shorter than ``jupyterhub-hub-login`` or
+``jupyterhub-user-<username>``.
+
+This cookie should not exist after you have successfully logged in.
+
+This cookie is restricted to the path ``/user/<username>``, so that only
+the user’s server receives it.

docs/source/getting-started/services-basics.md

@@ -5,7 +5,7 @@ that interacts with the Hub's REST API. A Service may perform a specific
 or action or task. For example, shutting down individuals' single user
 notebook servers that have been idle for some time is a good example of
 a task that could be automated by a Service. Let's look at how the
-[cull_idle_servers][] script can be used as a Service.
+[jupyterhub_idle_culler][] script can be used as a Service.
 
 ## Real-world example to cull idle servers
 
@@ -15,11 +15,11 @@ document will:
 - explain some basic information about API tokens
 - clarify that API tokens can be used to authenticate to
   single-user servers as of [version 0.8.0](../changelog)
-- show how the [cull_idle_servers][] script can be:
+- show how the [jupyterhub_idle_culler][] script can be:
     - used in a Hub-managed service
     - run as a standalone script
 
-Both examples for `cull_idle_servers` will communicate tasks to the
+Both examples for `jupyterhub_idle_culler` will communicate tasks to the
 Hub via the REST API.
 
 ## API Token basics
@@ -78,17 +78,23 @@ single-user servers, and only cookies can be used for authentication.
 0.8 supports using JupyterHub API tokens to authenticate to single-user
 servers.
 
-## Configure `cull-idle` to run as a Hub-Managed Service
+## Configure the idle culler to run as a Hub-Managed Service
+
+Install the idle culler:
+
+```
+pip install jupyterhub-idle-culler
+```
 
 In `jupyterhub_config.py`, add the following dictionary for the
-`cull-idle` Service to the `c.JupyterHub.services` list:
+`idle-culler` Service to the `c.JupyterHub.services` list:
 
 ```python
 c.JupyterHub.services = [
     {
-        'name': 'cull-idle',
+        'name': 'idle-culler',
         'admin': True,
-        'command': [sys.executable, 'cull_idle_servers.py', '--timeout=3600'],
+        'command': [sys.executable, '-m', 'jupyterhub_idle_culler', '--timeout=3600'],
     }
 ]
 ```
@@ -101,21 +107,21 @@ where:
 
 ## Run `cull-idle` manually as a standalone script
 
-Now you can run your script, i.e. `cull_idle_servers`, by providing it
+Now you can run your script by providing it
 the API token and it will authenticate through the REST API to
 interact with it.
 
-This will run `cull-idle` manually. `cull-idle` can be run as a standalone
+This will run the idle culler service manually. It can be run as a standalone
 script anywhere with access to the Hub, and will periodically check for idle
 servers and shut them down via the Hub's REST API. In order to shutdown the
 servers, the token given to cull-idle must have admin privileges.
 
 Generate an API token and store it in the `JUPYTERHUB_API_TOKEN` environment
-variable. Run `cull_idle_servers.py` manually.
+variable. Run `jupyterhub_idle_culler` manually.
 
 ```bash
     export JUPYTERHUB_API_TOKEN='token'
-    python3 cull_idle_servers.py [--timeout=900] [--url=http://127.0.0.1:8081/hub/api]
+    python -m jupyterhub_idle_culler [--timeout=900] [--url=http://127.0.0.1:8081/hub/api]
 ```
 
-[cull_idle_servers]: https://github.com/jupyterhub/jupyterhub/blob/master/examples/cull-idle/cull_idle_servers.py
+[jupyterhub_idle_culler]: https://github.com/jupyterhub/jupyterhub-idle-culler

docs/source/index.rst

@@ -3,11 +3,11 @@ JupyterHub
 ==========
 
 `JupyterHub`_ is the best way to serve `Jupyter notebook`_ for multiple users.
-It can be used in a classes of students, a corporate data science group or scientific
+It can be used in a class of students, a corporate data science group or scientific
 research group. It is a multi-user **Hub** that spawns, manages, and proxies multiple
 instances of the single-user `Jupyter notebook`_ server.
 
-To make life easier, JupyterHub have distributions. Be sure to
+To make life easier, JupyterHub has distributions. Be sure to
 take a look at them before continuing with the configuration of the broad
 original system of `JupyterHub`_. Today, you can find two main cases:
 

docs/source/installation-guide-hard.md

@@ -88,7 +88,7 @@ sudo apt install nodejs npm
 Then install `configurable-http-proxy`:
 
 ```sh
-npm install -g configurable-http-proxy
+sudo npm install -g configurable-http-proxy
 ```
 
 ### Create the configuration for JupyterHub
@@ -206,7 +206,7 @@ sudo install -o root -g root -m 644 conda.gpg /etc/apt/trusted.gpg.d/
 Add Debian repo
 
 ```sh
-sudo echo "deb [arch=amd64] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list
+echo "deb [arch=amd64] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | sudo tee /etc/apt/sources.list.d/conda.list
 ```
 
 Install conda
@@ -278,7 +278,7 @@ this way - instead, use a reverse proxy running on standard HTTP/HTTPS ports.
 > i.e. not protected within an institutional intranet or private home/office network. You should set up a firewall and
 > HTTPS encryption, which is outside of the scope of this guide. For HTTPS consider using [LetsEncrypt](https://letsencrypt.org/)
 > or setting up a [self-signed certificate](https://www.digitalocean.com/community/tutorials/how-to-create-a-self-signed-ssl-certificate-for-nginx-in-ubuntu-18-04).
-> Firewalls may be set up using `ufs` or `firewalld` and combined with `fail2ban`.
+> Firewalls may be set up using `ufw` or `firewalld` and combined with `fail2ban`.
 
 ### Using Nginx
 Nginx is a mature and established web server and reverse proxy and is easy to install using `sudo apt install nginx`.
@@ -316,6 +316,15 @@ Add the following snippet to your nginx configuration file (e.g. `/etc/nginx/sit
   }
 ```
 
+Also add this snippet before the *server* block:
+
+```
+map $http_upgrade $connection_upgrade {
+        default upgrade;
+        '' close;
+    }
+```
+
 Nginx will not run if there are errors in the configuration, check your configuration using:
 
 ```sh

docs/source/quickstart.md

@@ -26,6 +26,10 @@ Before installing JupyterHub, you will need:
     The `nodejs-legacy` package installs the `node` executable and is currently
     required for npm to work on Debian/Ubuntu.
 
+- A [pluggable authentication module (PAM)](https://en.wikipedia.org/wiki/Pluggable_authentication_module) 
+  to use the [default Authenticator](./getting-started/authenticators-users-basics.md).
+  PAM is often available by default on most distributions, if this is not the case it can be installed by 
+  using the operating system's package manager.
 - TLS certificate and key for HTTPS communication
 - Domain name
 

docs/source/reference/config-ghoauth.md

@@ -52,7 +52,7 @@ c.GitHubOAuthenticator.oauth_callback_url = os.environ['OAUTH_CALLBACK_URL']
 c.LocalAuthenticator.create_system_users = True
 
 # specify users and admin
-c.Authenticator.whitelist = {'rgbkrk', 'minrk', 'jhamrick'}
+c.Authenticator.allowed_users = {'rgbkrk', 'minrk', 'jhamrick'}
 c.Authenticator.admin_users = {'jhamrick', 'rgbkrk'}
 
 # uses the default spawner

docs/source/reference/config-proxy.md

@@ -83,8 +83,11 @@ server {
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 
         # websocket headers
+        proxy_http_version 1.1;
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection $connection_upgrade;
+
+        proxy_buffering off;
     }
 
     # Managing requests to verify letsencrypt host
@@ -139,6 +142,20 @@ Now restart `nginx`, restart the JupyterHub, and enjoy accessing
 `https://HUB.DOMAIN.TLD` while serving other content securely on
 `https://NO_HUB.DOMAIN.TLD`.
 
+### SELinux permissions for nginx
+On distributions with SELinux enabled (e.g. Fedora), one may encounter permission errors
+when the nginx service is started.
+
+We need to allow nginx to perform network relay and connect to the jupyterhub port. The
+following commands do that:
+
+```bash
+semanage port -a -t http_port_t -p tcp 8000
+setsebool -P httpd_can_network_relay 1
+setsebool -P httpd_can_network_connect 1
+```
+Replace 8000 with the port the jupyterhub server is running from.
+
 
 ## Apache
 
@@ -199,8 +216,8 @@ In case of the need to run the jupyterhub under /jhub/ or other location please
 
 httpd.conf amendments:
 ```bash
- RewriteRule /jhub/(.*) ws://127.0.0.1:8000/jhub/$1 [P,L]
- RewriteRule /jhub/(.*) http://127.0.0.1:8000/jhub/$1 [P,L]
+ RewriteRule /jhub/(.*) ws://127.0.0.1:8000/jhub/$1 [NE.P,L]
+ RewriteRule /jhub/(.*) http://127.0.0.1:8000/jhub/$1 [NE,P,L]
  
  ProxyPass /jhub/ http://127.0.0.1:8000/jhub/
  ProxyPassReverse /jhub/  http://127.0.0.1:8000/jhub/

docs/source/reference/config-reference.rst

@@ -0,0 +1,30 @@
+==============================
+Configuration Reference
+==============================
+
+.. important::
+
+    Make sure the version of JupyterHub for this documentation matches your
+    installation version, as the output of this command may change between versions.
+
+JupyterHub configuration
+------------------------
+
+As explained in the `Configuration Basics <../getting-started/config-basics.html#generate-a-default-config-file>`_
+section, the ``jupyterhub_config.py`` can be automatically generated via
+
+    .. code-block:: bash
+
+        jupyterhub --generate-config
+
+
+The following contains the output of that command for reference.
+
+.. jupyterhub-generate-config::
+
+JupyterHub help command output
+------------------------------
+
+This section contains the output of the command ``jupyterhub --help-all``.
+
+.. jupyterhub-help-all::

docs/source/reference/config-sudo.md

@@ -57,7 +57,7 @@ To do this we add to `/etc/sudoers` (use `visudo` for safe editing of sudoers):
 For example:
 
 ```bash
-# comma-separated whitelist of users that can spawn single-user servers
+# comma-separated list of users that can spawn single-user servers
 # this should include all of your Hub users
 Runas_Alias JUPYTER_USERS = rhea, zoe, wash
 
@@ -120,6 +120,11 @@ the shadow password database.
 
 ### Shadow group (Linux)
 
+**Note:** On Fedora based distributions there is no clear way to configure
+the PAM database to allow sufficient access for authenticating with the target user's password
+from JupyterHub. As a workaround we recommend use an
+[alternative authentication method](https://github.com/jupyterhub/jupyterhub/wiki/Authenticators).
+
 ```bash
 $ ls -l /etc/shadow
 -rw-r-----  1 root shadow   2197 Jul 21 13:41 shadow

docs/source/reference/index.rst

@@ -16,6 +16,7 @@ what happens under-the-hood when you deploy and configure your JupyterHub.
    proxy
    separate-proxy
    rest
+   monitoring
    database
    templates
    ../events/index
@@ -24,3 +25,4 @@ what happens under-the-hood when you deploy and configure your JupyterHub.
    config-ghoauth
    config-proxy
    config-sudo
+   config-reference

docs/source/reference/monitoring.rst

@@ -0,0 +1,20 @@
+Monitoring
+==========
+
+This section covers details on monitoring the state of your JupyterHub installation.
+
+JupyterHub expose the ``/metrics`` endpoint that returns text describing its current
+operational state formatted in a way `Prometheus <https://prometheus.io/docs/introduction/overview/>`_ understands.
+
+Prometheus is a separate open source tool that can be configured to repeatedly poll
+JupyterHub's ``/metrics`` endpoint to parse and save its current state.
+
+By doing so, Prometheus can describe JupyterHub's evolving state over time.
+This evolving state can then be accessed through Prometheus that expose its underlying
+storage to those allowed to access it, and be presented with dashboards by a
+tool like `Grafana <https://grafana.com/docs/grafana/latest/getting-started/what-is-grafana/>`_.
+
+.. toctree::
+   :maxdepth: 2
+
+   metrics

docs/source/reference/rest.md

@@ -57,6 +57,9 @@ generating an API token is available from the JupyterHub user interface:
 
 ## Add API tokens to the config file
 
+**This is deprecated. We are in no rush to remove this feature,
+but please consider if service tokens are right for you.**
+
 You may also add a dictionary of API tokens and usernames to the hub's
 configuration file, `jupyterhub_config.py` (note that
 the **key** is the 'secret-token' while the **value** is the 'username'):
@@ -67,6 +70,41 @@ c.JupyterHub.api_tokens = {
 }
 ```
 
+### Updating to admin services
+
+The `api_tokens` configuration has been softly deprecated since the introduction of services.
+We have no plans to remove it,
+but users are encouraged to use service configuration instead.
+
+If you have been using `api_tokens` to create an admin user
+and a token for that user to perform some automations,
+the services mechanism may be a better fit.
+If you have the following configuration:
+
+```python
+c.JupyterHub.admin_users = {"service-admin",}
+c.JupyterHub.api_tokens = {
+    "secret-token": "service-admin",
+}
+```
+
+This can be updated to create an admin service, with the following configuration:
+
+```python
+c.JupyterHub.services = [
+    {
+        "name": "service-token",
+        "admin": True,
+        "api_token": "secret-token",
+    },
+]
+```
+
+The token will have the same admin permissions,
+but there will no longer be a user account created to house it.
+The main noticeable difference is that there will be no notebook server associated with the account
+and the service will not show up in the various user list pages and APIs.
+
 ## Make an API request
 
 To authenticate your requests, pass the API token in the request's

docs/source/reference/services.md

@@ -151,6 +151,8 @@ c.JupyterHub.services = [
     {
         'name': 'my-web-service',
         'url': 'https://10.0.1.1:1984',
+        # any secret >8 characters, you'll use api_token to
+        # authenticate api requests to the hub from your service
         'api_token': 'super-secret',
     }
 ]
@@ -313,7 +315,7 @@ class MyHandler(HubAuthenticated, web.RequestHandler):
 The HubAuth will automatically load the desired configuration from the Service
 environment variables.
 
-If you want to limit user access, you can whitelist users through either the
+If you want to limit user access, you can specify allowed users through either the
 `.hub_users` attribute or `.hub_groups`. These are sets that check against the
 username and user group list, respectively. If a user matches neither the user
 list nor the group list, they will not be allowed access. If both are left
@@ -331,7 +333,9 @@ and taking note of the following process:
 1. retrieve the cookie `jupyterhub-services` from the request.
 2. Make an API request `GET /hub/api/authorizations/cookie/jupyterhub-services/cookie-value`,
     where cookie-value is the url-encoded value of the `jupyterhub-services` cookie.
-    This request must be authenticated with a Hub API token in the `Authorization` header.
+    This request must be authenticated with a Hub API token in the `Authorization` header,
+    for example using the `api_token` from your [external service's configuration](#externally-managed-services).
+
     For example, with [requests][]:
 
     ```python
@@ -360,7 +364,7 @@ and taking note of the following process:
 
 An example of using an Externally-Managed Service and authentication is
 in [nbviewer README][nbviewer example] section on securing the notebook viewer,
-and an example of its configuration is found [here](https://github.com/jupyter/nbviewer/blob/master/nbviewer/providers/base.py#L94).
+and an example of its configuration is found [here](https://github.com/jupyter/nbviewer/blob/ed942b10a52b6259099e2dd687930871dc8aac22/nbviewer/providers/base.py#L95).
 nbviewer can also be run as a Hub-Managed Service as described [nbviewer README][nbviewer example]
 section on securing the notebook viewer.
 

docs/source/reference/spawners.md

@@ -27,8 +27,8 @@ Some examples include:
   servers using batch systems
 - [YarnSpawner](https://github.com/jupyterhub/yarnspawner) for spawning notebook
   servers in YARN containers on a Hadoop cluster
-- [RemoteSpawner](https://github.com/zonca/remotespawner) to spawn notebooks
-  and a remote server and tunnel the port via SSH
+- [SSHSpawner](https://github.com/NERSC/sshspawner) to spawn notebooks
+  on a remote server using SSH
 
 
 ## Spawner control methods

docs/source/troubleshooting.md

@@ -7,8 +7,8 @@ problem and how to resolve it.
 [*Behavior*](#behavior)
 - JupyterHub proxy fails to start
 - sudospawner fails to run
-- What is the default behavior when none of the lists (admin, whitelist,
-  group whitelist) are set?
+- What is the default behavior when none of the lists (admin, allowed,
+  allowed groups) are set?
 - JupyterHub Docker container not accessible at localhost
 
 [*Errors*](#errors)
@@ -55,12 +55,12 @@ or add:
 
 to the config file, `jupyterhub_config.py`.
 
-### What is the default behavior when none of the lists (admin, whitelist, group whitelist) are set?
+### What is the default behavior when none of the lists (admin, allowed, allowed groups) are set?
 
 When nothing is given for these lists, there will be no admins, and all users
 who can authenticate on the system (i.e. all the unix users on the server with
-a password) will be allowed to start a server. The whitelist lets you limit
-this to a particular set of users, and the admin_users lets you specify who
+a password) will be allowed to start a server. The allowed username set lets you limit
+this to a particular set of users, and admin_users lets you specify who
 among them may use the admin interface (not necessary, unless you need to do
 things like inspect other users' servers, or modify the user list at runtime).
 
@@ -75,6 +75,50 @@ tell Jupyterhub to start at `0.0.0.0` which is visible to everyone. Try this
 command: 
 `docker run -p 8000:8000 -d --name jupyterhub jupyterhub/jupyterhub jupyterhub --ip 0.0.0.0 --port 8000`
 
+### How can I kill ports from JupyterHub managed services that have been orphaned?
+
+I started JupyterHub + nbgrader on the same host without containers. When I try to restart JupyterHub + nbgrader with this configuration, errors appear that the service accounts cannot start because the ports are being used.
+
+How can I kill the processes that are using these ports?
+
+Run the following command:
+
+    sudo kill -9 $(sudo lsof -t -i:<service_port>)
+
+Where `<service_port>` is the port used by the nbgrader course service. This configuration is specified in `jupyterhub_config.py`.
+
+### Why am I getting a Spawn failed error message?
+
+After successfully logging in to JupyterHub with a compatible authenticators, I get a 'Spawn failed' error message in the browser. The JupyterHub logs have `jupyterhub KeyError: "getpwnam(): name not found: <my_user_name>`.
+
+This issue occurs when the authenticator requires a local system user to exist. In these cases, you need to use a spawner
+that does not require an existing system user account, such as `DockerSpawner` or `KubeSpawner`.
+
+### How can I run JupyterHub with sudo but use my current env vars and virtualenv location?
+
+When launching JupyterHub with `sudo jupyterhub` I get import errors and my environment variables don't work.
+
+When launching services with `sudo ...` the shell won't have the same environment variables or `PATH`s in place. The most direct way to solve this issue is to use the full path to your python environment and add environment variables. For example:
+
+```bash
+sudo MY_ENV=abc123 \
+  /home/foo/venv/bin/python3 \
+  /srv/jupyterhub/jupyterhub
+```
+
+### How can I view the logs for JupyterHub or the user's Notebook servers when using the DockerSpawner?
+
+Use `docker logs <container>` where `<container>` is the container name defined within `docker-compose.yml`.  For example, to view the logs of the JupyterHub container use:
+
+    docker logs hub
+
+By default, the user's notebook server is named `jupyter-<username>` where `username` is the user's username within JupyterHub's db. So if you wanted to see the logs for user `foo` you would use:
+
+    docker logs jupyter-foo
+
+You can also tail logs to view them in real time using the `-f` option:
+
+    docker logs -f hub
 
 ## Errors
 
@@ -108,7 +152,7 @@ You should see a similar 200 message, as above, in the Hub log when you first
 visit your single-user notebook server. If you don't see this message in the log, it
 may mean that your single-user notebook server isn't connecting to your Hub.
 
-If you see 403 (forbidden) like this, it's a token problem:
+If you see 403 (forbidden) like this, it's likely a token problem:
 
 ```
 403 GET /hub/api/authorizations/cookie/jupyterhub-token-name/[secret] (@10.0.1.4) 4.14ms
@@ -152,6 +196,42 @@ After this, when you start your server via JupyterHub, it will build a
 new container. If this was the underlying cause of the issue, you should see
 your server again.
 
+##### Proxy settings (403 GET)
+
+When your whole JupyterHub sits behind a organization proxy (*not* a reverse proxy like NGINX as part of your setup and *not* the configurable-http-proxy) the environment variables `HTTP_PROXY`, `HTTPS_PROXY`, `http_proxy` and `https_proxy` might be set. This confuses the jupyterhub-singleuser servers: When connecting to the Hub for authorization they connect via the proxy instead of directly connecting to the Hub on localhost. The proxy might deny the request (403 GET). This results in the singleuser server thinking it has a wrong auth token. To circumvent this you should add `<hub_url>,<hub_ip>,localhost,127.0.0.1` to the environment variables `NO_PROXY` and `no_proxy`.
+
+### Launching Jupyter Notebooks to run as an externally managed JupyterHub service with the `jupyterhub-singleuser` command returns a `JUPYTERHUB_API_TOKEN` error
+
+[JupyterHub services](https://jupyterhub.readthedocs.io/en/stable/reference/services.html) allow processes to interact with JupyterHub's REST API. Example use-cases include:
+
+* **Secure Testing**: provide a canonical Jupyter Notebook for testing production data to reduce the number of entry points into production systems.
+* **Grading Assignments**: provide access to shared Jupyter Notebooks that may be used for management tasks such grading assignments.
+* **Private Dashboards**: share dashboards with certain group members.
+
+If possible, try to run the Jupyter Notebook as an externally managed service with one of the provided [jupyter/docker-stacks](https://github.com/jupyter/docker-stacks).
+
+Standard JupyterHub installations include a [jupyterhub-singleuser](https://github.com/jupyterhub/jupyterhub/blob/9fdab027daa32c9017845572ad9d5ba1722dbc53/setup.py#L116) command which is built from the `jupyterhub.singleuser:main` method. The `jupyterhub-singleuser` command is the default command when JupyterHub launches single-user Jupyter Notebooks. One of the goals of this command is to make sure the version of JupyterHub installed within the Jupyter Notebook coincides with the version of the JupyterHub server itself.
+
+If you launch a Jupyter Notebook with the `jupyterhub-singleuser` command directly from the command line the Jupyter Notebook won't have access to the `JUPYTERHUB_API_TOKEN` and will return:
+
+```
+    JUPYTERHUB_API_TOKEN env is required to run jupyterhub-singleuser.
+    Did you launch it manually?
+```
+
+If you plan on testing `jupyterhub-singleuser` independently from JupyterHub, then you can set the api token environment variable. For example, if were to run the single-user Jupyter Notebook on the host, then:
+
+    export JUPYTERHUB_API_TOKEN=my_secret_token
+    jupyterhub-singleuser
+
+With a docker container, pass in the environment variable with the run command:
+
+    docker run -d \
+      -p 8888:8888 \
+      -e JUPYTERHUB_API_TOKEN=my_secret_token \
+      jupyter/datascience-notebook:latest
+
+[This example](https://github.com/jupyterhub/jupyterhub/tree/master/examples/service-notebook/external) demonstrates how to combine the use of the `jupyterhub-singleuser` environment variables when launching a Notebook as an externally managed service.
 
 ## How do I...?
 
@@ -193,7 +273,7 @@ where `ssl_cert` is example-chained.crt and ssl_key to your private key.
 
 Then restart JupyterHub.
 
-See also [JupyterHub SSL encryption](getting-started.md#ssl-encryption).
+See also [JupyterHub SSL encryption](./getting-started/security-basics.html#ssl-encryption).
 
 ### Install JupyterHub without a network connection
 
@@ -252,8 +332,7 @@ notebook servers to default to JupyterLab:
 ### How do I set up JupyterHub for a workshop (when users are not known ahead of time)?
 
 1. Set up JupyterHub using OAuthenticator for GitHub authentication
-2. Configure whitelist to be an empty list in` jupyterhub_config.py`
-3. Configure admin list to have workshop leaders be listed with administrator privileges.
+2. Configure admin list to have workshop leaders be listed with administrator privileges.
 
 Users will need a GitHub account to login and be authenticated by the Hub.
 
@@ -281,7 +360,6 @@ Or use syslog:
 
     jupyterhub | logger -t jupyterhub
 
-
 ## Troubleshooting commands
 
 The following commands provide additional detail about installed packages,

examples/cull-idle/README.md

@@ -1,41 +1,4 @@
-# `cull-idle` Example
+# idle-culler example
 
-The `cull_idle_servers.py` file provides a script to cull and shut down idle
-single-user notebook servers. This script is used when `cull-idle` is run as
-a Service or when it is run manually as a standalone script.
-
-
-## Configure `cull-idle` to run as a Hub-Managed Service
-
-In `jupyterhub_config.py`, add the following dictionary for the `cull-idle`
-Service to the `c.JupyterHub.services` list:
-
-```python
-c.JupyterHub.services = [
-    {
-        'name': 'cull-idle',
-        'admin': True,
-        'command': [sys.executable, 'cull_idle_servers.py', '--timeout=3600'],
-    }
-]
-```
-
-where:
-
-- `'admin': True` indicates that the Service has 'admin' permissions, and
-- `'command'` indicates that the Service will be managed by the Hub.
-
-## Run `cull-idle` manually as a standalone script
-
-This will run `cull-idle` manually. `cull-idle` can be run as a standalone
-script anywhere with access to the Hub, and will periodically check for idle
-servers and shut them down via the Hub's REST API. In order to shutdown the
-servers, the token given to cull-idle must have admin privileges.
-
-Generate an API token and store it in the `JUPYTERHUB_API_TOKEN` environment
-variable. Run `cull_idle_servers.py` manually. 
-
-```bash
-    export JUPYTERHUB_API_TOKEN=$(jupyterhub token)
-    python3 cull_idle_servers.py [--timeout=900] [--url=http://127.0.0.1:8081/hub/api]
-```
+The idle culler has been moved to its own repository at
+[jupyterhub/jupyterhub-idle-culler](https://github.com/jupyterhub/jupyterhub-idle-culler).

examples/cull-idle/cull_idle_servers.py

@@ -1,401 +0,0 @@
-#!/usr/bin/env python3
-"""script to monitor and cull idle single-user servers
-
-Caveats:
-
-last_activity is not updated with high frequency,
-so cull timeout should be greater than the sum of:
-
-- single-user websocket ping interval (default: 30s)
-- JupyterHub.last_activity_interval (default: 5 minutes)
-
-You can run this as a service managed by JupyterHub with this in your config::
-
-
-    c.JupyterHub.services = [
-        {
-            'name': 'cull-idle',
-            'admin': True,
-            'command': [sys.executable, 'cull_idle_servers.py', '--timeout=3600'],
-        }
-    ]
-
-Or run it manually by generating an API token and storing it in `JUPYTERHUB_API_TOKEN`:
-
-    export JUPYTERHUB_API_TOKEN=$(jupyterhub token)
-    python3 cull_idle_servers.py [--timeout=900] [--url=http://127.0.0.1:8081/hub/api]
-
-This script uses the same ``--timeout`` and ``--max-age`` values for
-culling users and users' servers.  If you want a different value for
-users and servers, you should add this script to the services list
-twice, just with different ``name``s, different values, and one with
-the ``--cull-users`` option.
-"""
-import json
-import os
-from datetime import datetime
-from datetime import timezone
-from functools import partial
-
-try:
-    from urllib.parse import quote
-except ImportError:
-    from urllib import quote
-
-import dateutil.parser
-
-from tornado.gen import coroutine, multi
-from tornado.locks import Semaphore
-from tornado.log import app_log
-from tornado.httpclient import AsyncHTTPClient, HTTPRequest
-from tornado.ioloop import IOLoop, PeriodicCallback
-from tornado.options import define, options, parse_command_line
-
-
-def parse_date(date_string):
-    """Parse a timestamp
-
-    If it doesn't have a timezone, assume utc
-
-    Returned datetime object will always be timezone-aware
-    """
-    dt = dateutil.parser.parse(date_string)
-    if not dt.tzinfo:
-        # assume naïve timestamps are UTC
-        dt = dt.replace(tzinfo=timezone.utc)
-    return dt
-
-
-def format_td(td):
-    """
-    Nicely format a timedelta object
-
-    as HH:MM:SS
-    """
-    if td is None:
-        return "unknown"
-    if isinstance(td, str):
-        return td
-    seconds = int(td.total_seconds())
-    h = seconds // 3600
-    seconds = seconds % 3600
-    m = seconds // 60
-    seconds = seconds % 60
-    return "{h:02}:{m:02}:{seconds:02}".format(h=h, m=m, seconds=seconds)
-
-
-@coroutine
-def cull_idle(
-    url, api_token, inactive_limit, cull_users=False, max_age=0, concurrency=10
-):
-    """Shutdown idle single-user servers
-
-    If cull_users, inactive *users* will be deleted as well.
-    """
-    auth_header = {'Authorization': 'token %s' % api_token}
-    req = HTTPRequest(url=url + '/users', headers=auth_header)
-    now = datetime.now(timezone.utc)
-    client = AsyncHTTPClient()
-
-    if concurrency:
-        semaphore = Semaphore(concurrency)
-
-        @coroutine
-        def fetch(req):
-            """client.fetch wrapped in a semaphore to limit concurrency"""
-            yield semaphore.acquire()
-            try:
-                return (yield client.fetch(req))
-            finally:
-                yield semaphore.release()
-
-    else:
-        fetch = client.fetch
-
-    resp = yield fetch(req)
-    users = json.loads(resp.body.decode('utf8', 'replace'))
-    futures = []
-
-    @coroutine
-    def handle_server(user, server_name, server, max_age, inactive_limit):
-        """Handle (maybe) culling a single server
-
-        "server" is the entire server model from the API.
-
-        Returns True if server is now stopped (user removable),
-        False otherwise.
-        """
-        log_name = user['name']
-        if server_name:
-            log_name = '%s/%s' % (user['name'], server_name)
-        if server.get('pending'):
-            app_log.warning(
-                "Not culling server %s with pending %s", log_name, server['pending']
-            )
-            return False
-
-        # jupyterhub < 0.9 defined 'server.url' once the server was ready
-        # as an *implicit* signal that the server was ready.
-        # 0.9 adds a dedicated, explicit 'ready' field.
-        # By current (0.9) definitions, servers that have no pending
-        # events and are not ready shouldn't be in the model,
-        # but let's check just to be safe.
-
-        if not server.get('ready', bool(server['url'])):
-            app_log.warning(
-                "Not culling not-ready not-pending server %s: %s", log_name, server
-            )
-            return False
-
-        if server.get('started'):
-            age = now - parse_date(server['started'])
-        else:
-            # started may be undefined on jupyterhub < 0.9
-            age = None
-
-        # check last activity
-        # last_activity can be None in 0.9
-        if server['last_activity']:
-            inactive = now - parse_date(server['last_activity'])
-        else:
-            # no activity yet, use start date
-            # last_activity may be None with jupyterhub 0.9,
-            # which introduces the 'started' field which is never None
-            # for running servers
-            inactive = age
-
-        # CUSTOM CULLING TEST CODE HERE
-        # Add in additional server tests here.  Return False to mean "don't
-        # cull", True means "cull immediately", or, for example, update some
-        # other variables like inactive_limit.
-        #
-        # Here, server['state'] is the result of the get_state method
-        # on the spawner.  This does *not* contain the below by
-        # default, you may have to modify your spawner to make this
-        # work.  The `user` variable is the user model from the API.
-        #
-        # if server['state']['profile_name'] == 'unlimited'
-        #     return False
-        # inactive_limit = server['state']['culltime']
-
-        should_cull = (
-            inactive is not None and inactive.total_seconds() >= inactive_limit
-        )
-        if should_cull:
-            app_log.info(
-                "Culling server %s (inactive for %s)", log_name, format_td(inactive)
-            )
-
-        if max_age and not should_cull:
-            # only check started if max_age is specified
-            # so that we can still be compatible with jupyterhub 0.8
-            # which doesn't define the 'started' field
-            if age is not None and age.total_seconds() >= max_age:
-                app_log.info(
-                    "Culling server %s (age: %s, inactive for %s)",
-                    log_name,
-                    format_td(age),
-                    format_td(inactive),
-                )
-                should_cull = True
-
-        if not should_cull:
-            app_log.debug(
-                "Not culling server %s (age: %s, inactive for %s)",
-                log_name,
-                format_td(age),
-                format_td(inactive),
-            )
-            return False
-
-        if server_name:
-            # culling a named server
-            delete_url = url + "/users/%s/servers/%s" % (
-                quote(user['name']),
-                quote(server['name']),
-            )
-        else:
-            delete_url = url + '/users/%s/server' % quote(user['name'])
-
-        req = HTTPRequest(url=delete_url, method='DELETE', headers=auth_header)
-        resp = yield fetch(req)
-        if resp.code == 202:
-            app_log.warning("Server %s is slow to stop", log_name)
-            # return False to prevent culling user with pending shutdowns
-            return False
-        return True
-
-    @coroutine
-    def handle_user(user):
-        """Handle one user.
-
-        Create a list of their servers, and async exec them.  Wait for
-        that to be done, and if all servers are stopped, possibly cull
-        the user.
-        """
-        # shutdown servers first.
-        # Hub doesn't allow deleting users with running servers.
-        # jupyterhub 0.9 always provides a 'servers' model.
-        # 0.8 only does this when named servers are enabled.
-        if 'servers' in user:
-            servers = user['servers']
-        else:
-            # jupyterhub < 0.9 without named servers enabled.
-            # create servers dict with one entry for the default server
-            # from the user model.
-            # only if the server is running.
-            servers = {}
-            if user['server']:
-                servers[''] = {
-                    'last_activity': user['last_activity'],
-                    'pending': user['pending'],
-                    'url': user['server'],
-                }
-        server_futures = [
-            handle_server(user, server_name, server, max_age, inactive_limit)
-            for server_name, server in servers.items()
-        ]
-        results = yield multi(server_futures)
-        if not cull_users:
-            return
-        # some servers are still running, cannot cull users
-        still_alive = len(results) - sum(results)
-        if still_alive:
-            app_log.debug(
-                "Not culling user %s with %i servers still alive",
-                user['name'],
-                still_alive,
-            )
-            return False
-
-        should_cull = False
-        if user.get('created'):
-            age = now - parse_date(user['created'])
-        else:
-            # created may be undefined on jupyterhub < 0.9
-            age = None
-
-        # check last activity
-        # last_activity can be None in 0.9
-        if user['last_activity']:
-            inactive = now - parse_date(user['last_activity'])
-        else:
-            # no activity yet, use start date
-            # last_activity may be None with jupyterhub 0.9,
-            # which introduces the 'created' field which is never None
-            inactive = age
-
-        should_cull = (
-            inactive is not None and inactive.total_seconds() >= inactive_limit
-        )
-        if should_cull:
-            app_log.info("Culling user %s (inactive for %s)", user['name'], inactive)
-
-        if max_age and not should_cull:
-            # only check created if max_age is specified
-            # so that we can still be compatible with jupyterhub 0.8
-            # which doesn't define the 'started' field
-            if age is not None and age.total_seconds() >= max_age:
-                app_log.info(
-                    "Culling user %s (age: %s, inactive for %s)",
-                    user['name'],
-                    format_td(age),
-                    format_td(inactive),
-                )
-                should_cull = True
-
-        if not should_cull:
-            app_log.debug(
-                "Not culling user %s (created: %s, last active: %s)",
-                user['name'],
-                format_td(age),
-                format_td(inactive),
-            )
-            return False
-
-        req = HTTPRequest(
-            url=url + '/users/%s' % user['name'], method='DELETE', headers=auth_header
-        )
-        yield fetch(req)
-        return True
-
-    for user in users:
-        futures.append((user['name'], handle_user(user)))
-
-    for (name, f) in futures:
-        try:
-            result = yield f
-        except Exception:
-            app_log.exception("Error processing %s", name)
-        else:
-            if result:
-                app_log.debug("Finished culling %s", name)
-
-
-if __name__ == '__main__':
-    define(
-        'url',
-        default=os.environ.get('JUPYTERHUB_API_URL'),
-        help="The JupyterHub API URL",
-    )
-    define('timeout', default=600, help="The idle timeout (in seconds)")
-    define(
-        'cull_every',
-        default=0,
-        help="The interval (in seconds) for checking for idle servers to cull",
-    )
-    define(
-        'max_age',
-        default=0,
-        help="The maximum age (in seconds) of servers that should be culled even if they are active",
-    )
-    define(
-        'cull_users',
-        default=False,
-        help="""Cull users in addition to servers.
-                This is for use in temporary-user cases such as tmpnb.""",
-    )
-    define(
-        'concurrency',
-        default=10,
-        help="""Limit the number of concurrent requests made to the Hub.
-
-                Deleting a lot of users at the same time can slow down the Hub,
-                so limit the number of API requests we have outstanding at any given time.
-                """,
-    )
-
-    parse_command_line()
-    if not options.cull_every:
-        options.cull_every = options.timeout // 2
-    api_token = os.environ['JUPYTERHUB_API_TOKEN']
-
-    try:
-        AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
-    except ImportError as e:
-        app_log.warning(
-            "Could not load pycurl: %s\n"
-            "pycurl is recommended if you have a large number of users.",
-            e,
-        )
-
-    loop = IOLoop.current()
-    cull = partial(
-        cull_idle,
-        url=options.url,
-        api_token=api_token,
-        inactive_limit=options.timeout,
-        cull_users=options.cull_users,
-        max_age=options.max_age,
-        concurrency=options.concurrency,
-    )
-    # schedule first cull immediately
-    # because PeriodicCallback doesn't start until the end of the first interval
-    loop.add_callback(cull)
-    # schedule periodic cull
-    pc = PeriodicCallback(cull, 1e3 * options.cull_every)
-    pc.start()
-    try:
-        loop.start()
-    except KeyboardInterrupt:
-        pass

examples/cull-idle/jupyterhub_config.py

@@ -1,11 +0,0 @@
-import sys
-
-# run cull-idle as a service
-
-c.JupyterHub.services = [
-    {
-        'name': 'cull-idle',
-        'admin': True,
-        'command': [sys.executable, 'cull_idle_servers.py', '--timeout=3600'],
-    }
-]

examples/external-oauth/whoami-oauth-basic.py

@@ -5,13 +5,11 @@ so all URLs and requests necessary for OAuth with JupyterHub should be in one pl
 """
 import json
 import os
-import sys
 from urllib.parse import urlencode
 from urllib.parse import urlparse
 
 from tornado import log
 from tornado import web
-from tornado.auth import OAuth2Mixin
 from tornado.httpclient import AsyncHTTPClient
 from tornado.httpclient import HTTPRequest
 from tornado.httputil import url_concat

examples/service-announcement/announcement.py

@@ -4,7 +4,6 @@ import json
 import os
 
 from tornado import escape
-from tornado import gen
 from tornado import ioloop
 from tornado import web
 

examples/service-whoami-flask/jupyterhub_config.py

@@ -1,6 +1,3 @@
-import os
-import sys
-
 c.JupyterHub.services = [
     {
         'name': 'whoami',

examples/service-whoami/jupyterhub_config.py

@@ -1,4 +1,3 @@
-import os
 import sys
 
 c.JupyterHub.services = [

examples/service-whoami/whoami-oauth.py

@@ -6,7 +6,6 @@ showing the user their own info.
 """
 import json
 import os
-from getpass import getuser
 from urllib.parse import urlparse
 
 from tornado.httpserver import HTTPServer
@@ -25,6 +24,7 @@ class WhoAmIHandler(HubOAuthenticated, RequestHandler):
     # `getuser()` here would mean only the user who started the service
     # can access the service:
 
+    # from getpass import getuser
     # hub_users = {getuser()}
 
     @authenticated

examples/service-whoami/whoami.py

@@ -4,7 +4,6 @@ This serves `/services/whoami/`, authenticated with the Hub, showing the user th
 """
 import json
 import os
-from getpass import getuser
 from urllib.parse import urlparse
 
 from tornado.httpserver import HTTPServer
@@ -21,6 +20,7 @@ class WhoAmIHandler(HubAuthenticated, RequestHandler):
     # `getuser()` here would mean only the user who started the service
     # can access the service:
 
+    # from getpass import getuser
     # hub_users = {getuser()}
 
     @authenticated

hooks/post_build

@@ -1,4 +1,7 @@
 #!/bin/bash
 set -exuo pipefail
 
+# build jupyterhub-onbuild image
 docker build --build-arg BASE_IMAGE=$DOCKER_REPO:$DOCKER_TAG -t ${DOCKER_REPO}-onbuild:$DOCKER_TAG onbuild
+# build jupyterhub-demo image
+docker build --build-arg BASE_IMAGE=${DOCKER_REPO}-onbuild:$DOCKER_TAG -t ${DOCKER_REPO}-demo:$DOCKER_TAG demo-image

hooks/post_push

@@ -2,8 +2,11 @@
 set -exuo pipefail
 
 export ONBUILD=${DOCKER_REPO}-onbuild
+export DEMO=${DOCKER_REPO}-demo
+export REPOS="${DOCKER_REPO} ${ONBUILD} ${DEMO}"
 # push ONBUILD image
 docker push $ONBUILD:$DOCKER_TAG
+docker push $DEMO:$DOCKER_TAG
 
 function get_hub_version() {
   rm -f hub_version
@@ -20,25 +23,20 @@ function get_hub_version() {
   fi
 }
 
-
 get_hub_version
 
+for repo in ${REPOS}; do
   # when building master, push 0.9.0.dev as well
-docker tag $DOCKER_REPO:$DOCKER_TAG $DOCKER_REPO:$hub_xyz
-docker push $DOCKER_REPO:$hub_xyz
-docker tag $ONBUILD:$DOCKER_TAG $ONBUILD:$hub_xyz
-docker push $ONBUILD:$hub_xyz
+  docker tag $repo:$DOCKER_TAG $repo:$hub_xyz
+  docker push $repo:$hub_xyz
 
   # when building 0.9.x, push 0.9 as well
-docker tag $DOCKER_REPO:$DOCKER_TAG $DOCKER_REPO:$hub_xy
-docker push $DOCKER_REPO:$hub_xy
-docker tag $ONBUILD:$DOCKER_TAG $ONBUILD:$hub_xy
-docker push $ONBUILD:$hub_xyz
+  docker tag $repo:$DOCKER_TAG $repo:$hub_xy
+  docker push $repo:$hub_xy
 
   # if building a stable release, tag latest as well
   if [[ "$latest" == "1" ]]; then
-  docker tag $DOCKER_REPO:$DOCKER_TAG $DOCKER_REPO:latest
-  docker push $DOCKER_REPO:latest
-  docker tag $ONBUILD:$DOCKER_TAG $ONBUILD:latest
-  docker push $ONBUILD:latest
+    docker tag $repo:$DOCKER_TAG $repo:latest
+    docker push $repo:latest
   fi
+done

jupyterhub/_version.py

@@ -4,9 +4,9 @@
 
 version_info = (
     1,
-    1,
-    0,
-    # "",  # release (b1, rc1, or "" for final or dev)
+    2,
+    2,
+    "",  # release (b1, rc1, or "" for final or dev)
     # "dev",  # dev or nothing for beta/rc/stable releases
 )
 
@@ -18,6 +18,15 @@ version_info = (
 
 __version__ = ".".join(map(str, version_info[:3])) + ".".join(version_info[3:])
 
+# Singleton flag to only log the major/minor mismatch warning once per mismatch combo.
+_version_mismatch_warning_logged = {}
+
+
+def reset_globals():
+    """Used to reset globals between test cases."""
+    global _version_mismatch_warning_logged
+    _version_mismatch_warning_logged = {}
+
 
 def _check_version(hub_version, singleuser_version, log):
     """Compare Hub and single-user server versions"""
@@ -42,14 +51,22 @@ def _check_version(hub_version, singleuser_version, log):
         hub_major_minor = V(hub_version).version[:2]
         singleuser_major_minor = V(singleuser_version).version[:2]
         extra = ""
+        do_log = True
         if singleuser_major_minor == hub_major_minor:
             # patch-level mismatch or lower, log difference at debug-level
             # because this should be fine
             log_method = log.debug
         else:
             # log warning-level for more significant mismatch, such as 0.8 vs 0.9, etc.
+            key = '%s-%s' % (hub_version, singleuser_version)
+            global _version_mismatch_warning_logged
+            if _version_mismatch_warning_logged.get(key):
+                do_log = False  # We already logged this warning so don't log it again.
+            else:
                 log_method = log.warning
                 extra = " This could cause failure to authenticate and result in redirect loops!"
+                _version_mismatch_warning_logged[key] = True
+        if do_log:
             log_method(
                 "jupyterhub version %s != jupyterhub-singleuser version %s." + extra,
                 hub_version,

jupyterhub/apihandlers/auth.py

@@ -201,7 +201,7 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler):
     def needs_oauth_confirm(self, user, oauth_client):
         """Return whether the given oauth client needs to prompt for access for the given user
 
-        Checks whitelist for oauth clients
+        Checks list for oauth clients that don't need confirmation
 
         (i.e. the user's own server)
 
@@ -214,9 +214,9 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler):
         if (
             # it's the user's own server
             oauth_client.identifier in own_oauth_client_ids
-            # or it's in the global whitelist
+            # or it's in the global no-confirm list
             or oauth_client.identifier
-            in self.settings.get('oauth_no_confirm_whitelist', set())
+            in self.settings.get('oauth_no_confirm_list', set())
         ):
             return False
         # default: require confirmation
@@ -229,7 +229,7 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler):
         Render oauth confirmation page:
         "Server at ... would like permission to ...".
 
-        Users accessing their own server or a service whitelist
+        Users accessing their own server or a blessed service
         will skip confirmation.
         """
 
@@ -275,9 +275,26 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler):
         uri, http_method, body, headers = self.extract_oauth_params()
         referer = self.request.headers.get('Referer', 'no referer')
         full_url = self.request.full_url()
-        if referer != full_url:
+        # trim protocol, which cannot be trusted with multiple layers of proxies anyway
+        # Referer is set by browser, but full_url can be modified by proxy layers to appear as http
+        # when it is actually https
+        referer_proto, _, stripped_referer = referer.partition("://")
+        referer_proto = referer_proto.lower()
+        req_proto, _, stripped_full_url = full_url.partition("://")
+        req_proto = req_proto.lower()
+        if referer_proto != req_proto:
+            self.log.warning("Protocol mismatch: %s != %s", referer, full_url)
+            if req_proto == "https":
+                # insecure origin to secure target is not allowed
+                raise web.HTTPError(
+                    403, "Not allowing authorization form submitted from insecure page"
+                )
+        if stripped_referer != stripped_full_url:
             # OAuth post must be made to the URL it came from
-            self.log.error("OAuth POST from %s != %s", referer, full_url)
+            self.log.error("Original OAuth POST from %s != %s", referer, full_url)
+            self.log.error(
+                "Stripped OAuth POST from %s != %s", stripped_referer, stripped_full_url
+            )
             raise web.HTTPError(
                 403, "Authorization form must be sent from authorization page"
             )

jupyterhub/apihandlers/groups.py

@@ -3,7 +3,6 @@
 # Distributed under the terms of the Modified BSD License.
 import json
 
-from tornado import gen
 from tornado import web
 
 from .. import orm

jupyterhub/apihandlers/proxy.py

@@ -2,12 +2,9 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 import json
-from urllib.parse import urlparse
 
-from tornado import gen
 from tornado import web
 
-from .. import orm
 from ..utils import admin_only
 from .base import APIHandler
 

jupyterhub/apihandlers/services.py

@@ -23,6 +23,7 @@ def service_model(service):
         'command': service.command,
         'pid': service.proc.pid if service.proc else 0,
         'info': service.info,
+        'display': service.display,
     }
 
 

jupyterhub/apihandlers/users.py

@@ -625,14 +625,14 @@ def _parse_timestamp(timestamp):
 
     - raise HTTPError(400) on parse error
     - handle and strip tz info for internal consistency
-      (we use naïve utc timestamps everywhere)
+      (we use naive utc timestamps everywhere)
     """
     try:
         dt = parse_date(timestamp)
     except Exception:
         raise web.HTTPError(400, "Not a valid timestamp: %r", timestamp)
     if dt.tzinfo:
-        # strip timezone info to naïve UTC datetime
+        # strip timezone info to naive UTC datetime
         dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
 
     now = datetime.utcnow()

jupyterhub/app.py

@@ -55,6 +55,7 @@ from traitlets import (
     Instance,
     Bytes,
     Float,
+    Union,
     observe,
     default,
     validate,
@@ -76,6 +77,7 @@ from .user import UserDict
 from .oauth.provider import make_provider
 from ._data import DATA_FILES_PATH
 from .log import CoroutineLogFormatter, log_request
+from .pagination import Pagination
 from .proxy import Proxy, ConfigurableHTTPProxy
 from .traitlets import URLPrefix, Command, EntryPointType, Callable
 from .utils import (
@@ -278,7 +280,7 @@ class JupyterHub(Application):
 
     @default('classes')
     def _load_classes(self):
-        classes = [Spawner, Authenticator, CryptKeeper]
+        classes = [Spawner, Authenticator, CryptKeeper, Pagination]
         for name, trait in self.traits(config=True).items():
             # load entry point groups into configurable class list
             # so that they show up in config files, etc.
@@ -316,7 +318,7 @@ class JupyterHub(Application):
 
     @validate("config_file")
     def _validate_config_file(self, proposal):
-        if not os.path.isfile(proposal.value):
+        if not self.generate_config and not os.path.isfile(proposal.value):
             print(
                 "ERROR: Failed to find specified config file: {}".format(
                     proposal.value
@@ -561,10 +563,23 @@ class JupyterHub(Application):
     def _url_part_changed(self, change):
         """propagate deprecated ip/port/base_url config to the bind_url"""
         urlinfo = urlparse(self.bind_url)
-        urlinfo = urlinfo._replace(netloc='%s:%i' % (self.ip, self.port))
+        if ':' in self.ip:
+            fmt = '[%s]:%i'
+        else:
+            fmt = '%s:%i'
+        urlinfo = urlinfo._replace(netloc=fmt % (self.ip, self.port))
         urlinfo = urlinfo._replace(path=self.base_url)
         bind_url = urlunparse(urlinfo)
+
+        # Warn if both bind_url and ip/port/base_url are set
         if bind_url != self.bind_url:
+            if self.bind_url != self._bind_url_default():
+                self.log.warning(
+                    "Both bind_url and ip/port/base_url have been configured. "
+                    "JupyterHub.ip, JupyterHub.port, JupyterHub.base_url are"
+                    " deprecated in JupyterHub 0.9,"
+                    " please use JupyterHub.bind_url instead."
+                )
             self.bind_url = bind_url
 
     bind_url = Unicode(
@@ -576,6 +591,22 @@ class JupyterHub(Application):
         """,
     ).tag(config=True)
 
+    @validate('bind_url')
+    def _validate_bind_url(self, proposal):
+        """ensure protocol field of bind_url matches ssl"""
+        v = proposal['value']
+        proto, sep, rest = v.partition('://')
+        if self.ssl_cert and proto != 'https':
+            return 'https' + sep + rest
+        elif proto != 'http' and not self.ssl_cert:
+            return 'http' + sep + rest
+        return v
+
+    @default('bind_url')
+    def _bind_url_default(self):
+        proto = 'https' if self.ssl_cert else 'http'
+        return proto + '://:8000'
+
     subdomain_host = Unicode(
         '',
         help="""Run single-user servers on subdomains of this host.
@@ -711,10 +742,10 @@ class JupyterHub(Application):
         help="""The ip or hostname for proxies and spawners to use
         for connecting to the Hub.
 
-        Use when the bind address (`hub_ip`) is 0.0.0.0 or otherwise different
+        Use when the bind address (`hub_ip`) is 0.0.0.0, :: or otherwise different
         from the connect address.
 
-        Default: when `hub_ip` is 0.0.0.0, use `socket.gethostname()`, otherwise use `hub_ip`.
+        Default: when `hub_ip` is 0.0.0.0 or ::, use `socket.gethostname()`, otherwise use `hub_ip`.
 
         Note: Some spawners or proxy implementations might not support hostnames. Check your
         spawner or proxy documentation to see if they have extra requirements.
@@ -917,6 +948,25 @@ class JupyterHub(Application):
     def _authenticator_default(self):
         return self.authenticator_class(parent=self, db=self.db)
 
+    implicit_spawn_seconds = Float(
+        0,
+        help="""Trigger implicit spawns after this many seconds.
+
+        When a user visits a URL for a server that's not running,
+        they are shown a page indicating that the requested server
+        is not running with a button to spawn the server.
+
+        Setting this to a positive value will redirect the user
+        after this many seconds, effectively clicking this button
+        automatically for the users,
+        automatically beginning the spawn process.
+
+        Warning: this can result in errors and surprising behavior
+        when sharing access URLs to actual servers,
+        since the wrong server is likely to be started.
+        """,
+    ).tag(config=True)
+
     allow_named_servers = Bool(
         False, help="Allow named single-user servers per user"
     ).tag(config=True)
@@ -1266,12 +1316,25 @@ class JupyterHub(Application):
         """
     ).tag(config=True)
 
-    default_url = Unicode(
+    default_url = Union(
+        [Unicode(), Callable()],
         help="""
         The default URL for users when they arrive (e.g. when user directs to "/")
 
         By default, redirects users to their own server.
-        """
+
+        Can be a Unicode string (e.g. '/hub/home') or a callable based on the handler object:
+
+        ::
+        
+            def default_url_fn(handler):
+                user = handler.current_user
+                if user and user.admin:
+                    return '/hub/admin'
+                return '/hub/home'
+
+            c.JupyterHub.default_url = default_url_fn
+        """,
     ).tag(config=True)
 
     user_redirect_hook = Callable(
@@ -1641,22 +1704,22 @@ class JupyterHub(Application):
         # the admin_users config variable will never be used after this point.
         # only the database values will be referenced.
 
-        whitelist = [
+        allowed_users = [
             self.authenticator.normalize_username(name)
-            for name in self.authenticator.whitelist
+            for name in self.authenticator.allowed_users
         ]
-        self.authenticator.whitelist = set(whitelist)  # force normalization
-        for username in whitelist:
+        self.authenticator.allowed_users = set(allowed_users)  # force normalization
+        for username in allowed_users:
             if not self.authenticator.validate_username(username):
                 raise ValueError("username %r is not valid" % username)
 
-        if not whitelist:
+        if not allowed_users:
             self.log.info(
-                "Not using whitelist. Any authenticated user will be allowed."
+                "Not using allowed_users. Any authenticated user will be allowed."
             )
 
-        # add whitelisted users to the db
-        for name in whitelist:
+        # add allowed users to the db
+        for name in allowed_users:
             user = orm.User.find(db, name)
             if user is None:
                 user = orm.User(name=name)
@@ -1666,13 +1729,16 @@ class JupyterHub(Application):
         db.commit()
 
         # Notify authenticator of all users.
-        # This ensures Auth whitelist is up-to-date with the database.
-        # This lets whitelist be used to set up initial list,
-        # but changes to the whitelist can occur in the database,
+        # This ensures Authenticator.allowed_users is up-to-date with the database.
+        # This lets .allowed_users be used to set up initial list,
+        # but changes to the allowed_users set can occur in the database,
         # and persist across sessions.
+        total_users = 0
         for user in db.query(orm.User):
             try:
-                await maybe_future(self.authenticator.add_user(user))
+                f = self.authenticator.add_user(user)
+                if f:
+                    await maybe_future(f)
             except Exception:
                 self.log.exception("Error adding user %s already in db", user.name)
                 if self.authenticator.delete_invalid_users:
@@ -1694,6 +1760,7 @@ class JupyterHub(Application):
                         )
                     )
             else:
+                total_users += 1
                 # handle database upgrades where user.created is undefined.
                 # we don't want to allow user.created to be undefined,
                 # so initialize it to last_activity (if defined) or now.
@@ -1701,9 +1768,11 @@ class JupyterHub(Application):
                     user.created = user.last_activity or datetime.utcnow()
         db.commit()
 
-        # The whitelist set and the users in the db are now the same.
+        # The allowed_users set and the users in the db are now the same.
         # From this point on, any user changes should be done simultaneously
-        # to the whitelist set and user db, unless the whitelist is empty (all users allowed).
+        # to the allowed_users set and user db, unless the allowed set is empty (all users allowed).
+
+        TOTAL_USERS.set(total_users)
 
     async def init_groups(self):
         """Load predefined groups into the database"""
@@ -1716,11 +1785,11 @@ class JupyterHub(Application):
             for username in usernames:
                 username = self.authenticator.normalize_username(username)
                 if not (
-                    await maybe_future(
-                        self.authenticator.check_whitelist(username, None)
-                    )
+                    await maybe_future(self.authenticator.check_allowed(username, None))
                 ):
-                    raise ValueError("Username %r is not in whitelist" % username)
+                    raise ValueError(
+                        "Username %r is not in Authenticator.allowed_users" % username
+                    )
                 user = orm.User.find(db, name=username)
                 if user is None:
                     if not self.authenticator.validate_username(username):
@@ -1744,11 +1813,14 @@ class JupyterHub(Application):
             if kind == 'user':
                 name = self.authenticator.normalize_username(name)
                 if not (
-                    await maybe_future(self.authenticator.check_whitelist(name, None))
+                    await maybe_future(self.authenticator.check_allowed(name, None))
                 ):
-                    raise ValueError("Token name %r is not in whitelist" % name)
+                    raise ValueError(
+                        "Token user name %r is not in Authenticator.allowed_users"
+                        % name
+                    )
                 if not self.authenticator.validate_username(name):
-                    raise ValueError("Token name %r is not valid" % name)
+                    raise ValueError("Token user name %r is not valid" % name)
             if kind == 'service':
                 if not any(service["name"] == name for service in self.services):
                     self.log.warning(
@@ -1787,17 +1859,27 @@ class JupyterHub(Application):
     # purge expired tokens hourly
     purge_expired_tokens_interval = 3600
 
+    def purge_expired_tokens(self):
+        """purge all expiring token objects from the database
+
+        run periodically
+        """
+        # this should be all the subclasses of Expiring
+        for cls in (orm.APIToken, orm.OAuthAccessToken, orm.OAuthCode):
+            self.log.debug("Purging expired {name}s".format(name=cls.__name__))
+            cls.purge_expired(self.db)
+
     async def init_api_tokens(self):
         """Load predefined API tokens (for services) into database"""
         await self._add_tokens(self.service_tokens, kind='service')
         await self._add_tokens(self.api_tokens, kind='user')
-        purge_expired_tokens = partial(orm.APIToken.purge_expired, self.db)
-        purge_expired_tokens()
+
+        self.purge_expired_tokens()
         # purge expired tokens hourly
         # we don't need to be prompt about this
         # because expired tokens cannot be used anyway
         pc = PeriodicCallback(
-            purge_expired_tokens, 1e3 * self.purge_expired_tokens_interval
+            self.purge_expired_tokens, 1e3 * self.purge_expired_tokens_interval
         )
         pc.start()
 
@@ -2005,22 +2087,31 @@ class JupyterHub(Application):
             spawner._check_pending = False
 
         # parallelize checks for running Spawners
+        # run query on extant Server objects
+        # so this is O(running servers) not O(total users)
+        # Server objects can be associated with either a Spawner or a Service,
+        # we are only interested in the ones associated with a Spawner
         check_futures = []
-        for orm_user in db.query(orm.User):
-            user = self.users[orm_user]
-            self.log.debug("Loading state for %s from db", user.name)
-            for name, orm_spawner in user.orm_spawners.items():
-                if orm_spawner.server is not None:
-                    # spawner should be running
+        for orm_server in db.query(orm.Server):
+            orm_spawner = orm_server.spawner
+            if not orm_spawner:
+                # check for orphaned Server rows
+                # this shouldn't happen if we've got our sqlachemy right
+                if not orm_server.service:
+                    self.log.warning("deleting orphaned server %s", orm_server)
+                    self.db.delete(orm_server)
+                    self.db.commit()
+                continue
             # instantiate Spawner wrapper and check if it's still alive
-                    spawner = user.spawners[name]
+            # spawner should be running
+            user = self.users[orm_spawner.user]
+            spawner = user.spawners[orm_spawner.name]
+            self.log.debug("Loading state for %s from db", spawner._log_name)
             # signal that check is pending to avoid race conditions
             spawner._check_pending = True
-                    f = asyncio.ensure_future(check_spawner(user, name, spawner))
+            f = asyncio.ensure_future(check_spawner(user, spawner.name, spawner))
             check_futures.append(f)
 
-        TOTAL_USERS.set(len(self.users))
-
         # it's important that we get here before the first await
         # so that we know all spawners are instantiated and in the check-pending state
 
@@ -2120,14 +2211,14 @@ class JupyterHub(Application):
         else:
             version_hash = datetime.now().strftime("%Y%m%d%H%M%S")
 
-        oauth_no_confirm_whitelist = set()
+        oauth_no_confirm_list = set()
         for service in self._service_map.values():
             if service.oauth_no_confirm:
                 self.log.warning(
                     "Allowing service %s to complete OAuth without confirmation on an authorization web page",
                     service.name,
                 )
-                oauth_no_confirm_whitelist.add(service.oauth_client_id)
+                oauth_no_confirm_list.add(service.oauth_client_id)
 
         settings = dict(
             log_function=log_request,
@@ -2158,11 +2249,12 @@ class JupyterHub(Application):
             subdomain_host=self.subdomain_host,
             domain=self.domain,
             statsd=self.statsd,
+            implicit_spawn_seconds=self.implicit_spawn_seconds,
             allow_named_servers=self.allow_named_servers,
             default_server_name=self._default_server_name,
             named_server_limit_per_user=self.named_server_limit_per_user,
             oauth_provider=self.oauth_provider,
-            oauth_no_confirm_whitelist=oauth_no_confirm_whitelist,
+            oauth_no_confirm_list=oauth_no_confirm_list,
             concurrent_spawn_limit=self.concurrent_spawn_limit,
             spawn_throttle_retry_range=self.spawn_throttle_retry_range,
             active_server_limit=self.active_server_limit,
@@ -2296,7 +2388,6 @@ class JupyterHub(Application):
         if init_spawners_timeout < 0:
             # negative timeout means forever (previous, most stable behavior)
             init_spawners_timeout = 86400
-        print(init_spawners_timeout)
 
         init_start_time = time.perf_counter()
         init_spawners_future = asyncio.ensure_future(self.init_spawners())
@@ -2452,7 +2543,7 @@ class JupyterHub(Application):
                 continue
             dt = parse_date(route_data['last_activity'])
             if dt.tzinfo:
-                # strip timezone info to naïve UTC datetime
+                # strip timezone info to naive UTC datetime
                 dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
 
             if user.last_activity:
@@ -2665,6 +2756,40 @@ class JupyterHub(Application):
         self.log.critical("Received signalnum %s, , initiating shutdown...", signum)
         raise SystemExit(128 + signum)
 
+    def _init_asyncio_patch(self):
+        """Set default asyncio policy to be compatible with Tornado.
+
+        Tornado 6 (at least) is not compatible with the default
+        asyncio implementation on Windows.
+
+        Pick the older SelectorEventLoopPolicy on Windows
+        if the known-incompatible default policy is in use.
+
+        Do this as early as possible to make it a low priority and overrideable.
+
+        ref: https://github.com/tornadoweb/tornado/issues/2608
+
+        FIXME: If/when tornado supports the defaults in asyncio,
+               remove and bump tornado requirement for py38.
+        """
+        if sys.platform.startswith("win") and sys.version_info >= (3, 8):
+            try:
+                from asyncio import (
+                    WindowsProactorEventLoopPolicy,
+                    WindowsSelectorEventLoopPolicy,
+                )
+            except ImportError:
+                pass
+                # not affected
+            else:
+                if (
+                    type(asyncio.get_event_loop_policy())
+                    is WindowsProactorEventLoopPolicy
+                ):
+                    # WindowsProactorEventLoopPolicy is not compatible with Tornado 6.
+                    # Fallback to the pre-3.8 default of WindowsSelectorEventLoopPolicy.
+                    asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy())
+
     _atexit_ran = False
 
     def atexit(self):
@@ -2672,6 +2797,7 @@ class JupyterHub(Application):
         if self._atexit_ran:
             return
         self._atexit_ran = True
+        self._init_asyncio_patch()
         # run the cleanup step (in a new loop, because the interrupted one is unclean)
         asyncio.set_event_loop(asyncio.new_event_loop())
         IOLoop.clear_current()
@@ -2721,6 +2847,7 @@ class JupyterHub(Application):
     @classmethod
     def launch_instance(cls, argv=None):
         self = cls.instance()
+        self._init_asyncio_patch()
         loop = IOLoop.current()
         task = asyncio.ensure_future(self.launch_instance_async(argv))
         try:

jupyterhub/auth.py

@@ -7,6 +7,7 @@ import re
 import sys
 import warnings
 from concurrent.futures import ThreadPoolExecutor
+from functools import partial
 from shutil import which
 from subprocess import PIPE
 from subprocess import Popen
@@ -100,41 +101,74 @@ class Authenticator(LoggingConfigurable):
         """
     ).tag(config=True)
 
-    whitelist = Set(
+    whitelist = Set(help="Deprecated, use `Authenticator.allowed_users`", config=True,)
+
+    allowed_users = Set(
         help="""
-        Whitelist of usernames that are allowed to log in.
+        Set of usernames that are allowed to log in.
 
         Use this with supported authenticators to restrict which users can log in. This is an
-        additional whitelist that further restricts users, beyond whatever restrictions the
+        additional list that further restricts users, beyond whatever restrictions the
         authenticator has in place.
 
         If empty, does not perform any additional restriction.
+
+        .. versionchanged:: 1.2
+            `Authenticator.whitelist` renamed to `allowed_users`
         """
     ).tag(config=True)
 
-    blacklist = Set(
+    blocked_users = Set(
         help="""
-        Blacklist of usernames that are not allowed to log in.
+        Set of usernames that are not allowed to log in.
 
         Use this with supported authenticators to restrict which users can not log in. This is an
-        additional blacklist that further restricts users, beyond whatever restrictions the
+        additional block list that further restricts users, beyond whatever restrictions the
         authenticator has in place.
 
         If empty, does not perform any additional restriction.
 
         .. versionadded: 0.9
+
+        .. versionchanged:: 1.2
+            `Authenticator.blacklist` renamed to `blocked_users`
         """
     ).tag(config=True)
 
-    @observe('whitelist')
-    def _check_whitelist(self, change):
+    _deprecated_aliases = {
+        "whitelist": ("allowed_users", "1.2"),
+        "blacklist": ("blocked_users", "1.2"),
+    }
+
+    @observe(*list(_deprecated_aliases))
+    def _deprecated_trait(self, change):
+        """observer for deprecated traits"""
+        old_attr = change.name
+        new_attr, version = self._deprecated_aliases.get(old_attr)
+        new_value = getattr(self, new_attr)
+        if new_value != change.new:
+            # only warn if different
+            # protects backward-compatible config from warnings
+            # if they set the same value under both names
+            self.log.warning(
+                "{cls}.{old} is deprecated in JupyterHub {version}, use {cls}.{new} instead".format(
+                    cls=self.__class__.__name__,
+                    old=old_attr,
+                    new=new_attr,
+                    version=version,
+                )
+            )
+            setattr(self, new_attr, change.new)
+
+    @observe('allowed_users')
+    def _check_allowed_users(self, change):
         short_names = [name for name in change['new'] if len(name) <= 1]
         if short_names:
             sorted_names = sorted(short_names)
             single = ''.join(sorted_names)
             string_set_typo = "set('%s')" % single
             self.log.warning(
-                "whitelist contains single-character names: %s; did you mean set([%r]) instead of %s?",
+                "Allowed set contains single-character names: %s; did you mean set([%r]) instead of %s?",
                 sorted_names[:8],
                 single,
                 string_set_typo,
@@ -206,6 +240,7 @@ class Authenticator(LoggingConfigurable):
 
     delete_invalid_users = Bool(
         False,
+        config=True,
         help="""Delete any users from the database that do not pass validation
 
         When JupyterHub starts, `.add_user` will be called
@@ -260,39 +295,74 @@ class Authenticator(LoggingConfigurable):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        for method_name in (
-            'check_whitelist',
-            'check_blacklist',
-            'check_group_whitelist',
+        self._init_deprecated_methods()
+
+    def _init_deprecated_methods(self):
+        # handles deprecated signature *and* name
+        # with correct subclass override priority!
+        for old_name, new_name in (
+            ('check_whitelist', 'check_allowed'),
+            ('check_blacklist', 'check_blocked_users'),
+            ('check_group_whitelist', 'check_allowed_groups'),
         ):
-            original_method = getattr(self, method_name, None)
-            if original_method is None:
+            old_method = getattr(self, old_name, None)
+            if old_method is None:
                 # no such method (check_group_whitelist is optional)
                 continue
-            signature = inspect.signature(original_method)
-            if 'authentication' not in signature.parameters:
+
+            # allow old name to have higher priority
+            # if and only if it's defined in a later subclass
+            # than the new name
+            for cls in self.__class__.mro():
+                has_old_name = old_name in cls.__dict__
+                has_new_name = new_name in cls.__dict__
+                if has_new_name:
+                    break
+                if has_old_name and not has_new_name:
+                    warnings.warn(
+                        "{0}.{1} should be renamed to {0}.{2} for JupyterHub >= 1.2".format(
+                            cls.__name__, old_name, new_name
+                        ),
+                        DeprecationWarning,
+                    )
+                    # use old name instead of new
+                    # if old name is overridden in subclass
+                    def _new_calls_old(old_name, *args, **kwargs):
+                        return getattr(self, old_name)(*args, **kwargs)
+
+                    setattr(self, new_name, partial(_new_calls_old, old_name))
+                    break
+
+            # deprecate pre-1.0 method signatures
+            signature = inspect.signature(old_method)
+            if 'authentication' not in signature.parameters and not any(
+                param.kind == inspect.Parameter.VAR_KEYWORD
+                for param in signature.parameters.values()
+            ):
                 # adapt to pre-1.0 signature for compatibility
                 warnings.warn(
                     """
                     {0}.{1} does not support the authentication argument,
-                    added in JupyterHub 1.0.
+                    added in JupyterHub 1.0. and is renamed to {2} in JupyterHub 1.2.
 
                     It should have the signature:
 
-                    def {1}(self, username, authentication=None):
+                    def {2}(self, username, authentication=None):
                         ...
 
                     Adapting for compatibility.
                     """.format(
-                        self.__class__.__name__, method_name
+                        self.__class__.__name__, old_name, new_name
                     ),
                     DeprecationWarning,
                 )
 
-                def wrapped_method(username, authentication=None, **kwargs):
+                def wrapped_method(
+                    original_method, username, authentication=None, **kwargs
+                ):
                     return original_method(username, **kwargs)
 
-                setattr(self, method_name, wrapped_method)
+                setattr(self, old_name, partial(wrapped_method, old_method))
 
     async def run_post_auth_hook(self, handler, authentication):
         """
@@ -326,39 +396,45 @@ class Authenticator(LoggingConfigurable):
         username = self.username_map.get(username, username)
         return username
 
-    def check_whitelist(self, username, authentication=None):
-        """Check if a username is allowed to authenticate based on whitelist configuration
+    def check_allowed(self, username, authentication=None):
+        """Check if a username is allowed to authenticate based on configuration
 
         Return True if username is allowed, False otherwise.
-        No whitelist means any username is allowed.
+        No allowed_users set means any username is allowed.
 
-        Names are normalized *before* being checked against the whitelist.
+        Names are normalized *before* being checked against the allowed set.
 
         .. versionchanged:: 1.0
             Signature updated to accept authentication data and any future changes
-        """
-        if not self.whitelist:
-            # No whitelist means any name is allowed
-            return True
-        return username in self.whitelist
 
-    def check_blacklist(self, username, authentication=None):
-        """Check if a username is blocked to authenticate based on blacklist configuration
+        .. versionchanged:: 1.2
+            Renamed check_whitelist to check_allowed
+        """
+        if not self.allowed_users:
+            # No allowed set means any name is allowed
+            return True
+        return username in self.allowed_users
+
+    def check_blocked_users(self, username, authentication=None):
+        """Check if a username is blocked to authenticate based on Authenticator.blocked configuration
 
         Return True if username is allowed, False otherwise.
-        No blacklist means any username is allowed.
+        No block list means any username is allowed.
 
-        Names are normalized *before* being checked against the blacklist.
+        Names are normalized *before* being checked against the block list.
 
         .. versionadded: 0.9
 
         .. versionchanged:: 1.0
             Signature updated to accept authentication data as second argument
+
+        .. versionchanged:: 1.2
+            Renamed check_blacklist to check_blocked_users
         """
-        if not self.blacklist:
-            # No blacklist means any name is allowed
+        if not self.blocked_users:
+            # No block list means any name is allowed
             return True
-        return username not in self.blacklist
+        return username not in self.blocked_users
 
     async def get_authenticated_user(self, handler, data):
         """Authenticate the user who is attempting to log in
@@ -367,7 +443,7 @@ class Authenticator(LoggingConfigurable):
 
         This calls `authenticate`, which should be overridden in subclasses,
         normalizes the username if any normalization should be done,
-        and then validates the name in the whitelist.
+        and then validates the name in the allowed set.
 
         This is the outer API for authenticating a user.
         Subclasses should not override this method.
@@ -375,7 +451,7 @@ class Authenticator(LoggingConfigurable):
         The various stages can be overridden separately:
          - `authenticate` turns formdata into a username
          - `normalize_username` normalizes the username
-         - `check_whitelist` checks against the user whitelist
+         - `check_allowed` checks against the allowed usernames
 
         .. versionchanged:: 0.8
             return dict instead of username
@@ -389,7 +465,7 @@ class Authenticator(LoggingConfigurable):
         else:
             authenticated = {'name': authenticated}
         authenticated.setdefault('auth_state', None)
-        # Leave the default as None, but reevaluate later post-whitelist
+        # Leave the default as None, but reevaluate later post-allowed-check
         authenticated.setdefault('admin', None)
 
         # normalize the username
@@ -400,20 +476,18 @@ class Authenticator(LoggingConfigurable):
             self.log.warning("Disallowing invalid username %r.", username)
             return
 
-        blacklist_pass = await maybe_future(
-            self.check_blacklist(username, authenticated)
-        )
-        whitelist_pass = await maybe_future(
-            self.check_whitelist(username, authenticated)
+        blocked_pass = await maybe_future(
+            self.check_blocked_users(username, authenticated)
         )
+        allowed_pass = await maybe_future(self.check_allowed(username, authenticated))
 
-        if blacklist_pass:
+        if blocked_pass:
             pass
         else:
-            self.log.warning("User %r in blacklist. Stop authentication", username)
+            self.log.warning("User %r blocked. Stop authentication", username)
             return
 
-        if whitelist_pass:
+        if allowed_pass:
             if authenticated['admin'] is None:
                 authenticated['admin'] = await maybe_future(
                     self.is_admin(handler, authenticated)
@@ -423,7 +497,7 @@ class Authenticator(LoggingConfigurable):
 
             return authenticated
         else:
-            self.log.warning("User %r not in whitelist.", username)
+            self.log.warning("User %r not allowed.", username)
             return
 
     async def refresh_user(self, user, handler=None):
@@ -479,7 +553,7 @@ class Authenticator(LoggingConfigurable):
         It must return the username on successful authentication,
         and return None on failed authentication.
 
-        Checking the whitelist is handled separately by the caller.
+        Checking allowed_users/blocked_users is handled separately by the caller.
 
         .. versionchanged:: 0.8
             Allow `authenticate` to return a dict containing auth_state.
@@ -520,10 +594,10 @@ class Authenticator(LoggingConfigurable):
 
         This method may be a coroutine.
 
-        By default, this just adds the user to the whitelist.
+        By default, this just adds the user to the allowed_users set.
 
         Subclasses may do more extensive things, such as adding actual unix users,
-        but they should call super to ensure the whitelist is updated.
+        but they should call super to ensure the allowed_users set is updated.
 
         Note that this should be idempotent, since it is called whenever the hub restarts
         for all users.
@@ -533,19 +607,19 @@ class Authenticator(LoggingConfigurable):
         """
         if not self.validate_username(user.name):
             raise ValueError("Invalid username: %s" % user.name)
-        if self.whitelist:
-            self.whitelist.add(user.name)
+        if self.allowed_users:
+            self.allowed_users.add(user.name)
 
     def delete_user(self, user):
         """Hook called when a user is deleted
 
-        Removes the user from the whitelist.
-        Subclasses should call super to ensure the whitelist is updated.
+        Removes the user from the allowed_users set.
+        Subclasses should call super to ensure the allowed_users set is updated.
 
         Args:
             user (User): The User wrapper object
         """
-        self.whitelist.discard(user.name)
+        self.allowed_users.discard(user.name)
 
     auto_login = Bool(
         False,
@@ -610,6 +684,41 @@ class Authenticator(LoggingConfigurable):
         return [('/login', LoginHandler)]
 
 
+def _deprecated_method(old_name, new_name, version):
+    """Create a deprecated method wrapper for a deprecated method name"""
+
+    def deprecated(self, *args, **kwargs):
+        warnings.warn(
+            (
+                "{cls}.{old_name} is deprecated in JupyterHub {version}."
+                " Please use {cls}.{new_name} instead."
+            ).format(
+                cls=self.__class__.__name__,
+                old_name=old_name,
+                new_name=new_name,
+                version=version,
+            ),
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        old_method = getattr(self, new_name)
+        return old_method(*args, **kwargs)
+
+    return deprecated
+
+
+import types
+
+# deprecate white/blacklist method names
+for _old_name, _new_name, _version in [
+    ("check_whitelist", "check_allowed", "1.2"),
+    ("check_blacklist", "check_blocked_users", "1.2"),
+]:
+    setattr(
+        Authenticator, _old_name, _deprecated_method(_old_name, _new_name, _version),
+    )
+
+
 class LocalAuthenticator(Authenticator):
     """Base class for Authenticators that work with local Linux/UNIX users
 
@@ -669,37 +778,37 @@ class LocalAuthenticator(Authenticator):
         """
     ).tag(config=True)
 
-    group_whitelist = Set(
-        help="""
-        Whitelist all users from this UNIX group.
+    group_whitelist = Set(help="""DEPRECATED: use allowed_groups""",).tag(config=True)
 
-        This makes the username whitelist ineffective.
+    allowed_groups = Set(
+        help="""
+        Allow login from all users in these UNIX groups.
+
+        If set, allowed username set is ignored.
         """
     ).tag(config=True)
 
-    @observe('group_whitelist')
-    def _group_whitelist_changed(self, change):
-        """
-        Log a warning if both group_whitelist and user whitelist are set.
-        """
-        if self.whitelist:
+    @observe('allowed_groups')
+    def _allowed_groups_changed(self, change):
+        """Log a warning if mutually exclusive user and group allowed sets are specified."""
+        if self.allowed_users:
             self.log.warning(
-                "Ignoring username whitelist because group whitelist supplied!"
+                "Ignoring Authenticator.allowed_users set because Authenticator.allowed_groups supplied!"
             )
 
-    def check_whitelist(self, username, authentication=None):
-        if self.group_whitelist:
-            return self.check_group_whitelist(username, authentication)
+    def check_allowed(self, username, authentication=None):
+        if self.allowed_groups:
+            return self.check_allowed_groups(username, authentication)
         else:
-            return super().check_whitelist(username, authentication)
+            return super().check_allowed(username, authentication)
 
-    def check_group_whitelist(self, username, authentication=None):
+    def check_allowed_groups(self, username, authentication=None):
         """
-        If group_whitelist is configured, check if authenticating user is part of group.
+        If allowed_groups is configured, check if authenticating user is part of group.
         """
-        if not self.group_whitelist:
+        if not self.allowed_groups:
             return False
-        for grnam in self.group_whitelist:
+        for grnam in self.allowed_groups:
             try:
                 group = self._getgrnam(grnam)
             except KeyError:
@@ -843,7 +952,7 @@ class PAMAuthenticator(LocalAuthenticator):
         Authoritative list of user groups that determine admin access.
         Users not in these groups can still be granted admin status through admin_users.
 
-        White/blacklisting rules still apply.
+        allowed/blocked rules still apply.
         """
     ).tag(config=True)
 
@@ -986,6 +1095,16 @@ class PAMAuthenticator(LocalAuthenticator):
             return super().normalize_username(username)
 
 
+for _old_name, _new_name, _version in [
+    ("check_group_whitelist", "check_group_allowed", "1.2"),
+]:
+    setattr(
+        LocalAuthenticator,
+        _old_name,
+        _deprecated_method(_old_name, _new_name, _version),
+    )
+
+
 class DummyAuthenticator(Authenticator):
     """Dummy Authenticator for testing
 

jupyterhub/crypto.py

@@ -6,7 +6,6 @@ from concurrent.futures import ThreadPoolExecutor
 
 from traitlets import Any
 from traitlets import default
-from traitlets import Dict
 from traitlets import Integer
 from traitlets import List
 from traitlets import observe

jupyterhub/handlers/base.py

@@ -2,7 +2,6 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 import asyncio
-import copy
 import json
 import math
 import random
@@ -27,14 +26,12 @@ from tornado.httputil import url_concat
 from tornado.ioloop import IOLoop
 from tornado.log import app_log
 from tornado.web import addslash
-from tornado.web import MissingArgumentError
 from tornado.web import RequestHandler
 
 from .. import __version__
 from .. import orm
 from ..metrics import PROXY_ADD_DURATION_SECONDS
 from ..metrics import PROXY_DELETE_DURATION_SECONDS
-from ..metrics import ProxyAddStatus
 from ..metrics import ProxyDeleteStatus
 from ..metrics import RUNNING_SERVERS
 from ..metrics import SERVER_POLL_DURATION_SECONDS
@@ -637,9 +634,22 @@ class BaseHandler(RequestHandler):
                 next_url,
             )
 
+        # this is where we know if next_url is coming from ?next= param or we are using a default url
+        if next_url:
+            next_url_from_param = True
+        else:
+            next_url_from_param = False
+
         if not next_url:
-            # custom default URL
-            next_url = default or self.default_url
+            # custom default URL, usually passed because user landed on that page but was not logged in
+            if default:
+                next_url = default
+            else:
+                # As set in jupyterhub_config.py
+                if callable(self.default_url):
+                    next_url = self.default_url(self)
+                else:
+                    next_url = self.default_url
 
         if not next_url:
             # default URL after login
@@ -654,8 +664,45 @@ class BaseHandler(RequestHandler):
                     next_url = url_path_join(self.hub.base_url, 'spawn')
             else:
                 next_url = url_path_join(self.hub.base_url, 'home')
+
+        if not next_url_from_param:
+            # when a request made with ?next=... assume all the params have already been encoded
+            # otherwise, preserve params from the current request across the redirect
+            next_url = self.append_query_parameters(next_url, exclude=['next'])
         return next_url
 
+    def append_query_parameters(self, url, exclude=None):
+        """Append the current request's query parameters to the given URL.
+
+        Supports an extra optional parameter ``exclude`` that when provided must
+        contain a list of parameters to be ignored, i.e. these parameters will
+        not be added to the URL.
+
+        This is important to avoid infinite loops with the next parameter being
+        added over and over, for instance.
+
+        The default value for ``exclude`` is an array with "next". This is useful
+        as most use cases in JupyterHub (all?) won't want to include the next
+        parameter twice (the next parameter is added elsewhere to the query
+        parameters).
+
+        :param str url: a URL
+        :param list exclude: optional list of parameters to be ignored, defaults to
+        a list with "next" (to avoid redirect-loops)
+        :rtype (str)
+        """
+        if exclude is None:
+            exclude = ['next']
+        if self.request.query:
+            query_string = [
+                param
+                for param in parse_qsl(self.request.query)
+                if param[0] not in exclude
+            ]
+            if query_string:
+                url = url_concat(url, query_string)
+        return url
+
     async def auth_to_user(self, authenticated, user=None):
         """Persist data from .authenticate() or .refresh_user() to the User database
 
@@ -676,9 +723,10 @@ class BaseHandler(RequestHandler):
             raise ValueError("Username doesn't match! %s != %s" % (username, user.name))
 
         if user is None:
-            new_user = username not in self.users
-            user = self.user_from_username(username)
+            user = self.find_user(username)
+            new_user = user is None
             if new_user:
+                user = self.user_from_username(username)
                 await maybe_future(self.authenticator.add_user(user))
         # Only set `admin` if the authenticator returned an explicit value.
         if admin is not None and admin != user.admin:
@@ -877,7 +925,7 @@ class BaseHandler(RequestHandler):
                 self.log.error(
                     "Stopping %s to avoid inconsistent state", user_server_name
                 )
-                await user.stop()
+                await user.stop(server_name)
                 PROXY_ADD_DURATION_SECONDS.labels(status='failure').observe(
                     time.perf_counter() - proxy_add_start_time
                 )
@@ -910,6 +958,9 @@ class BaseHandler(RequestHandler):
                 self.settings['failure_count'] = 0
                 return
             # spawn failed, increment count and abort if limit reached
+            SERVER_SPAWN_DURATION_SECONDS.labels(
+                status=ServerSpawnStatus.failure
+            ).observe(time.perf_counter() - spawn_start_time)
             self.settings.setdefault('failure_count', 0)
             self.settings['failure_count'] += 1
             failure_count = self.settings['failure_count']
@@ -942,6 +993,9 @@ class BaseHandler(RequestHandler):
             # waiting_for_response indicates server process has started,
             # but is yet to become responsive.
             if spawner._spawn_pending and not spawner._waiting_for_response:
+                # If slow_spawn_timeout is intentionally disabled then we
+                # don't need to log a warning, just return.
+                if self.slow_spawn_timeout > 0:
                     # still in Spawner.start, which is taking a long time
                     # we shouldn't poll while spawn is incomplete.
                     self.log.warning(
@@ -1085,7 +1139,10 @@ class BaseHandler(RequestHandler):
         except gen.TimeoutError:
             # hit timeout, but stop is still pending
             self.log.warning(
-                "User %s:%s server is slow to stop", user.name, server_name
+                "User %s:%s server is slow to stop (timeout=%s)",
+                user.name,
+                server_name,
+                self.slow_stop_timeout,
             )
 
         # return handle on the future for hooking up callbacks
@@ -1144,6 +1201,8 @@ class BaseHandler(RequestHandler):
         for service in self.services.values():
             if not service.url:
                 continue
+            if not service.display:
+                continue
             accessible_services.append(service)
         return accessible_services
 
@@ -1426,11 +1485,12 @@ class UserUrlHandler(BaseHandler):
         # serve a page prompting for spawn and 503 error
         # visiting /user/:name no longer triggers implicit spawn
         # without explicit user action
-        self.set_status(503)
         spawn_url = url_concat(
             url_path_join(self.hub.base_url, "spawn", user.escaped_name, server_name),
             {"next": self.request.uri},
         )
+        self.set_status(503)
+
         auth_state = await user.get_auth_state()
         html = self.render_template(
             "not_running.html",
@@ -1438,6 +1498,7 @@ class UserUrlHandler(BaseHandler):
             server_name=server_name,
             spawn_url=spawn_url,
             auth_state=auth_state,
+            implicit_spawn_seconds=self.settings.get("implicit_spawn_seconds", 0),
         )
         self.finish(html)
 

jupyterhub/handlers/metrics.py

@@ -1,7 +1,6 @@
 from prometheus_client import CONTENT_TYPE_LATEST
 from prometheus_client import generate_latest
 from prometheus_client import REGISTRY
-from tornado import gen
 
 from ..utils import metrics_authentication
 from .base import BaseHandler

jupyterhub/handlers/pages.py

@@ -2,22 +2,21 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 import asyncio
-import codecs
-import copy
 import time
 from collections import defaultdict
 from datetime import datetime
 from http.client import responses
 
 from jinja2 import TemplateNotFound
-from tornado import gen
 from tornado import web
 from tornado.httputil import url_concat
+from tornado.httputil import urlparse
 
 from .. import __version__
 from .. import orm
 from ..metrics import SERVER_POLL_DURATION_SECONDS
 from ..metrics import ServerPollStatus
+from ..pagination import Pagination
 from ..utils import admin_only
 from ..utils import maybe_future
 from ..utils import url_path_join
@@ -41,11 +40,15 @@ class RootHandler(BaseHandler):
     def get(self):
         user = self.current_user
         if self.default_url:
+            # As set in jupyterhub_config.py
+            if callable(self.default_url):
+                url = self.default_url(self)
+            else:
                 url = self.default_url
         elif user:
             url = self.get_next_url(user)
         else:
-            url = self.settings['login_url']
+            url = url_concat(self.settings["login_url"], dict(next=self.request.uri))
         self.redirect(url)
 
 
@@ -151,17 +154,7 @@ class SpawnHandler(BaseHandler):
 
         spawner = user.spawners[server_name]
 
-        # resolve `?next=...`, falling back on the spawn-pending url
-        # must not be /user/server for named servers,
-        # which may get handled by the default server if they aren't ready yet
-
-        pending_url = url_path_join(
-            self.hub.base_url, "spawn-pending", user.escaped_name, server_name
-        )
-
-        if self.get_argument('next', None):
-            # preserve `?next=...` through spawn-pending
-            pending_url = url_concat(pending_url, {'next': self.get_argument('next')})
+        pending_url = self._get_pending_url(user, server_name)
 
         # spawner is active, redirect back to get progress, etc.
         if spawner.ready:
@@ -183,35 +176,50 @@ class SpawnHandler(BaseHandler):
         auth_state = await user.get_auth_state()
         await spawner.run_auth_state_hook(auth_state)
 
+        # Try to start server directly when query arguments are passed.
+        error_message = ''
+        query_options = {}
+        for key, byte_list in self.request.query_arguments.items():
+            query_options[key] = [bs.decode('utf8') for bs in byte_list]
+
+        # 'next' is reserved argument for redirect after spawn
+        query_options.pop('next', None)
+
+        if len(query_options) > 0:
+            try:
+                self.log.debug(
+                    "Triggering spawn with supplied query arguments for %s",
+                    spawner._log_name,
+                )
+                options = await maybe_future(spawner.options_from_query(query_options))
+                pending_url = self._get_pending_url(user, server_name)
+                return await self._wrap_spawn_single_user(
+                    user, server_name, spawner, pending_url, options
+                )
+            except Exception as e:
+                self.log.error(
+                    "Failed to spawn single-user server with query arguments",
+                    exc_info=True,
+                )
+                error_message = str(e)
+                # fallback to behavior without failing query arguments
+
         spawner_options_form = await spawner.get_options_form()
         if spawner_options_form:
             self.log.debug("Serving options form for %s", spawner._log_name)
             form = await self._render_form(
-                for_user=user, spawner_options_form=spawner_options_form
+                for_user=user,
+                spawner_options_form=spawner_options_form,
+                message=error_message,
             )
             self.finish(form)
         else:
             self.log.debug(
                 "Triggering spawn with default options for %s", spawner._log_name
             )
-            # Explicit spawn request: clear _spawn_future
-            # which may have been saved to prevent implicit spawns
-            # after a failure.
-            if spawner._spawn_future and spawner._spawn_future.done():
-                spawner._spawn_future = None
-            # not running, no form. Trigger spawn and redirect back to /user/:name
-            f = asyncio.ensure_future(self.spawn_single_user(user, server_name))
-            done, pending = await asyncio.wait([f], timeout=1)
-            # If spawn_single_user throws an exception, raise a 500 error
-            # otherwise it may cause a redirect loop
-            if f.done() and f.exception():
-                exc = f.exception()
-                raise web.HTTPError(
-                    500,
-                    "Error in Authenticator.pre_spawn_start: %s %s"
-                    % (type(exc).__name__, str(exc)),
+            return await self._wrap_spawn_single_user(
+                user, server_name, spawner, pending_url
             )
-            self.redirect(pending_url)
 
     @web.authenticated
     async def post(self, for_user=None, server_name=''):
@@ -241,8 +249,14 @@ class SpawnHandler(BaseHandler):
         for key, byte_list in self.request.files.items():
             form_options["%s_file" % key] = byte_list
         try:
+            self.log.debug(
+                "Triggering spawn with supplied form options for %s", spawner._log_name
+            )
             options = await maybe_future(spawner.options_from_form(form_options))
-            await self.spawn_single_user(user, server_name=server_name, options=options)
+            pending_url = self._get_pending_url(user, server_name)
+            return await self._wrap_spawn_single_user(
+                user, server_name, spawner, pending_url, options
+            )
         except Exception as e:
             self.log.error(
                 "Failed to spawn single-user server with form", exc_info=True
@@ -263,6 +277,47 @@ class SpawnHandler(BaseHandler):
         )
         self.redirect(next_url)
 
+    def _get_pending_url(self, user, server_name):
+        # resolve `?next=...`, falling back on the spawn-pending url
+        # must not be /user/server for named servers,
+        # which may get handled by the default server if they aren't ready yet
+
+        pending_url = url_path_join(
+            self.hub.base_url, "spawn-pending", user.escaped_name, server_name
+        )
+
+        pending_url = self.append_query_parameters(pending_url, exclude=['next'])
+
+        if self.get_argument('next', None):
+            # preserve `?next=...` through spawn-pending
+            pending_url = url_concat(pending_url, {'next': self.get_argument('next')})
+
+        return pending_url
+
+    async def _wrap_spawn_single_user(
+        self, user, server_name, spawner, pending_url, options=None
+    ):
+        # Explicit spawn request: clear _spawn_future
+        # which may have been saved to prevent implicit spawns
+        # after a failure.
+        if spawner._spawn_future and spawner._spawn_future.done():
+            spawner._spawn_future = None
+        # not running, no form. Trigger spawn and redirect back to /user/:name
+        f = asyncio.ensure_future(
+            self.spawn_single_user(user, server_name, options=options)
+        )
+        done, pending = await asyncio.wait([f], timeout=1)
+        # If spawn_single_user throws an exception, raise a 500 error
+        # otherwise it may cause a redirect loop
+        if f.done() and f.exception():
+            exc = f.exception()
+            raise web.HTTPError(
+                500,
+                "Error in Authenticator.pre_spawn_start: %s %s"
+                % (type(exc).__name__, str(exc)),
+            )
+        return self.redirect(pending_url)
+
 
 class SpawnPendingHandler(BaseHandler):
     """Handle /hub/spawn-pending/:user/:server
@@ -402,12 +457,15 @@ class AdminHandler(BaseHandler):
     @web.authenticated
     @admin_only
     async def get(self):
+        page, per_page, offset = Pagination(config=self.config).get_page_args(self)
+
         available = {'name', 'admin', 'running', 'last_activity'}
         default_sort = ['admin', 'name']
         mapping = {'running': orm.Spawner.server_id}
         for name in available:
             if name not in mapping:
-                mapping[name] = getattr(orm.User, name)
+                table = orm.User if name != "last_activity" else orm.Spawner
+                mapping[name] = getattr(table, name)
 
         default_order = {
             'name': 'asc',
@@ -442,14 +500,28 @@ class AdminHandler(BaseHandler):
         # get User.col.desc() order objects
         ordered = [getattr(c, o)() for c, o in zip(cols, orders)]
 
-        users = self.db.query(orm.User).outerjoin(orm.Spawner).order_by(*ordered)
+        users = (
+            self.db.query(orm.User)
+            .outerjoin(orm.Spawner)
+            .order_by(*ordered)
+            .limit(per_page)
+            .offset(offset)
+        )
         users = [self._user_from_orm(u) for u in users]
-        from itertools import chain
 
         running = []
         for u in users:
             running.extend(s for s in u.spawners.values() if s.active)
 
+        total = self.db.query(orm.User.id).count()
+        pagination = Pagination(
+            url=self.request.uri,
+            total=total,
+            page=page,
+            per_page=per_page,
+            config=self.config,
+        )
+
         auth_state = await self.current_user.get_auth_state()
         html = self.render_template(
             'admin.html',
@@ -462,6 +534,7 @@ class AdminHandler(BaseHandler):
             allow_named_servers=self.allow_named_servers,
             named_server_limit_per_user=self.named_server_limit_per_user,
             server_version='{} {}'.format(__version__, self.version_hash),
+            pagination=pagination,
         )
         self.finish(html)
 
@@ -585,10 +658,14 @@ class ProxyErrorHandler(BaseHandler):
 
 
 class HealthCheckHandler(BaseHandler):
-    """Answer to health check"""
+    """Serve health check probes as quickly as possible"""
 
-    def get(self, *args):
-        self.finish()
+    # There is nothing for us to do other than return a positive
+    # HTTP status code as quickly as possible for GET or HEAD requests
+    def get(self):
+        pass
+
+    head = get
 
 
 default_handlers = [

jupyterhub/log.py

@@ -12,6 +12,7 @@ from tornado.log import LogFormatter
 from tornado.web import HTTPError
 from tornado.web import StaticFileHandler
 
+from .handlers.pages import HealthCheckHandler
 from .metrics import prometheus_log_method
 
 
@@ -98,8 +99,12 @@ def _scrub_headers(headers):
     headers = dict(headers)
     if 'Authorization' in headers:
         auth = headers['Authorization']
-        if auth.startswith('token '):
-            headers['Authorization'] = 'token [secret]'
+        if ' ' in auth:
+            auth_type = auth.split(' ', 1)[0]
+        else:
+            # no space, hide the whole thing in case there was a mistake
+            auth_type = ''
+        headers['Authorization'] = '{} [secret]'.format(auth_type)
     if 'Cookie' in headers:
         c = SimpleCookie(headers['Cookie'])
         redacted = []
@@ -123,7 +128,9 @@ def log_request(handler):
     """
     status = handler.get_status()
     request = handler.request
-    if status == 304 or (status < 300 and isinstance(handler, StaticFileHandler)):
+    if status == 304 or (
+        status < 300 and isinstance(handler, (StaticFileHandler, HealthCheckHandler))
+    ):
         # static-file success and 304 Found are debug-level
         log_method = access_log.debug
     elif status < 400:

jupyterhub/metrics.py

@@ -175,9 +175,9 @@ def prometheus_log_method(handler):
     Tornado log handler for recording RED metrics.
 
     We record the following metrics:
-       Rate – the number of requests, per second, your services are serving.
-       Errors – the number of failed requests per second.
-       Duration – The amount of time each request takes expressed as a time interval.
+       Rate: the number of requests, per second, your services are serving.
+       Errors: the number of failed requests per second.
+       Duration: the amount of time each request takes expressed as a time interval.
 
     We use a fully qualified name of the handler as a label,
     rather than every url path to reduce cardinality.

jupyterhub/oauth/provider.py

@@ -2,16 +2,11 @@
 
 implements https://oauthlib.readthedocs.io/en/latest/oauth2/server.html
 """
-from datetime import datetime
-from urllib.parse import urlparse
-
 from oauthlib import uri_validate
 from oauthlib.oauth2 import RequestValidator
 from oauthlib.oauth2 import WebApplicationServer
 from oauthlib.oauth2.rfc6749.grant_types import authorization_code
 from oauthlib.oauth2.rfc6749.grant_types import base
-from sqlalchemy.orm import scoped_session
-from tornado import web
 from tornado.escape import url_escape
 from tornado.log import app_log
 
@@ -250,7 +245,7 @@ class JupyterHubRequestValidator(RequestValidator):
             client=orm_client,
             code=code['code'],
             # oauth has 5 minutes to complete
-            expires_at=int(datetime.utcnow().timestamp() + 300),
+            expires_at=int(orm.OAuthCode.now() + 300),
             # TODO: persist oauth scopes
             # scopes=request.scopes,
             user=request.user.orm_user,
@@ -347,7 +342,7 @@ class JupyterHubRequestValidator(RequestValidator):
         orm_access_token = orm.OAuthAccessToken(
             client=client,
             grant_type=orm.GrantType.authorization_code,
-            expires_at=datetime.utcnow().timestamp() + token['expires_in'],
+            expires_at=orm.OAuthAccessToken.now() + token['expires_in'],
             refresh_token=token['refresh_token'],
             # TODO: save scopes,
             # scopes=scopes,
@@ -441,7 +436,7 @@ class JupyterHubRequestValidator(RequestValidator):
         Method is used by:
             - Authorization Code Grant
         """
-        orm_code = self.db.query(orm.OAuthCode).filter_by(code=code).first()
+        orm_code = orm.OAuthCode.find(self.db, code=code)
         if orm_code is None:
             app_log.debug("No such code: %s", code)
             return False

jupyterhub/objects.py

@@ -53,7 +53,7 @@ class Server(HasTraits):
         Never used in APIs, only logging,
         since it can be non-connectable value, such as '', meaning all interfaces.
         """
-        if self.ip in {'', '0.0.0.0'}:
+        if self.ip in {'', '0.0.0.0', '::'}:
             return self.url.replace(self._connect_ip, self.ip or '*', 1)
         return self.url
 
@@ -87,13 +87,13 @@ class Server(HasTraits):
         """The address to use when connecting to this server
 
         When `ip` is set to a real ip address, the same value is used.
-        When `ip` refers to 'all interfaces' (e.g. '0.0.0.0'),
+        When `ip` refers to 'all interfaces' (e.g. '0.0.0.0' or '::'),
         clients connect via hostname by default.
         Setting `connect_ip` explicitly overrides any default behavior.
         """
         if self.connect_ip:
             return self.connect_ip
-        elif self.ip in {'', '0.0.0.0'}:
+        elif self.ip in {'', '0.0.0.0', '::'}:
             # if listening on all interfaces, default to hostname for connect
             return socket.gethostname()
         else:
@@ -149,7 +149,12 @@ class Server(HasTraits):
         if self.connect_url:
             parsed = urlparse(self.connect_url)
             return "{proto}://{host}".format(proto=parsed.scheme, host=parsed.netloc)
-        return "{proto}://{ip}:{port}".format(
+
+        if ':' in self._connect_ip:
+            fmt = "{proto}://[{ip}]:{port}"
+        else:
+            fmt = "{proto}://{ip}:{port}"
+        return fmt.format(
             proto=self.proto, ip=self._connect_ip, port=self._connect_port
         )
 

jupyterhub/orm.py

@@ -26,6 +26,7 @@ from sqlalchemy import select
 from sqlalchemy import Table
 from sqlalchemy import Unicode
 from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import backref
 from sqlalchemy.orm import interfaces
 from sqlalchemy.orm import object_session
 from sqlalchemy.orm import relationship
@@ -230,7 +231,12 @@ class Spawner(Base):
     user_id = Column(Integer, ForeignKey('users.id', ondelete='CASCADE'))
 
     server_id = Column(Integer, ForeignKey('servers.id', ondelete='SET NULL'))
-    server = relationship(Server, cascade="all")
+    server = relationship(
+        Server,
+        backref=backref('spawner', uselist=False),
+        single_parent=True,
+        cascade="all, delete-orphan",
+    )
 
     state = Column(JSONDict)
     name = Column(Unicode(255))
@@ -282,7 +288,12 @@ class Service(Base):
 
     # service-specific interface
     _server_id = Column(Integer, ForeignKey('servers.id', ondelete='SET NULL'))
-    server = relationship(Server, cascade='all')
+    server = relationship(
+        Server,
+        backref=backref('service', uselist=False),
+        single_parent=True,
+        cascade="all, delete-orphan",
+    )
     pid = Column(Integer)
 
     def new_api_token(self, token=None, **kwargs):
@@ -300,7 +311,46 @@ class Service(Base):
         return db.query(cls).filter(cls.name == name).first()
 
 
-class Hashed(object):
+class Expiring:
+    """Mixin for expiring entries
+
+    Subclass must define at least expires_at property,
+    which should be unix timestamp or datetime object
+    """
+
+    now = utcnow  # funciton, must return float timestamp or datetime
+    expires_at = None  # must be defined
+
+    @property
+    def expires_in(self):
+        """Property returning expiration in seconds from now
+
+        or None
+        """
+        if self.expires_at:
+            delta = self.expires_at - self.now()
+            if isinstance(delta, timedelta):
+                delta = delta.total_seconds()
+            return delta
+        else:
+            return None
+
+    @classmethod
+    def purge_expired(cls, db):
+        """Purge expired API Tokens from the database"""
+        now = cls.now()
+        deleted = False
+        for obj in (
+            db.query(cls).filter(cls.expires_at != None).filter(cls.expires_at < now)
+        ):
+            app_log.debug("Purging expired %s", obj)
+            deleted = True
+            db.delete(obj)
+        if deleted:
+            db.commit()
+
+
+class Hashed(Expiring):
     """Mixin for tables with hashed tokens"""
 
     prefix_length = 4
@@ -357,11 +407,21 @@ class Hashed(object):
         """Start the query for matching token.
 
         Returns an SQLAlchemy query already filtered by prefix-matches.
+
+        .. versionchanged:: 1.2
+
+            Excludes expired matches.
         """
         prefix = token[: cls.prefix_length]
         # since we can't filter on hashed values, filter on prefix
         # so we aren't comparing with all tokens
-        return db.query(cls).filter(bindparam('prefix', prefix).startswith(cls.prefix))
+        prefix_match = db.query(cls).filter(
+            bindparam('prefix', prefix).startswith(cls.prefix)
+        )
+        prefix_match = prefix_match.filter(
+            or_(cls.expires_at == None, cls.expires_at >= cls.now())
+        )
+        return prefix_match
 
     @classmethod
     def find(cls, db, token):
@@ -397,6 +457,7 @@ class APIToken(Hashed, Base):
         return 'a%i' % self.id
 
     # token metadata for bookkeeping
+    now = datetime.utcnow  # for expiry
     created = Column(DateTime, default=datetime.utcnow)
     expires_at = Column(DateTime, default=None, nullable=True)
     last_activity = Column(DateTime)
@@ -417,20 +478,6 @@ class APIToken(Hashed, Base):
             cls=self.__class__.__name__, pre=self.prefix, kind=kind, name=name
         )
 
-    @classmethod
-    def purge_expired(cls, db):
-        """Purge expired API Tokens from the database"""
-        now = utcnow()
-        deleted = False
-        for token in (
-            db.query(cls).filter(cls.expires_at != None).filter(cls.expires_at < now)
-        ):
-            app_log.debug("Purging expired %s", token)
-            deleted = True
-            db.delete(token)
-        if deleted:
-            db.commit()
-
     @classmethod
     def find(cls, db, token, *, kind=None):
         """Find a token object by value.
@@ -441,9 +488,6 @@ class APIToken(Hashed, Base):
         `kind='service'` only returns API tokens for services
         """
         prefix_match = cls.find_prefix(db, token)
-        prefix_match = prefix_match.filter(
-            or_(cls.expires_at == None, cls.expires_at >= utcnow())
-        )
         if kind == 'user':
             prefix_match = prefix_match.filter(cls.user_id != None)
         elif kind == 'service':
@@ -486,7 +530,7 @@ class APIToken(Hashed, Base):
             assert service.id is not None
             orm_token.service = service
         if expires_in is not None:
-            orm_token.expires_at = utcnow() + timedelta(seconds=expires_in)
+            orm_token.expires_at = cls.now() + timedelta(seconds=expires_in)
         db.add(orm_token)
         db.commit()
         return token
@@ -510,6 +554,10 @@ class OAuthAccessToken(Hashed, Base):
     __tablename__ = 'oauth_access_tokens'
     id = Column(Integer, primary_key=True, autoincrement=True)
 
+    @staticmethod
+    def now():
+        return datetime.utcnow().timestamp()
+
     @property
     def api_id(self):
         return 'o%i' % self.id
@@ -536,11 +584,12 @@ class OAuthAccessToken(Hashed, Base):
     last_activity = Column(DateTime, nullable=True)
 
     def __repr__(self):
-        return "<{cls}('{prefix}...', client_id={client_id!r}, user={user!r}>".format(
+        return "<{cls}('{prefix}...', client_id={client_id!r}, user={user!r}, expires_in={expires_in}>".format(
             cls=self.__class__.__name__,
             client_id=self.client_id,
             user=self.user and self.user.name,
             prefix=self.prefix,
+            expires_in=self.expires_in,
         )
 
     @classmethod
@@ -557,8 +606,9 @@ class OAuthAccessToken(Hashed, Base):
         return orm_token
 
 
-class OAuthCode(Base):
+class OAuthCode(Expiring, Base):
     __tablename__ = 'oauth_codes'
+
     id = Column(Integer, primary_key=True, autoincrement=True)
     client_id = Column(
         Unicode(255), ForeignKey('oauth_clients.identifier', ondelete='CASCADE')
@@ -570,6 +620,19 @@ class OAuthCode(Base):
     # state = Column(Unicode(1023))
     user_id = Column(Integer, ForeignKey('users.id', ondelete='CASCADE'))
 
+    @staticmethod
+    def now():
+        return datetime.utcnow().timestamp()
+
+    @classmethod
+    def find(cls, db, code):
+        return (
+            db.query(cls)
+            .filter(cls.code == code)
+            .filter(or_(cls.expires_at == None, cls.expires_at >= cls.now()))
+            .first()
+        )
+
 
 class OAuthClient(Base):
     __tablename__ = 'oauth_clients'
@@ -623,7 +686,10 @@ def _expire_relationship(target, relationship_prop):
         return
     # many-to-many and one-to-many have a list of peers
     # many-to-one has only one
-    if relationship_prop.direction is interfaces.MANYTOONE:
+    if (
+        relationship_prop.direction is interfaces.MANYTOONE
+        or not relationship_prop.uselist
+    ):
         peers = [peers]
     for obj in peers:
         if inspect(obj).persistent:

jupyterhub/pagination.py

@@ -0,0 +1,213 @@
+"""Basic class to manage pagination utils."""
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+from traitlets import Bool
+from traitlets import default
+from traitlets import Integer
+from traitlets import observe
+from traitlets import Unicode
+from traitlets import validate
+from traitlets.config import Configurable
+
+
+class Pagination(Configurable):
+
+    # configurable options
+    default_per_page = Integer(
+        100,
+        config=True,
+        help="Default number of entries per page for paginated results.",
+    )
+
+    max_per_page = Integer(
+        250,
+        config=True,
+        help="Maximum number of entries per page for paginated results.",
+    )
+
+    # state variables
+    url = Unicode("")
+    page = Integer(1)
+    per_page = Integer(1, min=1)
+
+    @default("per_page")
+    def _default_per_page(self):
+        return self.default_per_page
+
+    @validate("per_page")
+    def _limit_per_page(self, proposal):
+        if self.max_per_page and proposal.value > self.max_per_page:
+            return self.max_per_page
+        if proposal.value <= 1:
+            return 1
+        return proposal.value
+
+    @observe("max_per_page")
+    def _apply_max(self, change):
+        if change.new:
+            self.per_page = min(change.new, self.per_page)
+
+    total = Integer(0)
+
+    total_pages = Integer(0)
+
+    @default("total_pages")
+    def _calculate_total_pages(self):
+        total_pages = self.total // self.per_page
+        if self.total % self.per_page:
+            # there's a remainder, add 1
+            total_pages += 1
+        return total_pages
+
+    @observe("per_page", "total")
+    def _update_total_pages(self, change):
+        """Update total_pages when per_page or total is changed"""
+        self.total_pages = self._calculate_total_pages()
+
+    separator = Unicode("...")
+
+    def get_page_args(self, handler):
+        """
+        This method gets the arguments used in the webpage to configurate the pagination
+        In case of no arguments, it uses the default values from this class
+
+        Returns:
+          - page: The page requested for paginating or the default value (1)
+          - per_page: The number of items to return in this page. No more than max_per_page
+          - offset: The offset to consider when managing pagination via the ORM
+        """
+        page = handler.get_argument("page", 1)
+        per_page = handler.get_argument("per_page", self.default_per_page)
+        try:
+            self.per_page = int(per_page)
+        except Exception:
+            self.per_page = self._default_per_page
+
+        try:
+            self.page = int(page)
+            if self.page < 1:
+                self.page = 1
+        except:
+            self.page = 1
+
+        return self.page, self.per_page, self.per_page * (self.page - 1)
+
+    @property
+    def info(self):
+        """Get the pagination information."""
+        start = 1 + (self.page - 1) * self.per_page
+        end = start + self.per_page - 1
+        if end > self.total:
+            end = self.total
+
+        if start > self.total:
+            start = self.total
+
+        return {'total': self.total, 'start': start, 'end': end}
+
+    def calculate_pages_window(self):
+        """Calculates the set of pages to render later in links() method.
+           It returns the list of pages to render via links for the pagination
+           By default, as we've observed in other applications, we're going to render
+           only a finite and predefined number of pages, avoiding visual fatigue related
+           to a long list of pages. By default, we render 7 pages plus some inactive links with the characters '...'
+           to point out that there are other pages that aren't explicitly rendered.
+           The primary way of work is to provide current webpage and 5 next pages, the last 2 ones
+           (in case the current page + 5 does not overflow the total lenght of pages) and the first one for reference.
+           """
+
+        before_page = 2
+        after_page = 2
+        window_size = before_page + after_page + 1
+
+        # Add 1 to total_pages since our starting page is 1 and not 0
+        last_page = self.total_pages
+
+        pages = []
+
+        # will default window + start, end fit without truncation?
+        if self.total_pages > window_size + 2:
+            if self.page - before_page > 1:
+                # before_page will not reach page 1
+                pages.append(1)
+                if self.page - before_page > 2:
+                    # before_page will not reach page 2, need separator
+                    pages.append(self.separator)
+
+            pages.extend(range(max(1, self.page - before_page), self.page))
+            # we now have up to but not including self.page
+
+            if self.page + after_page + 1 >= last_page:
+                # after_page gets us to the end
+                pages.extend(range(self.page, last_page + 1))
+            else:
+                # add full after_page entries
+                pages.extend(range(self.page, self.page + after_page + 1))
+                # add separator *if* this doesn't get to last page - 1
+                if self.page + after_page < last_page - 1:
+                    pages.append(self.separator)
+                pages.append(last_page)
+
+            return pages
+
+        else:
+            # everything will fit, nothing to think about
+            # always return at least one page
+            return list(range(1, last_page + 1)) or [1]
+
+    @property
+    def links(self):
+        """Get the links for the pagination.
+           Getting the input from calculate_pages_window(), generates the HTML code
+           for the pages to render, plus the arrows to go onwards and backwards (if needed).
+           """
+        if self.total_pages == 1:
+            return []
+
+        pages_to_render = self.calculate_pages_window()
+
+        links = ['<nav>']
+        links.append('<ul class="pagination">')
+
+        if self.page > 1:
+            prev_page = self.page - 1
+            links.append(
+                '<li><a href="?page={prev_page}">«</a></li>'.format(prev_page=prev_page)
+            )
+        else:
+            links.append(
+                '<li class="disabled"><span><span aria-hidden="true">«</span></span></li>'
+            )
+
+        for page in list(pages_to_render):
+            if page == self.page:
+                links.append(
+                    '<li class="active"><span>{page}<span class="sr-only">(current)</span></span></li>'.format(
+                        page=page
+                    )
+                )
+            elif page == self.separator:
+                links.append(
+                    '<li class="disabled"><span> <span aria-hidden="true">{separator}</span></span></li>'.format(
+                        separator=self.separator
+                    )
+                )
+            else:
+                links.append(
+                    '<li><a href="?page={page}">{page}</a></li>'.format(page=page)
+                )
+
+        if self.page >= 1 and self.page < self.total_pages:
+            next_page = self.page + 1
+            links.append(
+                '<li><a href="?page={next_page}">»</a></li>'.format(next_page=next_page)
+            )
+        else:
+            links.append(
+                '<li class="disabled"><span><span aria-hidden="true">»</span></span></li>'
+            )
+
+        links.append('</ul>')
+        links.append('</nav>')
+
+        return ''.join(links)

jupyterhub/proxy.py

@@ -24,7 +24,6 @@ import time
 from functools import wraps
 from subprocess import Popen
 from urllib.parse import quote
-from urllib.parse import urlparse
 
 from tornado import gen
 from tornado.httpclient import AsyncHTTPClient
@@ -44,6 +43,7 @@ from . import utils
 from .metrics import CHECK_ROUTES_DURATION_SECONDS
 from .metrics import PROXY_POLL_DURATION_SECONDS
 from .objects import Server
+from .utils import exponential_backoff
 from .utils import make_ssl_context
 from .utils import url_path_join
 from jupyterhub.traitlets import Command
@@ -497,6 +497,19 @@ class ConfigurableHTTPProxy(Proxy):
 
             if not psutil.pid_exists(pid):
                 raise ProcessLookupError
+
+            try:
+                process = psutil.Process(pid)
+                if self.command and self.command[0]:
+                    process_cmd = process.cmdline()
+                    if process_cmd and not any(
+                        self.command[0] in clause for clause in process_cmd
+                    ):
+                        raise ProcessLookupError
+            except (psutil.AccessDenied, psutil.NoSuchProcess):
+                # If there is a process at the proxy's PID but we don't have permissions to see it,
+                # then it is unlikely to actually be the proxy.
+                raise ProcessLookupError
         else:
             os.kill(pid, 0)
 
@@ -692,8 +705,17 @@ class ConfigurableHTTPProxy(Proxy):
         parent = psutil.Process(pid)
         children = parent.children(recursive=True)
         for child in children:
-            child.kill()
-        psutil.wait_procs(children, timeout=5)
+            child.terminate()
+        gone, alive = psutil.wait_procs(children, timeout=5)
+        for p in alive:
+            p.kill()
+        # Clear the shell, too, if it still exists.
+        try:
+            parent.terminate()
+            parent.wait(timeout=5)
+            parent.kill()
+        except psutil.NoSuchProcess:
+            pass
 
     def _terminate(self):
         """Terminate our process"""
@@ -769,9 +791,35 @@ class ConfigurableHTTPProxy(Proxy):
             method=method,
             headers={'Authorization': 'token {}'.format(self.auth_token)},
             body=body,
+            connect_timeout=3,  # default: 20s
+            request_timeout=10,  # default: 20s
         )
+
+        async def _wait_for_api_request():
+            try:
                 async with self.semaphore:
-            result = await client.fetch(req)
+                    return await client.fetch(req)
+            except HTTPError as e:
+                # Retry on potentially transient errors in CHP, typically
+                # numbered 500 and up. Note that CHP isn't able to emit 429
+                # errors.
+                if e.code >= 500:
+                    self.log.warning(
+                        "api_request to the proxy failed with status code {}, retrying...".format(
+                            e.code
+                        )
+                    )
+                    return False  # a falsy return value make exponential_backoff retry
+                else:
+                    self.log.error("api_request to proxy failed: {0}".format(e))
+                    # An unhandled error here will help the hub invoke cleanup logic
+                    raise
+
+        result = await exponential_backoff(
+            _wait_for_api_request,
+            'Repeated api_request to proxy path "{}" failed.'.format(path),
+            timeout=30,
+        )
         return result
 
     async def add_route(self, routespec, target, data):

jupyterhub/services/auth.py

@@ -371,9 +371,13 @@ class HubAuth(SingletonConfigurable):
             )
             app_log.warning(r.text)
             msg = "Failed to check authorization"
-            # pass on error_description from oauth failure
+            # pass on error from oauth failure
             try:
-                description = r.json().get("error_description")
+                response = r.json()
+                # prefer more specific 'error_description', fallback to 'error'
+                description = response.get(
+                    "error_description", response.get("error", "Unknown error")
+                )
             except Exception:
                 pass
             else:
@@ -860,15 +864,15 @@ class HubAuthenticated(object):
         if kind == 'service':
             # it's a service, check hub_services
             if self.hub_services and name in self.hub_services:
-                app_log.debug("Allowing whitelisted Hub service %s", name)
+                app_log.debug("Allowing Hub service %s", name)
                 return model
             else:
                 app_log.warning("Not allowing Hub service %s", name)
                 raise UserNotAllowed(model)
 
         if self.hub_users and name in self.hub_users:
-            # user in whitelist
-            app_log.debug("Allowing whitelisted Hub user %s", name)
+            # user in allowed list
+            app_log.debug("Allowing Hub user %s", name)
             return model
         elif self.hub_groups and set(model['groups']).intersection(self.hub_groups):
             allowed_groups = set(model['groups']).intersection(self.hub_groups)
@@ -877,7 +881,7 @@ class HubAuthenticated(object):
                 name,
                 ','.join(sorted(allowed_groups)),
             )
-            # group in whitelist
+            # group in allowed list
             return model
         else:
             app_log.warning("Not allowing Hub user %s", name)

jupyterhub/services/service.py

@@ -201,6 +201,10 @@ class Service(LoggingConfigurable):
         """
     ).tag(input=True)
 
+    display = Bool(
+        True, help="""Whether to list the service on the JupyterHub UI"""
+    ).tag(input=True)
+
     oauth_no_confirm = Bool(
         False,
         help="""Skip OAuth confirmation when users access this service.
@@ -342,7 +346,7 @@ class Service(LoggingConfigurable):
             env['JUPYTERHUB_SERVICE_PREFIX'] = self.server.base_url
 
         hub = self.hub
-        if self.hub.ip in ('0.0.0.0', ''):
+        if self.hub.ip in ('', '0.0.0.0', '::'):
             # if the Hub is listening on all interfaces,
             # tell services to connect via localhost
             # since they are always local subprocesses

jupyterhub/singleuser/__init__.py

@@ -0,0 +1,13 @@
+"""JupyterHub single-user server entrypoints
+
+Contains default notebook-app subclass and mixins
+"""
+from .app import main
+from .app import SingleUserNotebookApp
+from .mixins import HubAuthenticatedHandler
+from .mixins import make_singleuser_app
+
+# backward-compatibility
+JupyterHubLoginHandler = SingleUserNotebookApp.login_handler_class
+JupyterHubLogoutHandler = SingleUserNotebookApp.logout_handler_class
+OAuthCallbackHandler = SingleUserNotebookApp.oauth_callback_handler_class

jupyterhub/singleuser/__main__.py

@@ -0,0 +1,4 @@
+from .app import main
+
+if __name__ == '__main__':
+    main()

jupyterhub/singleuser/app.py

@@ -0,0 +1,20 @@
+"""Make a single-user app based on the environment:
+
+- $JUPYTERHUB_SINGLEUSER_APP, the base Application class, to be wrapped in JupyterHub authentication.
+  default: notebook.notebookapp.NotebookApp
+"""
+import os
+
+from traitlets import import_item
+
+from .mixins import make_singleuser_app
+
+JUPYTERHUB_SINGLEUSER_APP = (
+    os.environ.get("JUPYTERHUB_SINGLEUSER_APP") or "notebook.notebookapp.NotebookApp"
+)
+
+App = import_item(JUPYTERHUB_SINGLEUSER_APP)
+
+SingleUserNotebookApp = make_singleuser_app(App)
+
+main = SingleUserNotebookApp.launch_instance

jupyterhub/singleuser/mixins.py

@@ -1,8 +1,15 @@
 #!/usr/bin/env python
-"""Extend regular notebook server to be aware of multiuser things."""
+"""Mixins to regular notebook server to add JupyterHub auth.
+
+Meant to be compatible with jupyter_server and classic notebook
+
+Use make_singleuser_app to create a compatible Application class
+with JupyterHub authentication mixins enabled.
+"""
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 import asyncio
+import importlib
 import json
 import os
 import random
@@ -19,38 +26,29 @@ from tornado.httpclient import AsyncHTTPClient
 from tornado.httpclient import HTTPRequest
 from tornado.web import HTTPError
 from tornado.web import RequestHandler
+from traitlets import Any
+from traitlets import Bool
+from traitlets import Bytes
+from traitlets import CUnicode
+from traitlets import default
+from traitlets import import_item
+from traitlets import Integer
+from traitlets import observe
+from traitlets import TraitError
+from traitlets import Unicode
+from traitlets import validate
+from traitlets.config import Configurable
 
-try:
-    import notebook
-except ImportError:
-    raise ImportError("JupyterHub single-user server requires notebook >= 4.0")
-
-from traitlets import (
-    Any,
-    Bool,
-    Bytes,
-    Integer,
-    Unicode,
-    CUnicode,
-    default,
-    observe,
-    validate,
-    TraitError,
-)
-
-from notebook.notebookapp import (
-    NotebookApp,
-    aliases as notebook_aliases,
-    flags as notebook_flags,
-)
-from notebook.auth.login import LoginHandler
-from notebook.auth.logout import LogoutHandler
-from notebook.base.handlers import IPythonHandler
-
-from ._version import __version__, _check_version
-from .log import log_request
-from .services.auth import HubOAuth, HubOAuthenticated, HubOAuthCallbackHandler
-from .utils import isoformat, url_path_join, make_ssl_context, exponential_backoff
+from .._version import __version__
+from .._version import _check_version
+from ..log import log_request
+from ..services.auth import HubOAuth
+from ..services.auth import HubOAuthCallbackHandler
+from ..services.auth import HubOAuthenticated
+from ..utils import exponential_backoff
+from ..utils import isoformat
+from ..utils import make_ssl_context
+from ..utils import url_path_join
 
 
 # Authenticate requests with the Hub
@@ -80,7 +78,7 @@ class HubAuthenticatedHandler(HubOAuthenticated):
         return set()
 
 
-class JupyterHubLoginHandler(LoginHandler):
+class JupyterHubLoginHandlerMixin:
     """LoginHandler that hooks up Hub authentication"""
 
     @staticmethod
@@ -113,7 +111,7 @@ class JupyterHubLoginHandler(LoginHandler):
         return
 
 
-class JupyterHubLogoutHandler(LogoutHandler):
+class JupyterHubLogoutHandlerMixin:
     def get(self):
         self.settings['hub_auth'].clear_cookie(self)
         self.redirect(
@@ -122,7 +120,7 @@ class JupyterHubLogoutHandler(LogoutHandler):
         )
 
 
-class OAuthCallbackHandler(HubOAuthCallbackHandler, IPythonHandler):
+class OAuthCallbackHandlerMixin(HubOAuthCallbackHandler):
     """Mixin IPythonHandler to get the right error pages, etc."""
 
     @property
@@ -131,9 +129,7 @@ class OAuthCallbackHandler(HubOAuthCallbackHandler, IPythonHandler):
 
 
 # register new hub related command-line aliases
-aliases = dict(notebook_aliases)
-aliases.update(
-    {
+aliases = {
     'user': 'SingleUserNotebookApp.user',
     'group': 'SingleUserNotebookApp.group',
     'cookie-name': 'HubAuth.cookie_name',
@@ -142,16 +138,13 @@ aliases.update(
     'hub-api-url': 'SingleUserNotebookApp.hub_api_url',
     'base-url': 'SingleUserNotebookApp.base_url',
 }
-)
-flags = dict(notebook_flags)
-flags.update(
-    {
+flags = {
     'disable-user-config': (
         {'SingleUserNotebookApp': {'disable_user_config': True}},
         "Disable user-controlled configuration of the notebook server.",
     )
 }
-)
+
 
 page_template = """
 {% extends "templates/page.html" %}
@@ -216,21 +209,29 @@ def _exclude_home(path_list):
             yield p
 
 
-class SingleUserNotebookApp(NotebookApp):
+class SingleUserNotebookAppMixin(Configurable):
     """A Subclass of the regular NotebookApp that is aware of the parent multiuser context."""
 
     description = dedent(
         """
     Single-user server for JupyterHub. Extends the Jupyter Notebook server.
 
-    Meant to be invoked by JupyterHub Spawners, and not directly.
+    Meant to be invoked by JupyterHub Spawners, not directly.
     """
     )
 
     examples = ""
     subcommands = {}
     version = __version__
-    classes = NotebookApp.classes + [HubOAuth]
+
+    # must be set in mixin subclass
+    # make_singleuser_app sets these
+    # aliases = aliases
+    # flags = flags
+    # login_handler_class = JupyterHubLoginHandler
+    # logout_handler_class = JupyterHubLogoutHandler
+    # oauth_callback_handler_class = OAuthCallbackHandler
+    # classes = NotebookApp.classes + [HubOAuth]
 
     # disable single-user app's localhost checking
     allow_remote_access = True
@@ -323,16 +324,12 @@ class SingleUserNotebookApp(NotebookApp):
                 return url.hostname
         return '127.0.0.1'
 
-    aliases = aliases
-    flags = flags
-
-    # disble some single-user configurables
+    # disable some single-user configurables
     token = ''
     open_browser = False
     quit_button = False
     trust_xheaders = True
-    login_handler_class = JupyterHubLoginHandler
-    logout_handler_class = JupyterHubLogoutHandler
+
     port_retries = (
         0  # disable port-retries, since the Spawner will tell us what port to use
     )
@@ -381,11 +378,11 @@ class SingleUserNotebookApp(NotebookApp):
             # disable config-migration when user config is disabled
             return
         else:
-            super(SingleUserNotebookApp, self).migrate_config()
+            super().migrate_config()
 
     @property
     def config_file_paths(self):
-        path = super(SingleUserNotebookApp, self).config_file_paths
+        path = super().config_file_paths
 
         if self.disable_user_config:
             # filter out user-writable config dirs if user config is disabled
@@ -394,7 +391,7 @@ class SingleUserNotebookApp(NotebookApp):
 
     @property
     def nbextensions_path(self):
-        path = super(SingleUserNotebookApp, self).nbextensions_path
+        path = super().nbextensions_path
 
         if self.disable_user_config:
             path = list(_exclude_home(path))
@@ -490,7 +487,7 @@ class SingleUserNotebookApp(NotebookApp):
             # protect against mixed timezone comparisons
             if not last_activity.tzinfo:
                 # assume naive timestamps are utc
-                self.log.warning("last activity is using naïve timestamps")
+                self.log.warning("last activity is using naive timestamps")
                 last_activity = last_activity.replace(tzinfo=timezone.utc)
 
         if self._last_activity_sent and last_activity < self._last_activity_sent:
@@ -562,7 +559,7 @@ class SingleUserNotebookApp(NotebookApp):
         # start by hitting Hub to check version
         ioloop.IOLoop.current().run_sync(self.check_hub_version)
         ioloop.IOLoop.current().add_callback(self.keep_activity_updated)
-        super(SingleUserNotebookApp, self).start()
+        super().start()
 
     def init_hub_auth(self):
         api_token = None
@@ -610,12 +607,17 @@ class SingleUserNotebookApp(NotebookApp):
             'Content-Security-Policy',
             ';'.join(["frame-ancestors 'self'", "report-uri " + csp_report_uri]),
         )
-        super(SingleUserNotebookApp, self).init_webapp()
+        super().init_webapp()
 
         # add OAuth callback
         self.web_app.add_handlers(
             r".*$",
-            [(urlparse(self.hub_auth.oauth_redirect_uri).path, OAuthCallbackHandler)],
+            [
+                (
+                    urlparse(self.hub_auth.oauth_redirect_uri).path,
+                    self.oauth_callback_handler_class,
+                )
+            ],
         )
 
         # apply X-JupyterHub-Version to *all* request handlers (even redirects)
@@ -656,9 +658,82 @@ class SingleUserNotebookApp(NotebookApp):
         env.loader = ChoiceLoader([FunctionLoader(get_page), orig_loader])
 
 
-def main(argv=None):
-    return SingleUserNotebookApp.launch_instance(argv)
+def detect_base_package(App):
+    """Detect the base package for an App class
+
+    Will return 'notebook' or 'jupyter_server'
+    based on which package App subclasses from.
+
+    Will return None if neither is identified (e.g. fork package, or duck-typing).
+    """
+    # guess notebook or jupyter_server based on App class inheritance
+    for cls in App.mro():
+        pkg = cls.__module__.split(".", 1)[0]
+        if pkg in {"notebook", "jupyter_server"}:
+            return pkg
+    return None
 
 
-if __name__ == "__main__":
-    main()
+def make_singleuser_app(App):
+    """Make and return a singleuser notebook app
+
+    given existing notebook or jupyter_server Application classes,
+    mix-in jupyterhub auth.
+
+    Instances of App must have the following attributes defining classes:
+
+    - .login_handler_class
+    - .logout_handler_class
+    - .base_handler_class (only required if not a subclass of the default app
+      in jupyter_server or notebook)
+
+    App should be a subclass of `notebook.notebookapp.NotebookApp`
+    or `jupyter_server.serverapp.ServerApp`.
+    """
+
+    empty_parent_app = App()
+
+    # detect base classes
+    LoginHandler = empty_parent_app.login_handler_class
+    LogoutHandler = empty_parent_app.logout_handler_class
+    BaseHandler = getattr(empty_parent_app, "base_handler_class", None)
+    if BaseHandler is None:
+        pkg = detect_base_package(App)
+        if pkg == "jupyter_server":
+            BaseHandler = import_item("jupyter_server.base.handlers.JupyterHandler")
+        elif pkg == "notebook":
+            BaseHandler = import_item("notebook.base.handlers.IPythonHandler")
+        else:
+            raise ValueError(
+                "{}.base_handler_class must be defined".format(App.__name__)
+            )
+
+    # create Handler classes from mixins + bases
+    class JupyterHubLoginHandler(JupyterHubLoginHandlerMixin, LoginHandler):
+        pass
+
+    class JupyterHubLogoutHandler(JupyterHubLogoutHandlerMixin, LogoutHandler):
+        pass
+
+    class OAuthCallbackHandler(OAuthCallbackHandlerMixin, BaseHandler):
+        pass
+
+    # create merged aliases & flags
+    merged_aliases = {}
+    merged_aliases.update(empty_parent_app.aliases or {})
+    merged_aliases.update(aliases)
+
+    merged_flags = {}
+    merged_flags.update(empty_parent_app.flags or {})
+    merged_flags.update(flags)
+    # create mixed-in App class, bringing it all together
+    class SingleUserNotebookApp(SingleUserNotebookAppMixin, App):
+        aliases = merged_aliases
+        flags = merged_flags
+        classes = empty_parent_app.classes + [HubOAuth]
+
+        login_handler_class = JupyterHubLoginHandler
+        logout_handler_class = JupyterHubLogoutHandler
+        oauth_callback_handler_class = OAuthCallbackHandler
+
+    return SingleUserNotebookApp

jupyterhub/spawner.py

@@ -4,8 +4,6 @@ Contains base Spawner class & default implementation
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 import ast
-import asyncio
-import errno
 import json
 import os
 import pipes
@@ -384,6 +382,37 @@ class Spawner(LoggingConfigurable):
         """
         return form_data
 
+    def options_from_query(self, query_data):
+        """Interpret query arguments passed to /spawn
+
+        Query arguments will always arrive as a dict of unicode strings.
+        Override this function to understand single-values, numbers, etc.
+
+        By default, options_from_form is called from this function. You can however override
+        this function if you need to process the query arguments differently.
+
+        This should coerce form data into the structure expected by self.user_options,
+        which must be a dict, and should be JSON-serializeable,
+        though it can contain bytes in addition to standard JSON data types.
+
+        This method should not have any side effects.
+        Any handling of `user_options` should be done in `.start()`
+        to ensure consistent behavior across servers
+        spawned via the API and form submission page.
+
+        Instances will receive this data on self.user_options, after passing through this function,
+        prior to `Spawner.start`.
+
+        .. versionadded:: 1.2
+            user_options are persisted in the JupyterHub database to be reused
+            on subsequent spawns if no options are given.
+            user_options is serialized to JSON as part of this persistence
+            (with additional support for bytes in case of uploaded file data),
+            and any non-bytes non-jsonable values will be replaced with None
+            if the user_options are re-used.
+        """
+        return self.options_from_form(query_data)
+
     user_options = Dict(
         help="""
         Dict of user specified options for the user's spawned instance of a single-user server.
@@ -402,11 +431,12 @@ class Spawner(LoggingConfigurable):
             'VIRTUAL_ENV',
             'LANG',
             'LC_ALL',
+            'JUPYTERHUB_SINGLEUSER_APP',
         ],
         help="""
-        Whitelist of environment variables for the single-user server to inherit from the JupyterHub process.
+        List of environment variables for the single-user server to inherit from the JupyterHub process.
 
-        This whitelist is used to ensure that sensitive information in the JupyterHub process's environment
+        This list is used to ensure that sensitive information in the JupyterHub process's environment
         (such as `CONFIGPROXY_AUTH_TOKEN`) is not passed to the single-user server's process.
         """,
     ).tag(config=True)
@@ -425,7 +455,7 @@ class Spawner(LoggingConfigurable):
 
         Environment variables that end up in the single-user server's process come from 3 sources:
           - This `environment` configurable
-          - The JupyterHub process' environment variables that are whitelisted in `env_keep`
+          - The JupyterHub process' environment variables that are listed in `env_keep`
           - Variables to establish contact between the single-user notebook and the hub (such as JUPYTERHUB_API_TOKEN)
 
         The `environment` configurable should be set by JupyterHub administrators to add
@@ -436,6 +466,11 @@ class Spawner(LoggingConfigurable):
 
         Note that the spawner class' interface is not guaranteed to be exactly same across upgrades,
         so if you are using the callable take care to verify it continues to work after upgrades!
+
+        .. versionchanged:: 1.2
+            environment from this configuration has highest priority,
+            allowing override of 'default' env variables,
+            such as JUPYTERHUB_API_URL.
         """
     ).tag(config=True)
 
@@ -709,16 +744,6 @@ class Spawner(LoggingConfigurable):
             if key in os.environ:
                 env[key] = os.environ[key]
 
-        # config overrides. If the value is a callable, it will be called with
-        # one parameter - the current spawner instance - and the return value
-        # will be assigned to the environment variable. This will be called at
-        # spawn time.
-        for key, value in self.environment.items():
-            if callable(value):
-                env[key] = value(self)
-            else:
-                env[key] = value
-
         env['JUPYTERHUB_API_TOKEN'] = self.api_token
         # deprecated (as of 0.7.2), for old versions of singleuser
         env['JPY_API_TOKEN'] = self.api_token
@@ -766,6 +791,18 @@ class Spawner(LoggingConfigurable):
             env['JUPYTERHUB_SSL_CERTFILE'] = self.cert_paths['certfile']
             env['JUPYTERHUB_SSL_CLIENT_CA'] = self.cert_paths['cafile']
 
+        # env overrides from config. If the value is a callable, it will be called with
+        # one parameter - the current spawner instance - and the return value
+        # will be assigned to the environment variable. This will be called at
+        # spawn time.
+        # Called last to ensure highest priority, in case of overriding other
+        # 'default' variables like the API url
+        for key, value in self.environment.items():
+            if callable(value):
+                env[key] = value(self)
+            else:
+                env[key] = value
+
         return env
 
     async def get_url(self):
@@ -910,10 +947,9 @@ class Spawner(LoggingConfigurable):
                 but not necessarily by the notebook server.
 
         Returns:
-            dict: a list (potentially altered) of paths for key, cert,
-            and CA.
-            These paths should be resolvable and readable
-            by the notebook server to be launched.
+            dict: a list (potentially altered) of paths for key, cert, and CA.
+                These paths should be resolvable and readable by the notebook
+                server to be launched.
 
 
         `.move_certs` is called after certs for the singleuser notebook have
@@ -952,7 +988,9 @@ class Spawner(LoggingConfigurable):
             args.append('--notebook-dir=%s' % _quote_safe(notebook_dir))
         if self.default_url:
             default_url = self.format_string(self.default_url)
-            args.append('--NotebookApp.default_url=%s' % _quote_safe(default_url))
+            args.append(
+                '--SingleUserNotebookApp.default_url=%s' % _quote_safe(default_url)
+            )
 
         if self.debug:
             args.append('--debug')
@@ -1580,5 +1618,5 @@ class SimpleLocalProcessSpawner(LocalProcessSpawner):
         return env
 
     def move_certs(self, paths):
-        """No-op for installing certs"""
+        """No-op for installing certs."""
         return paths

jupyterhub/tests/mocking.py

@@ -173,6 +173,9 @@ class FormSpawner(MockSpawner):
             options['energy'] = form_data['energy'][0]
         if 'hello_file' in form_data:
             options['hello'] = form_data['hello_file'][0]
+
+        if 'illegal_argument' in form_data:
+            raise ValueError("You are not allowed to specify 'illegal_argument'")
         return options
 
 
@@ -391,6 +394,17 @@ class MockSingleUserServer(SingleUserNotebookApp):
 class StubSingleUserSpawner(MockSpawner):
     """Spawner that starts a MockSingleUserServer in a thread."""
 
+    @default("default_url")
+    def _default_url(self):
+        """Use a default_url that any jupyter server will provide
+
+        Should be:
+
+        - authenticated, so we are testing auth
+        - always available (i.e. in base ServerApp and NotebookApp
+        """
+        return "/api/status"
+
     _thread = None
 
     @gen.coroutine

jupyterhub/tests/mockserverapp.py

@@ -0,0 +1,17 @@
+"""Example JupyterServer app subclass"""
+from jupyter_server.base.handlers import JupyterHandler
+from jupyter_server.serverapp import ServerApp
+from tornado import web
+
+
+class TreeHandler(JupyterHandler):
+    @web.authenticated
+    def get(self):
+        self.write("OK!")
+
+
+class MockServerApp(ServerApp):
+    def initialize(self, argv=None):
+        self.default_url = "/tree"
+        super().initialize(argv)
+        self.web_app.add_handlers(".*$", [(self.base_url + "tree/?", TreeHandler)])

jupyterhub/tests/test_api.py

@@ -1,5 +1,4 @@
 """Tests for the REST API."""
-import asyncio
 import json
 import re
 import sys
@@ -1514,6 +1513,7 @@ async def test_get_services(app, mockservice_url):
             'prefix': mockservice.server.base_url,
             'url': mockservice.url,
             'info': {},
+            'display': True,
         }
     }
 
@@ -1538,6 +1538,7 @@ async def test_get_service(app, mockservice_url):
         'prefix': mockservice.server.base_url,
         'url': mockservice.url,
         'info': {},
+        'display': True,
     }
 
     r = await api_request(

jupyterhub/tests/test_app.py

@@ -7,13 +7,11 @@ import time
 from subprocess import check_output
 from subprocess import PIPE
 from subprocess import Popen
-from subprocess import run
 from tempfile import NamedTemporaryFile
 from tempfile import TemporaryDirectory
 from unittest.mock import patch
 
 import pytest
-from tornado import gen
 from traitlets.config import Config
 
 from .. import orm
@@ -93,7 +91,7 @@ def test_generate_config():
     os.remove(cfg_file)
     assert cfg_file in out
     assert 'Spawner.cmd' in cfg_text
-    assert 'Authenticator.whitelist' in cfg_text
+    assert 'Authenticator.allowed_users' in cfg_text
 
 
 async def test_init_tokens(request):

jupyterhub/tests/test_auth.py

@@ -1,11 +1,12 @@
 """Tests for PAM authentication"""
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
-import os
+import logging
 from unittest import mock
 
 import pytest
 from requests import HTTPError
+from traitlets.config import Config
 
 from .mocking import MockPAMAuthenticator
 from .mocking import MockStructGroup
@@ -137,8 +138,8 @@ async def test_pam_auth_admin_groups():
     assert authorized['admin'] is False
 
 
-async def test_pam_auth_whitelist():
-    authenticator = MockPAMAuthenticator(whitelist={'wash', 'kaylee'})
+async def test_pam_auth_allowed():
+    authenticator = MockPAMAuthenticator(allowed_users={'wash', 'kaylee'})
     authorized = await authenticator.get_authenticated_user(
         None, {'username': 'kaylee', 'password': 'kaylee'}
     )
@@ -155,11 +156,11 @@ async def test_pam_auth_whitelist():
     assert authorized is None
 
 
-async def test_pam_auth_group_whitelist():
+async def test_pam_auth_allowed_groups():
     def getgrnam(name):
         return MockStructGroup('grp', ['kaylee'])
 
-    authenticator = MockPAMAuthenticator(group_whitelist={'group'})
+    authenticator = MockPAMAuthenticator(allowed_groups={'group'})
 
     with mock.patch.object(authenticator, '_getgrnam', getgrnam):
         authorized = await authenticator.get_authenticated_user(
@@ -174,7 +175,7 @@ async def test_pam_auth_group_whitelist():
     assert authorized is None
 
 
-async def test_pam_auth_blacklist():
+async def test_pam_auth_blocked():
     # Null case compared to next case
     authenticator = MockPAMAuthenticator()
     authorized = await authenticator.get_authenticated_user(
@@ -183,33 +184,33 @@ async def test_pam_auth_blacklist():
     assert authorized['name'] == 'wash'
 
     # Blacklist basics
-    authenticator = MockPAMAuthenticator(blacklist={'wash'})
+    authenticator = MockPAMAuthenticator(blocked_users={'wash'})
     authorized = await authenticator.get_authenticated_user(
         None, {'username': 'wash', 'password': 'wash'}
     )
     assert authorized is None
 
-    # User in both white and blacklists: default deny.  Make error someday?
+    # User in both allowed and blocked: default deny.  Make error someday?
     authenticator = MockPAMAuthenticator(
-        blacklist={'wash'}, whitelist={'wash', 'kaylee'}
+        blocked_users={'wash'}, allowed_users={'wash', 'kaylee'}
     )
     authorized = await authenticator.get_authenticated_user(
         None, {'username': 'wash', 'password': 'wash'}
     )
     assert authorized is None
 
-    # User not in blacklist can log in
+    # User not in blocked set can log in
     authenticator = MockPAMAuthenticator(
-        blacklist={'wash'}, whitelist={'wash', 'kaylee'}
+        blocked_users={'wash'}, allowed_users={'wash', 'kaylee'}
     )
     authorized = await authenticator.get_authenticated_user(
         None, {'username': 'kaylee', 'password': 'kaylee'}
     )
     assert authorized['name'] == 'kaylee'
 
-    # User in whitelist, blacklist irrelevent
+    # User in allowed, blocked irrelevent
     authenticator = MockPAMAuthenticator(
-        blacklist={'mal'}, whitelist={'wash', 'kaylee'}
+        blocked_users={'mal'}, allowed_users={'wash', 'kaylee'}
     )
     authorized = await authenticator.get_authenticated_user(
         None, {'username': 'wash', 'password': 'wash'}
@@ -218,15 +219,16 @@ async def test_pam_auth_blacklist():
 
     # User in neither list
     authenticator = MockPAMAuthenticator(
-        blacklist={'mal'}, whitelist={'wash', 'kaylee'}
+        blocked_users={'mal'}, allowed_users={'wash', 'kaylee'}
     )
     authorized = await authenticator.get_authenticated_user(
         None, {'username': 'simon', 'password': 'simon'}
     )
     assert authorized is None
 
-    # blacklist == {}
-    authenticator = MockPAMAuthenticator(blacklist=set(), whitelist={'wash', 'kaylee'})
+    authenticator = MockPAMAuthenticator(
+        blocked_users=set(), allowed_users={'wash', 'kaylee'}
+    )
     authorized = await authenticator.get_authenticated_user(
         None, {'username': 'kaylee', 'password': 'kaylee'}
     )
@@ -253,7 +255,7 @@ async def test_deprecated_signatures():
 
 
 async def test_pam_auth_no_such_group():
-    authenticator = MockPAMAuthenticator(group_whitelist={'nosuchcrazygroup'})
+    authenticator = MockPAMAuthenticator(allowed_groups={'nosuchcrazygroup'})
     authorized = await authenticator.get_authenticated_user(
         None, {'username': 'kaylee', 'password': 'kaylee'}
     )
@@ -262,7 +264,7 @@ async def test_pam_auth_no_such_group():
 
 async def test_wont_add_system_user():
     user = orm.User(name='lioness4321')
-    authenticator = auth.PAMAuthenticator(whitelist={'mal'})
+    authenticator = auth.PAMAuthenticator(allowed_users={'mal'})
     authenticator.create_system_users = False
     with pytest.raises(KeyError):
         await authenticator.add_user(user)
@@ -270,7 +272,7 @@ async def test_wont_add_system_user():
 
 async def test_cant_add_system_user():
     user = orm.User(name='lioness4321')
-    authenticator = auth.PAMAuthenticator(whitelist={'mal'})
+    authenticator = auth.PAMAuthenticator(allowed_users={'mal'})
     authenticator.add_user_cmd = ['jupyterhub-fake-command']
     authenticator.create_system_users = True
 
@@ -296,7 +298,7 @@ async def test_cant_add_system_user():
 
 async def test_add_system_user():
     user = orm.User(name='lioness4321')
-    authenticator = auth.PAMAuthenticator(whitelist={'mal'})
+    authenticator = auth.PAMAuthenticator(allowed_users={'mal'})
     authenticator.create_system_users = True
     authenticator.add_user_cmd = ['echo', '/home/USERNAME']
 
@@ -317,13 +319,13 @@ async def test_add_system_user():
 
 async def test_delete_user():
     user = orm.User(name='zoe')
-    a = MockPAMAuthenticator(whitelist={'mal'})
+    a = MockPAMAuthenticator(allowed_users={'mal'})
 
-    assert 'zoe' not in a.whitelist
+    assert 'zoe' not in a.allowed_users
     await a.add_user(user)
-    assert 'zoe' in a.whitelist
+    assert 'zoe' in a.allowed_users
     a.delete_user(user)
-    assert 'zoe' not in a.whitelist
+    assert 'zoe' not in a.allowed_users
 
 
 def test_urls():
@@ -461,3 +463,55 @@ async def test_post_auth_hook():
     )
 
     assert authorized['testkey'] == 'testvalue'
+
+
+class MyAuthenticator(auth.Authenticator):
+    def check_whitelist(self, username, authentication=None):
+        return username == "subclass-allowed"
+
+
+def test_deprecated_config(caplog):
+    cfg = Config()
+    cfg.Authenticator.whitelist = {'user'}
+    log = logging.getLogger("testlog")
+    authenticator = auth.Authenticator(config=cfg, log=log)
+    assert caplog.record_tuples == [
+        (
+            log.name,
+            logging.WARNING,
+            'Authenticator.whitelist is deprecated in JupyterHub 1.2, use '
+            'Authenticator.allowed_users instead',
+        )
+    ]
+    assert authenticator.allowed_users == {'user'}
+
+
+def test_deprecated_methods():
+    cfg = Config()
+    cfg.Authenticator.whitelist = {'user'}
+    authenticator = auth.Authenticator(config=cfg)
+
+    assert authenticator.check_allowed("user")
+    with pytest.deprecated_call():
+        assert authenticator.check_whitelist("user")
+    assert not authenticator.check_allowed("otheruser")
+    with pytest.deprecated_call():
+        assert not authenticator.check_whitelist("otheruser")
+
+
+def test_deprecated_config_subclass():
+    cfg = Config()
+    cfg.MyAuthenticator.whitelist = {'user'}
+    with pytest.deprecated_call():
+        authenticator = MyAuthenticator(config=cfg)
+    assert authenticator.allowed_users == {'user'}
+
+
+def test_deprecated_methods_subclass():
+    with pytest.deprecated_call():
+        authenticator = MyAuthenticator()
+
+    assert authenticator.check_allowed("subclass-allowed")
+    assert authenticator.check_whitelist("subclass-allowed")
+    assert not authenticator.check_allowed("otheruser")
+    assert not authenticator.check_whitelist("otheruser")

jupyterhub/tests/test_auth_expiry.py

@@ -7,7 +7,6 @@ authentication can expire in a number of ways:
 - doesn't need refresh
 - needs refresh and cannot be refreshed without new login
 """
-import asyncio
 from contextlib import contextmanager
 from unittest import mock
 from urllib.parse import parse_qs

jupyterhub/tests/test_dummyauth.py

@@ -1,8 +1,6 @@
 """Tests for dummy authentication"""
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
-import pytest
-
 from jupyterhub.auth import DummyAuthenticator
 
 

jupyterhub/tests/test_internal_ssl_app.py

@@ -1,8 +1,6 @@
 """Test the JupyterHub entry point with internal ssl"""
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
-import sys
-
 import jupyterhub.tests.mocking
 from jupyterhub.tests.test_app import *
 

jupyterhub/tests/test_named_servers.py

@@ -368,3 +368,28 @@ async def test_user_redirect_hook_default_server_name(
     assert redirected_url.path == url_path_join(
         app.base_url, 'user', username, 'terminals/1'
     )
+
+
+async def test_named_server_stop_server(app, username, named_servers):
+    server_name = "myserver"
+    await app.login_user(username)
+    user = app.users[username]
+
+    r = await api_request(app, 'users', username, 'server', method='post')
+    assert r.status_code == 201
+    assert r.text == ''
+    assert user.spawners[''].server
+
+    with mock.patch.object(
+        app.proxy, 'add_user', side_effect=Exception('mock exception')
+    ):
+        r = await api_request(
+            app, 'users', username, 'servers', server_name, method='post'
+        )
+        r.raise_for_status()
+        assert r.status_code == 201
+        assert r.text == ''
+
+    assert user.spawners[server_name].server is None
+    assert user.spawners[''].server
+    assert user.running

jupyterhub/tests/test_orm.py

@@ -134,7 +134,7 @@ def test_token_expiry(db):
     assert orm_token.expires_at > now + timedelta(seconds=50)
     assert orm_token.expires_at < now + timedelta(seconds=70)
     the_future = mock.patch(
-        'jupyterhub.orm.utcnow', lambda: now + timedelta(seconds=70)
+        'jupyterhub.orm.APIToken.now', lambda: now + timedelta(seconds=70)
     )
     with the_future:
         found = orm.APIToken.find(db, token=token)
@@ -482,3 +482,78 @@ def test_group_delete_cascade(db):
     db.delete(user1)
     db.commit()
     assert user1 not in group1.users
+
+
+def test_expiring_api_token(app, user):
+    db = app.db
+    token = orm.APIToken.new(expires_in=30, user=user)
+    orm_token = orm.APIToken.find(db, token, kind='user')
+    assert orm_token
+
+    # purge_expired doesn't delete non-expired
+    orm.APIToken.purge_expired(db)
+    found = orm.APIToken.find(db, token)
+    assert found is orm_token
+
+    with mock.patch.object(
+        orm.APIToken, 'now', lambda: datetime.utcnow() + timedelta(seconds=60)
+    ):
+        found = orm.APIToken.find(db, token)
+        assert found is None
+        assert orm_token in db.query(orm.APIToken)
+        orm.APIToken.purge_expired(db)
+        assert orm_token not in db.query(orm.APIToken)
+
+
+def test_expiring_oauth_token(app, user):
+    db = app.db
+    token = "abc123"
+    now = orm.OAuthAccessToken.now
+    client = orm.OAuthClient(identifier="xxx", secret="yyy")
+    db.add(client)
+    orm_token = orm.OAuthAccessToken(
+        token=token,
+        grant_type=orm.GrantType.authorization_code,
+        client=client,
+        user=user,
+        expires_at=now() + 30,
+    )
+    db.add(orm_token)
+    db.commit()
+
+    found = orm.OAuthAccessToken.find(db, token)
+    assert found is orm_token
+    # purge_expired doesn't delete non-expired
+    orm.OAuthAccessToken.purge_expired(db)
+    found = orm.OAuthAccessToken.find(db, token)
+    assert found is orm_token
+
+    with mock.patch.object(orm.OAuthAccessToken, 'now', lambda: now() + 60):
+        found = orm.OAuthAccessToken.find(db, token)
+        assert found is None
+        assert orm_token in db.query(orm.OAuthAccessToken)
+        orm.OAuthAccessToken.purge_expired(db)
+        assert orm_token not in db.query(orm.OAuthAccessToken)
+
+
+def test_expiring_oauth_code(app, user):
+    db = app.db
+    code = "abc123"
+    now = orm.OAuthCode.now
+    orm_code = orm.OAuthCode(code=code, expires_at=now() + 30)
+    db.add(orm_code)
+    db.commit()
+
+    found = orm.OAuthCode.find(db, code)
+    assert found is orm_code
+    # purge_expired doesn't delete non-expired
+    orm.OAuthCode.purge_expired(db)
+    found = orm.OAuthCode.find(db, code)
+    assert found is orm_code
+
+    with mock.patch.object(orm.OAuthCode, 'now', lambda: now() + 60):
+        found = orm.OAuthCode.find(db, code)
+        assert found is None
+        assert orm_code in db.query(orm.OAuthCode)
+        orm.OAuthCode.purge_expired(db)
+        assert orm_code not in db.query(orm.OAuthCode)

jupyterhub/tests/test_pages.py

@@ -9,6 +9,7 @@ from urllib.parse import urlparse
 import pytest
 from bs4 import BeautifulSoup
 from tornado import gen
+from tornado.escape import url_escape
 from tornado.httputil import url_concat
 
 from .. import orm
@@ -30,7 +31,7 @@ async def test_root_no_auth(app):
     url = ujoin(public_host(app), app.hub.base_url)
     r = await async_requests.get(url)
     r.raise_for_status()
-    assert r.url == ujoin(url, 'login')
+    assert r.url == url_concat(ujoin(url, 'login'), dict(next=app.hub.base_url))
 
 
 async def test_root_auth(app):
@@ -255,6 +256,47 @@ async def test_spawn_page_admin(app, admin_access):
         assert "Spawning server for {}".format(u.name) in r.text
 
 
+async def test_spawn_with_query_arguments(app):
+    with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}):
+        base_url = ujoin(public_host(app), app.hub.base_url)
+        cookies = await app.login_user('jones')
+        orm_u = orm.User.find(app.db, 'jones')
+        u = app.users[orm_u]
+        await u.stop()
+        next_url = ujoin(app.base_url, 'user/jones/tree')
+        r = await async_requests.get(
+            url_concat(
+                ujoin(base_url, 'spawn'), {'next': next_url, 'energy': '510keV'},
+            ),
+            cookies=cookies,
+        )
+        r.raise_for_status()
+        assert r.history
+        assert u.spawner.user_options == {
+            'energy': '510keV',
+            'notspecified': 5,
+        }
+
+
+async def test_spawn_with_query_arguments_error(app):
+    with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}):
+        base_url = ujoin(public_host(app), app.hub.base_url)
+        cookies = await app.login_user('jones')
+        orm_u = orm.User.find(app.db, 'jones')
+        u = app.users[orm_u]
+        await u.stop()
+        next_url = ujoin(app.base_url, 'user/jones/tree')
+        r = await async_requests.get(
+            url_concat(
+                ujoin(base_url, 'spawn'),
+                {'next': next_url, 'energy': '510keV', 'illegal_argument': '42'},
+            ),
+            cookies=cookies,
+        )
+        r.raise_for_status()
+        assert "You are not allowed to specify " in r.text
+
+
 async def test_spawn_form(app):
     with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}):
         base_url = ujoin(public_host(app), app.hub.base_url)
@@ -354,7 +396,7 @@ async def test_spawn_pending(app, username, slow_spawn):
     assert page.find('div', {'class': 'progress'})
 
     # validate event source url by consuming it
-    script = page.body.find('script').text
+    script = page.body.find('script').string
     assert 'EventSource' in script
     # find EventSource url in javascript
     # maybe not the most robust way to check this?
@@ -475,6 +517,58 @@ async def test_user_redirect_deprecated(app, username):
     )
 
 
+@pytest.mark.parametrize(
+    'url, params, redirected_url, form_action',
+    [
+        (
+            # spawn?param=value
+            # will encode given parameters for an unauthenticated URL in the next url
+            # the next parameter will contain the app base URL (replaces BASE_URL in tests)
+            'spawn',
+            [('param', 'value')],
+            '/hub/login?next={{BASE_URL}}hub%2Fspawn%3Fparam%3Dvalue',
+            '/hub/login?next={{BASE_URL}}hub%2Fspawn%3Fparam%3Dvalue',
+        ),
+        (
+            # login?param=fromlogin&next=encoded(/hub/spawn?param=value)
+            # will drop parameters given to the login page, passing only the next url
+            'login',
+            [('param', 'fromlogin'), ('next', '/hub/spawn?param=value')],
+            '/hub/login?param=fromlogin&next=%2Fhub%2Fspawn%3Fparam%3Dvalue',
+            '/hub/login?next=%2Fhub%2Fspawn%3Fparam%3Dvalue',
+        ),
+        (
+            # login?param=value&anotherparam=anothervalue
+            # will drop parameters given to the login page, and use an empty next url
+            'login',
+            [('param', 'value'), ('anotherparam', 'anothervalue')],
+            '/hub/login?param=value&anotherparam=anothervalue',
+            '/hub/login?next=',
+        ),
+        (
+            # login
+            # simplest case, accessing the login URL, gives an empty next url
+            'login',
+            [],
+            '/hub/login',
+            '/hub/login?next=',
+        ),
+    ],
+)
+async def test_login_page(app, url, params, redirected_url, form_action):
+    url = url_concat(url, params)
+    r = await get_page(url, app)
+    redirected_url = redirected_url.replace('{{BASE_URL}}', url_escape(app.base_url))
+    assert r.url.endswith(redirected_url)
+    # now the login.html rendered template must include the given parameters in the form
+    # action URL, including the next URL
+    page = BeautifulSoup(r.text, "html.parser")
+    form = page.find("form", method="post")
+    action = form.attrs['action']
+    form_action = form_action.replace('{{BASE_URL}}', url_escape(app.base_url))
+    assert action.endswith(form_action)
+
+
 async def test_login_fail(app):
     name = 'wash'
     base_url = public_url(app)
@@ -505,37 +599,54 @@ async def test_login_strip(app):
 
 
 @pytest.mark.parametrize(
-    'running, next_url, location',
+    'running, next_url, location, params',
     [
         # default URL if next not specified, for both running and not
-        (True, '', ''),
-        (False, '', ''),
+        (True, '', '', None),
+        (False, '', '', None),
         # next_url is respected
-        (False, '/hub/admin', '/hub/admin'),
-        (False, '/user/other', '/hub/user/other'),
-        (False, '/absolute', '/absolute'),
-        (False, '/has?query#andhash', '/has?query#andhash'),
+        (False, '/hub/admin', '/hub/admin', None),
+        (False, '/user/other', '/hub/user/other', None),
+        (False, '/absolute', '/absolute', None),
+        (False, '/has?query#andhash', '/has?query#andhash', None),
         # next_url outside is not allowed
-        (False, 'relative/path', ''),
-        (False, 'https://other.domain', ''),
-        (False, 'ftp://other.domain', ''),
-        (False, '//other.domain', ''),
-        (False, '///other.domain/triple', ''),
-        (False, '\\\\other.domain/backslashes', ''),
+        (False, 'relative/path', '', None),
+        (False, 'https://other.domain', '', None),
+        (False, 'ftp://other.domain', '', None),
+        (False, '//other.domain', '', None),
+        (False, '///other.domain/triple', '', None),
+        (False, '\\\\other.domain/backslashes', '', None),
+        # params are handled correctly (ignored if ?next= specified)
+        (
+            True,
+            '/hub/admin?left=1&right=2',
+            'hub/admin?left=1&right=2',
+            {"left": "abc"},
+        ),
+        (False, '/hub/admin', 'hub/admin', [('left', 1), ('right', 2)]),
+        (True, '', '', {"keep": "yes"}),
+        (False, '', '', {"keep": "yes"}),
     ],
 )
-async def test_login_redirect(app, running, next_url, location):
+async def test_login_redirect(app, running, next_url, location, params):
     cookies = await app.login_user('river')
     user = app.users['river']
     if location:
         location = ujoin(app.base_url, location)
     elif running:
+        # location not specified,
         location = user.url
+        if params:
+            location = url_concat(location, params)
     else:
         # use default url
         location = ujoin(app.base_url, 'hub/spawn')
+        if params:
+            location = url_concat(location, params)
 
     url = 'login'
+    if params:
+        url = url_concat(url, params)
     if next_url:
         if '//' not in next_url and next_url.startswith('/'):
             next_url = ujoin(app.base_url, next_url, '')
@@ -550,7 +661,73 @@ async def test_login_redirect(app, running, next_url, location):
     r = await get_page(url, app, cookies=cookies, allow_redirects=False)
     r.raise_for_status()
     assert r.status_code == 302
-    assert location == r.headers['Location']
+    assert r.headers["Location"] == location
+
+
+@pytest.mark.parametrize(
+    'location, next, extra_params',
+    [
+        (
+            "{base_url}hub/spawn?a=5",
+            None,
+            {"a": "5"},
+        ),  # no ?next= given, preserve params
+        ("/x", "/x", {"a": "5"}),  # ?next=given, params ignored
+        (
+            "/x?b=10",
+            "/x?b=10",
+            {"a": "5"},
+        ),  # ?next=given with params, additional params ignored
+    ],
+)
+async def test_next_url(app, user, location, next, extra_params):
+    params = {}
+    if extra_params:
+        params.update(extra_params)
+    if next:
+        params["next"] = next
+    url = url_concat("/", params)
+    cookies = await app.login_user("monster")
+
+    # location can be a string template
+    location = location.format(base_url=app.base_url)
+
+    r = await get_page(url, app, cookies=cookies, allow_redirects=False)
+    r.raise_for_status()
+    assert r.status_code == 302
+    assert r.headers["Location"] == location
+
+
+async def test_next_url_params_sequence(app, user):
+    """Test each step of / -> login -> spawn
+
+    and whether they preserve url params
+    """
+    params = {"xyz": "5"}
+    # first request: root page, with params, not logged in
+    r = await get_page("/?xyz=5", app, allow_redirects=False)
+    r.raise_for_status()
+    location = r.headers["Location"]
+
+    # next page: login
+    cookies = await app.login_user(user.name)
+    assert location == url_concat(
+        ujoin(app.base_url, "/hub/login"), {"next": ujoin(app.base_url, "/hub/?xyz=5")}
+    )
+    r = await async_requests.get(
+        public_host(app) + location, cookies=cookies, allow_redirects=False
+    )
+    r.raise_for_status()
+    location = r.headers["Location"]
+
+    # after login, redirect back
+    assert location == ujoin(app.base_url, "/hub/?xyz=5")
+    r = await async_requests.get(
+        public_host(app) + location, cookies=cookies, allow_redirects=False
+    )
+    r.raise_for_status()
+    location = r.headers["Location"]
+    assert location == ujoin(app.base_url, "/hub/spawn?xyz=5")
 
 
 async def test_auto_login(app, request):
@@ -564,14 +741,18 @@ async def test_auto_login(app, request):
     )
     # no auto_login: end up at /hub/login
     r = await async_requests.get(base_url)
-    assert r.url == public_url(app, path='hub/login')
+    assert r.url == url_concat(
+        public_url(app, path="hub/login"), {"next": app.hub.base_url}
+    )
     # enable auto_login: redirect from /hub/login to /hub/dummy
     authenticator = Authenticator(auto_login=True)
     authenticator.login_url = lambda base_url: ujoin(base_url, 'dummy')
 
     with mock.patch.dict(app.tornado_settings, {'authenticator': authenticator}):
         r = await async_requests.get(base_url)
-    assert r.url == public_url(app, path='hub/dummy')
+    assert r.url == url_concat(
+        public_url(app, path="hub/dummy"), {"next": app.hub.base_url}
+    )
 
 
 async def test_auto_login_logout(app):
@@ -645,7 +826,7 @@ async def test_shutdown_on_logout(app, shutdown_on_logout):
     assert spawner.ready == (not shutdown_on_logout)
 
 
-async def test_login_no_whitelist_adds_user(app):
+async def test_login_no_allowed_adds_user(app):
     auth = app.authenticator
     mock_add_user = mock.Mock()
     with mock.patch.object(auth, 'add_user', mock_add_user):

jupyterhub/tests/test_pagination.py

@@ -0,0 +1,45 @@
+"""tests for pagination"""
+from pytest import mark
+from pytest import raises
+from traitlets.config import Config
+
+from jupyterhub.pagination import Pagination
+
+
+def test_per_page_bounds():
+    cfg = Config()
+    cfg.Pagination.max_per_page = 10
+    p = Pagination(config=cfg, per_page=20, total=100)
+    assert p.per_page == 10
+    with raises(Exception):
+        p.per_page = 0
+
+
+@mark.parametrize(
+    "page, per_page, total, expected",
+    [
+        (1, 10, 99, [1, 2, 3, "...", 10]),
+        (2, 10, 99, [1, 2, 3, 4, "...", 10]),
+        (3, 10, 99, [1, 2, 3, 4, 5, "...", 10]),
+        (4, 10, 99, [1, 2, 3, 4, 5, 6, "...", 10]),
+        (5, 10, 99, [1, "...", 3, 4, 5, 6, 7, "...", 10]),
+        (6, 10, 99, [1, "...", 4, 5, 6, 7, 8, "...", 10]),
+        (7, 10, 99, [1, "...", 5, 6, 7, 8, 9, 10]),
+        (8, 10, 99, [1, "...", 6, 7, 8, 9, 10]),
+        (9, 10, 99, [1, "...", 7, 8, 9, 10]),
+        (1, 20, 99, [1, 2, 3, 4, 5]),
+        (1, 10, 0, [1]),
+        (1, 10, 1, [1]),
+        (1, 10, 10, [1]),
+        (1, 10, 11, [1, 2]),
+        (1, 10, 50, [1, 2, 3, 4, 5]),
+        (1, 10, 60, [1, 2, 3, 4, 5, 6]),
+        (1, 10, 70, [1, 2, 3, 4, 5, 6, 7]),
+        (1, 10, 80, [1, 2, 3, "...", 8]),
+    ],
+)
+def test_window(page, per_page, total, expected):
+    cfg = Config()
+    cfg.Pagination
+    pagination = Pagination(page=page, per_page=per_page, total=total)
+    assert pagination.calculate_pages_window() == expected

jupyterhub/tests/test_proxy.py

@@ -2,7 +2,6 @@
 import json
 import os
 from contextlib import contextmanager
-from queue import Queue
 from subprocess import Popen
 from urllib.parse import quote
 from urllib.parse import urlparse

jupyterhub/tests/test_services.py

@@ -2,18 +2,13 @@
 import asyncio
 import os
 import sys
-import time
 from binascii import hexlify
 from contextlib import contextmanager
 from subprocess import Popen
-from threading import Event
 
-import pytest
-import requests
 from async_generator import async_generator
 from async_generator import asynccontextmanager
 from async_generator import yield_
-from tornado import gen
 from tornado.ioloop import IOLoop
 
 from ..utils import maybe_future

jupyterhub/tests/test_services_auth.py

@@ -11,7 +11,6 @@ from threading import Thread
 from unittest import mock
 from urllib.parse import urlparse
 
-import pytest
 import requests
 import requests_mock
 from pytest import raises
@@ -185,7 +184,7 @@ def test_hub_authenticated(request):
 
         m.get(good_url, text=json.dumps(mock_model))
 
-        # no whitelist
+        # no specific allowed user
         r = requests.get(
             'http://127.0.0.1:%i' % port,
             cookies={'jubal': 'early'},
@@ -194,7 +193,7 @@ def test_hub_authenticated(request):
         r.raise_for_status()
         assert r.status_code == 200
 
-        # pass whitelist
+        # pass allowed user
         TestHandler.hub_users = {'jubalearly'}
         r = requests.get(
             'http://127.0.0.1:%i' % port,
@@ -204,7 +203,7 @@ def test_hub_authenticated(request):
         r.raise_for_status()
         assert r.status_code == 200
 
-        # no pass whitelist
+        # no pass allowed ser
         TestHandler.hub_users = {'kaylee'}
         r = requests.get(
             'http://127.0.0.1:%i' % port,
@@ -213,7 +212,7 @@ def test_hub_authenticated(request):
         )
         assert r.status_code == 403
 
-        # pass group whitelist
+        # pass allowed group
         TestHandler.hub_groups = {'lions'}
         r = requests.get(
             'http://127.0.0.1:%i' % port,
@@ -223,7 +222,7 @@ def test_hub_authenticated(request):
         r.raise_for_status()
         assert r.status_code == 200
 
-        # no pass group whitelist
+        # no pass allowed group
         TestHandler.hub_groups = {'tigers'}
         r = requests.get(
             'http://127.0.0.1:%i' % port,