Bump to 3.0.0b1

3.0 changelog

.github/dependabot.yml

@@ -0,0 +1,15 @@
+# dependabot.yml reference: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+#
+# Notes:
+# - Status and logs from dependabot are provided at
+#   https://github.com/jupyterhub/jupyterhub/network/updates.
+#
+version: 2
+updates:
+  # Maintain dependencies in our GitHub Workflows
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly
+      time: "05:00"
+      timezone: "Etc/UTC"

.github/workflows/release.yml

@@ -32,17 +32,18 @@ jobs:
   build-release:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.9"
 
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v3
         with:
           node-version: "14"
 
-      - name: install build package
+      - name: install build requirements
         run: |
+          npm install -g yarn
           pip install --upgrade pip
           pip install build
           pip freeze
@@ -52,28 +53,17 @@ jobs:
           python -m build --sdist --wheel .
           ls -l dist
 
-      - name: verify wheel
+      - name: verify sdist
         run: |
-          cd dist
-          pip install ./*.whl
-          # verify data-files are installed where they are found
-          cat <<EOF | python
-          import os
-          from jupyterhub._data import DATA_FILES_PATH
-          print(f"DATA_FILES_PATH={DATA_FILES_PATH}")
-          assert os.path.exists(DATA_FILES_PATH), DATA_FILES_PATH
-          for subpath in (
-              "templates/page.html",
-              "static/css/style.min.css",
-              "static/components/jquery/dist/jquery.js",
-          ):
-              path = os.path.join(DATA_FILES_PATH, subpath)
-              assert os.path.exists(path), path
-          print("OK")
-          EOF
+          ./ci/check_sdist.py dist/jupyterhub-*.tar.gz
+
+      - name: verify data-files are installed where they are found
+        run: |
+          pip install dist/*.whl
+          ./ci/check_installed_data.py
 
       # ref: https://github.com/actions/upload-artifact#readme
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: jupyterhub-${{ github.sha }}
           path: "dist/*"
@@ -108,16 +98,16 @@ jobs:
               echo "REGISTRY=localhost:5000/" >> $GITHUB_ENV
           fi
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       # Setup docker to build for multiple platforms, see:
       # https://github.com/docker/build-push-action/tree/v2.4.0#usage
       # https://github.com/docker/build-push-action/blob/v2.4.0/docs/advanced/multi-platform.md
       - name: Set up QEMU (for docker buildx)
-        uses: docker/setup-qemu-action@25f0500ff22e406f7191a2a8ba8cda16901ca018 # associated tag: v1.0.2
+        uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # associated tag: v1.0.2
 
       - name: Set up Docker Buildx (for multi-arch builds)
-        uses: docker/setup-buildx-action@2a4b53665e15ce7d7049afb11ff1f70ff1610609 # associated tag: v1.1.2
+        uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # associated tag: v1.1.2
         with:
           # Allows pushing to registry on localhost:5000
           driver-opts: network=host
@@ -155,7 +145,7 @@ jobs:
           branchRegex: ^\w[\w-.]*$
 
       - name: Build and push jupyterhub
-        uses: docker/build-push-action@e1b7f96249f2e4c8e4ac1519b9608c0d48944a1f
+        uses: docker/build-push-action@1cb9d22b932e4832bb29793b7777ec860fc1cde0
         with:
           context: .
           platforms: linux/amd64,linux/arm64
@@ -176,7 +166,7 @@ jobs:
           branchRegex: ^\w[\w-.]*$
 
       - name: Build and push jupyterhub-onbuild
-        uses: docker/build-push-action@e1b7f96249f2e4c8e4ac1519b9608c0d48944a1f
+        uses: docker/build-push-action@1cb9d22b932e4832bb29793b7777ec860fc1cde0
         with:
           build-args: |
             BASE_IMAGE=${{ fromJson(steps.jupyterhubtags.outputs.tags)[0] }}
@@ -197,7 +187,7 @@ jobs:
           branchRegex: ^\w[\w-.]*$
 
       - name: Build and push jupyterhub-demo
-        uses: docker/build-push-action@e1b7f96249f2e4c8e4ac1519b9608c0d48944a1f
+        uses: docker/build-push-action@1cb9d22b932e4832bb29793b7777ec860fc1cde0
         with:
           build-args: |
             BASE_IMAGE=${{ fromJson(steps.onbuildtags.outputs.tags)[0] }}
@@ -221,7 +211,7 @@ jobs:
           branchRegex: ^\w[\w-.]*$
 
       - name: Build and push jupyterhub/singleuser
-        uses: docker/build-push-action@e1b7f96249f2e4c8e4ac1519b9608c0d48944a1f
+        uses: docker/build-push-action@1cb9d22b932e4832bb29793b7777ec860fc1cde0
         with:
           build-args: |
             JUPYTERHUB_VERSION=${{ github.ref_type == 'tag' && github.ref_name || format('git:{0}', github.sha) }}

.github/workflows/test-docs.yml

@@ -15,15 +15,13 @@ on:
       - "docs/**"
       - "jupyterhub/_version.py"
       - "jupyterhub/scopes.py"
-      - ".github/workflows/*"
-      - "!.github/workflows/test-docs.yml"
+      - ".github/workflows/test-docs.yml"
   push:
     paths:
       - "docs/**"
       - "jupyterhub/_version.py"
       - "jupyterhub/scopes.py"
-      - ".github/workflows/*"
-      - "!.github/workflows/test-docs.yml"
+      - ".github/workflows/test-docs.yml"
     branches-ignore:
       - "dependabot/**"
       - "pre-commit-ci-update-config"
@@ -40,18 +38,18 @@ jobs:
   validate-rest-api-definition:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Validate REST API definition
-        uses: char0n/swagger-editor-validate@182d1a5d26ff5c2f4f452c43bd55e2c7d8064003
+        uses: char0n/swagger-editor-validate@v1.3.1
         with:
           definition-file: docs/source/_static/rest-api.yml
 
   test-docs:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
 

.github/workflows/test-jsx.yml

@@ -0,0 +1,52 @@
+# This is a GitHub workflow defining a set of jobs with a set of steps.
+# ref: https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions
+#
+name: Test jsx (admin-react.js)
+
+on:
+  pull_request:
+    paths:
+      - "jsx/**"
+      - ".github/workflows/test-jsx.yml"
+  push:
+    paths:
+      - "jsx/**"
+      - ".github/workflows/test-jsx.yml"
+    branches-ignore:
+      - "dependabot/**"
+      - "pre-commit-ci-update-config"
+    tags:
+      - "**"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  # The ./jsx folder contains React based source code files that are to compile
+  # to share/jupyterhub/static/js/admin-react.js. The ./jsx folder includes
+  # tests also has tests that this job is meant to run with `yarn test`
+  # according to the documentation in jsx/README.md.
+  test-jsx-admin-react:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 5
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "14"
+
+      - name: Install yarn
+        run: |
+          npm install -g yarn
+
+      - name: yarn
+        run: |
+          cd jsx
+          yarn
+
+      - name: yarn test
+        run: |
+          cd jsx
+          yarn test

.github/workflows/test.yml

@@ -30,34 +30,10 @@ env:
   LANG: C.UTF-8
   PYTEST_ADDOPTS: "--verbose --color=yes"
 
+permissions:
+  contents: read
+
 jobs:
-  jstest:
-    # Run javascript tests
-    runs-on: ubuntu-20.04
-    timeout-minutes: 5
-    steps:
-      - uses: actions/checkout@v2
-      # NOTE: actions/setup-node@v1 make use of a cache within the GitHub base
-      #       environment and setup in a fraction of a second.
-      - name: Install Node
-        uses: actions/setup-node@v1
-        with:
-          node-version: "14"
-
-      - name: Install Node dependencies
-        run: |
-          npm install -g yarn
-
-      - name: Run yarn
-        run: |
-          cd jsx
-          yarn
-
-      - name: yarn test
-        run: |
-          cd jsx
-          yarn test
-
   # Run "pytest jupyterhub/tests" in various configurations
   pytest:
     runs-on: ubuntu-20.04
@@ -80,9 +56,9 @@ jobs:
         #   Tests everything when JupyterHub works against a dedicated mysql or
         #   postgresql server.
         #
-        # nbclassic:
+        # legacy_notebook:
         #   Tests everything when the user instances are started with
-        #   notebook instead of jupyter_server.
+        #   the legacy notebook server instead of jupyter_server.
         #
         # ssl:
         #   Tests everything using internal SSL connections instead of
@@ -96,20 +72,24 @@ jobs:
         #       GitHub UI when the workflow run, we avoid using true/false as
         #       values by instead duplicating the name to signal true.
         include:
-          - python: "3.6"
+          - python: "3.7"
             oldest_dependencies: oldest_dependencies
-            nbclassic: nbclassic
-          - python: "3.6"
-            subdomain: subdomain
-          - python: "3.7"
-            db: mysql
-          - python: "3.7"
-            ssl: ssl
+            legacy_notebook: legacy_notebook
           - python: "3.8"
-            db: postgres
-          - python: "3.8"
-            nbclassic: nbclassic
+            legacy_notebook: legacy_notebook
           - python: "3.9"
+            db: mysql
+          - python: "3.10"
+            db: postgres
+          - python: "3.10"
+            subdomain: subdomain
+          - python: "3.10"
+            ssl: ssl
+          # can't test 3.11.0-beta.4 until a greenlet release
+          # greenlet is a dependency of sqlalchemy on linux
+          # see https://github.com/gevent/gevent/issues/1867
+          # - python: "3.11.0-beta.4"
+          - python: "3.10"
             main_dependencies: main_dependencies
 
     steps:
@@ -137,25 +117,25 @@ jobs:
           if [ "${{ matrix.jupyter_server }}" != "" ]; then
               echo "JUPYTERHUB_SINGLEUSER_APP=jupyterhub.tests.mockserverapp.MockServerApp" >> $GITHUB_ENV
           fi
-      - uses: actions/checkout@v2
-      # NOTE: actions/setup-node@v1 make use of a cache within the GitHub base
+      - uses: actions/checkout@v3
+      # NOTE: actions/setup-node@v3 make use of a cache within the GitHub base
       #       environment and setup in a fraction of a second.
       - name: Install Node v14
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
           node-version: "14"
-      - name: Install Node dependencies
+      - name: Install Javascript dependencies
         run: |
           npm install
-          npm install -g configurable-http-proxy
+          npm install -g configurable-http-proxy yarn
           npm list
 
-      # NOTE: actions/setup-python@v2 make use of a cache within the GitHub base
+      # NOTE: actions/setup-python@v4 make use of a cache within the GitHub base
       #       environment and setup in a fraction of a second.
       - name: Install Python ${{ matrix.python }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python }}
+          python-version: "${{ matrix.python }}"
       - name: Install Python dependencies
         run: |
           pip install --upgrade pip
@@ -172,9 +152,9 @@ jobs:
           if [ "${{ matrix.main_dependencies }}" != "" ]; then
               pip install git+https://github.com/ipython/traitlets#egg=traitlets --force
           fi
-          if [ "${{ matrix.nbclassic }}" != "" ]; then
+          if [ "${{ matrix.legacy_notebook }}" != "" ]; then
               pip uninstall jupyter_server --yes
-              pip install notebook
+              pip install 'notebook<7'
           fi
           if [ "${{ matrix.db }}" == "mysql" ]; then
               pip install mysql-connector-python
@@ -238,7 +218,7 @@ jobs:
     timeout-minutes: 20
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: build images
         run: |

.gitignore

@@ -10,6 +10,7 @@ docs/build
 docs/source/_static/rest-api
 docs/source/rbac/scope-table.md
 .ipynb_checkpoints
+jsx/build/
 # ignore config file at the top-level of the repo
 # but not sub-dirs
 /jupyterhub_config.py
@@ -19,6 +20,7 @@ package-lock.json
 share/jupyterhub/static/components
 share/jupyterhub/static/css/style.min.css
 share/jupyterhub/static/css/style.min.css.map
+share/jupyterhub/static/js/admin-react.js*
 *.egg-info
 MANIFEST
 .coverage

.pre-commit-config.yaml

@@ -1,30 +1,52 @@
+# pre-commit is a tool to perform a predefined set of tasks manually and/or
+# automatically before git commits are made.
+#
+# Config reference: https://pre-commit.com/#pre-commit-configyaml---top-level
+#
+# Common tasks
+#
+# - Run on all files:   pre-commit run --all-files
+# - Register git hooks: pre-commit install --install-hooks
+#
 repos:
+  # Autoformat: Python code, syntax patterns are modernized
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.31.0
+    rev: v2.37.3
     hooks:
       - id: pyupgrade
         args:
           - --py36-plus
-  - repo: https://github.com/asottile/reorder_python_imports
-    rev: v2.6.0
+
+  # Autoformat: Python code
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
     hooks:
-      - id: reorder-python-imports
+      - id: isort
+
+  # Autoformat: Python code
   - repo: https://github.com/psf/black
-    rev: 21.12b0
+    rev: 22.6.0
     hooks:
       - id: black
+
+  # Autoformat: markdown, yaml, javascript (see the file .prettierignore)
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.5.1
+    rev: v2.7.1
     hooks:
       - id: prettier
-  - repo: https://github.com/PyCQA/flake8
-    rev: "4.0.1"
-    hooks:
-      - id: flake8
+
+  # Autoformat and linting, misc. details
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.3.0
     hooks:
       - id: end-of-file-fixer
+        exclude: share/jupyterhub/static/js/admin-react.js
+      - id: requirements-txt-fixer
       - id: check-case-conflict
       - id: check-executables-have-shebangs
-      - id: requirements-txt-fixer
+
+  # Linting: Python code (see the file .flake8)
+  - repo: https://github.com/PyCQA/flake8
+    rev: "5.0.2"
+    hooks:
+      - id: flake8

.readthedocs.yaml

@@ -4,10 +4,12 @@ sphinx:
   configuration: docs/source/conf.py
 
 build:
-  image: latest
+  os: ubuntu-20.04
+  tools:
+    nodejs: "16"
+    python: "3.9"
 
 python:
-  version: 3.7
   install:
     - method: pip
       path: .

CONTRIBUTING.md

@@ -6,134 +6,9 @@ you can follow the [Jupyter contributor guide](https://jupyter.readthedocs.io/en
 Make sure to also follow [Project Jupyter's Code of Conduct](https://github.com/jupyter/governance/blob/HEAD/conduct/code_of_conduct.md)
 for a friendly and welcoming collaborative environment.
 
-## Setting up a development environment
+Please see our documentation on
 
-<!--
-https://jupyterhub.readthedocs.io/en/stable/contributing/setup.html
-contains a lot of the same information. Should we merge the docs and
-just have this page link to that one?
--->
+- [Setting up a development install](https://jupyterhub.readthedocs.io/en/latest/contributing/setup.html)
+- [Testing JupyterHub and linting code](https://jupyterhub.readthedocs.io/en/latest/contributing/tests.html)
 
-JupyterHub requires Python >= 3.5 and nodejs.
-
-As a Python project, a development install of JupyterHub follows standard practices for the basics (steps 1-2).
-
-1. clone the repo
-   ```bash
-   git clone https://github.com/jupyterhub/jupyterhub
-   ```
-2. do a development install with pip
-
-   ```bash
-   cd jupyterhub
-   python3 -m pip install --editable .
-   ```
-
-3. install the development requirements,
-   which include things like testing tools
-
-   ```bash
-   python3 -m pip install -r dev-requirements.txt
-   ```
-
-4. install configurable-http-proxy with npm:
-
-   ```bash
-   npm install -g configurable-http-proxy
-   ```
-
-5. set up pre-commit hooks for automatic code formatting, etc.
-
-   ```bash
-   pre-commit install
-   ```
-
-   You can also invoke the pre-commit hook manually at any time with
-
-   ```bash
-   pre-commit run
-   ```
-
-## Contributing
-
-JupyterHub has adopted automatic code formatting so you shouldn't
-need to worry too much about your code style.
-As long as your code is valid,
-the pre-commit hook should take care of how it should look.
-You can invoke the pre-commit hook by hand at any time with:
-
-```bash
-pre-commit run
-```
-
-which should run any autoformatting on your code
-and tell you about any errors it couldn't fix automatically.
-You may also install [black integration](https://github.com/psf/black#editor-integration)
-into your text editor to format code automatically.
-
-If you have already committed files before setting up the pre-commit
-hook with `pre-commit install`, you can fix everything up using
-`pre-commit run --all-files`. You need to make the fixing commit
-yourself after that.
-
-## Testing
-
-It's a good idea to write tests to exercise any new features,
-or that trigger any bugs that you have fixed to catch regressions.
-
-You can run the tests with:
-
-```bash
-pytest -v
-```
-
-in the repo directory. If you want to just run certain tests,
-check out the [pytest docs](https://pytest.readthedocs.io/en/latest/usage.html)
-for how pytest can be called.
-For instance, to test only spawner-related things in the REST API:
-
-```bash
-pytest -v -k spawn jupyterhub/tests/test_api.py
-```
-
-The tests live in `jupyterhub/tests` and are organized roughly into:
-
-1. `test_api.py` tests the REST API
-2. `test_pages.py` tests loading the HTML pages
-
-and other collections of tests for different components.
-When writing a new test, there should usually be a test of
-similar functionality already written and related tests should
-be added nearby.
-
-The fixtures live in `jupyterhub/tests/conftest.py`. There are
-fixtures that can be used for JupyterHub components, such as:
-
-- `app`: an instance of JupyterHub with mocked parts
-- `auth_state_enabled`: enables persisting auth_state (like authentication tokens)
-- `db`: a sqlite in-memory DB session
-- `io_loop`: a Tornado event loop
-- `event_loop`: a new asyncio event loop
-- `user`: creates a new temporary user
-- `admin_user`: creates a new temporary admin user
-- single user servers
-  - `cleanup_after`: allows cleanup of single user servers between tests
-- mocked service
-  - `MockServiceSpawner`: a spawner that mocks services for testing with a short poll interval
-  - `mockservice`: mocked service with no external service url
-  - `mockservice_url`: mocked service with a url to test external services
-
-And fixtures to add functionality or spawning behavior:
-
-- `admin_access`: grants admin access
-- `no_patience`: sets slow-spawning timeouts to zero
-- `slow_spawn`: enables the SlowSpawner (a spawner that takes a few seconds to start)
-- `never_spawn`: enables the NeverSpawner (a spawner that will never start)
-- `bad_spawn`: enables the BadSpawner (a spawner that fails immediately)
-- `slow_bad_spawn`: enables the SlowBadSpawner (a spawner that fails after a short delay)
-
-To read more about fixtures check out the
-[pytest docs](https://docs.pytest.org/en/latest/fixture.html)
-for how to use the existing fixtures, and how to create new ones.
-
-When in doubt, feel free to [ask](https://gitter.im/jupyterhub/jupyterhub).
+If you need some help, feel free to ask on [Gitter](https://gitter.im/jupyterhub/jupyterhub) or [Discourse](https://discourse.jupyter.org/).

Dockerfile

@@ -21,7 +21,7 @@
 # your jupyterhub_config.py will be added automatically
 # from your docker directory.
 
-ARG BASE_IMAGE=ubuntu:focal-20200729
+ARG BASE_IMAGE=ubuntu:22.04
 FROM $BASE_IMAGE AS builder
 
 USER root
@@ -37,6 +37,7 @@ RUN apt-get update \
     python3-pycurl \
     nodejs \
     npm \
+    yarn \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*
 

MANIFEST.in

@@ -8,6 +8,7 @@ include *requirements.txt
 include Dockerfile
 
 graft onbuild
+graft jsx
 graft jupyterhub
 graft scripts
 graft share
@@ -18,6 +19,10 @@ graft ci
 graft docs
 prune docs/node_modules
 
+# Intermediate javascript files
+prune jsx/node_modules
+prune jsx/build
+
 # prune some large unused files from components
 prune share/jupyterhub/static/components/bootstrap/dist/css
 exclude share/jupyterhub/static/components/bootstrap/dist/fonts/*.svg

README.md

@@ -59,7 +59,7 @@ JupyterHub also provides a
 [REST API][]
 for administration of the Hub and its users.
 
-[rest api]: https://juptyerhub.readthedocs.io/en/latest/reference/rest-api.html
+[rest api]: https://jupyterhub.readthedocs.io/en/latest/reference/rest-api.html
 
 ## Installation
 

ci/check_installed_data.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# Check that installed package contains everything we expect
+
+
+import os
+
+from jupyterhub._data import DATA_FILES_PATH
+
+print("Checking jupyterhub._data")
+print(f"DATA_FILES_PATH={DATA_FILES_PATH}")
+assert os.path.exists(DATA_FILES_PATH), DATA_FILES_PATH
+for subpath in (
+    "templates/page.html",
+    "static/css/style.min.css",
+    "static/components/jquery/dist/jquery.js",
+    "static/js/admin-react.js",
+):
+    path = os.path.join(DATA_FILES_PATH, subpath)
+    assert os.path.exists(path), path
+print("OK")

ci/check_sdist.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# Check that sdist contains everything we expect
+
+import sys
+import tarfile
+from tarfile import TarFile
+
+expected_files = [
+    "docs/requirements.txt",
+    "jsx/package.json",
+    "package.json",
+    "README.md",
+]
+
+assert len(sys.argv) == 2, "Expected one file"
+print(f"Checking {sys.argv[1]}")
+
+tar = tarfile.open(name=sys.argv[1], mode="r:gz")
+try:
+    # Remove leading jupyterhub-VERSION/
+    filelist = {f.partition('/')[2] for f in tar.getnames()}
+finally:
+    tar.close()
+
+for e in expected_files:
+    assert e in filelist, f"{e} not found"
+
+print("OK")

ci/init-db.sh

@@ -20,7 +20,7 @@ fi
 
 # Configure a set of databases in the database server for upgrade tests
 set -x
-for SUFFIX in '' _upgrade_100 _upgrade_122 _upgrade_130; do
+for SUFFIX in '' _upgrade_100 _upgrade_122 _upgrade_130 _upgrade_150 _upgrade_211; do
     $SQL_CLIENT "DROP DATABASE jupyterhub${SUFFIX};" 2>/dev/null || true
     $SQL_CLIENT "CREATE DATABASE jupyterhub${SUFFIX} ${EXTRA_CREATE_DATABASE_ARGS:-};"
 done

dev-requirements.txt

@@ -9,9 +9,13 @@ cryptography
 html5lib  # needed for beautifulsoup
 jupyterlab >=3
 mock
+# nbclassic provides the '/tree/' handler, which we use in tests
+# it is a transitive dependency via jupyterlab,
+# but depend on it directly
+nbclassic
 pre-commit
 pytest>=3.3
-pytest-asyncio
+pytest-asyncio>=0.17
 pytest-cov
 requests-mock
 tbump

docs/source/_static/rest-api.yml

@@ -6,7 +6,7 @@ info:
   description: The REST API for JupyterHub
   license:
     name: BSD-3-Clause
-  version: 2.1.0
+  version: 3.0.0b1
 servers:
   - url: /hub/api
 security:
@@ -139,6 +139,16 @@ paths:
             If unspecified, use api_page_default_limit.
           schema:
             type: number
+        - name: include_stopped_servers
+          in: query
+          description: |
+            Include stopped servers in user model(s).
+            Added in JupyterHub 3.0.
+            Allows retrieval of information about stopped servers,
+            such as activity and state fields.
+          schema:
+            type: boolean
+          allowEmptyValue: true
       responses:
         200:
           description: The Hub's user list
@@ -560,7 +570,19 @@ paths:
                   description: A note attached to the token for future bookkeeping
                 roles:
                   type: array
-                  description: A list of role names that the token should have
+                  description: |
+                    A list of role names from which to derive scopes.
+                    This is a shortcut for assigning collections of scopes;
+                    Tokens do not retain role assignment.
+                    (Changed in 3.0: roles are immediately resolved to scopes
+                    instead of stored on roles.)
+                  items:
+                    type: string
+                scopes:
+                  type: array
+                  description: |
+                    A list of scopes that the token should have.
+                    (new in JupyterHub 3.0).
                   items:
                     type: string
         required: false
@@ -1148,7 +1170,11 @@ components:
           format: date-time
         servers:
           type: array
-          description: The active servers for this user.
+          description: |
+            The servers for this user.
+            By default: only includes _active_ servers.
+            Changed in 3.0: if `?include_stopped_servers` parameter is specified,
+            stopped servers will be included as well.
           items:
             $ref: "#/components/schemas/Server"
         auth_state:
@@ -1170,6 +1196,15 @@ components:
           description: |
             Whether the server is ready for traffic.
             Will always be false when any transition is pending.
+        stopped:
+          type: boolean
+          description: |
+            Whether the server is stopped. Added in JupyterHub 3.0,
+            and only useful when using the `?include_stopped_servers`
+            request parameter.
+            Now that stopped servers may be included (since JupyterHub 3.0),
+            this is the simplest way to select stopped servers.
+            Always equivalent to `not (ready or pending)`.
         pending:
           type: string
           description: |
@@ -1314,7 +1349,16 @@ components:
           description: The service that owns the token (undefined of owned by a user)
         roles:
           type: array
-          description: The names of roles this token has
+          description:
+            Deprecated in JupyterHub 3, always an empty list. Tokens have
+            'scopes' starting from JupyterHub 3.
+          items:
+            type: string
+        scopes:
+          type: array
+          description:
+            List of scopes this token has been assigned. New in JupyterHub
+            3. In JupyterHub 2.x, tokens were assigned 'roles' insead of scopes.
           items:
             type: string
         note:
@@ -1370,6 +1414,9 @@ components:
             inherit:
               Everything that the token-owning entity can access _(metascope
               for tokens)_
+            admin-ui:
+              Access the admin page. Permission to take actions via the admin
+              page granted separately.
             admin:users:
               Read, write, create and delete users and their authentication
               state, not including their servers or tokens.

docs/source/admin/log-messages.md

@@ -0,0 +1,72 @@
+# Interpreting common log messages
+
+When debugging errors and outages, looking at the logs emitted by
+JupyterHub is very helpful. This document tries to document some common
+log messages, and what they mean.
+
+## Failing suspected API request to not-running server
+
+### Example
+
+Your logs might be littered with lines that might look slightly scary
+
+```
+[W 2022-03-10 17:25:19.774 JupyterHub base:1349] Failing suspected API request to not-running server: /hub/user/<user-name>/api/metrics/v1
+```
+
+### Most likely cause
+
+This likely means is that the user's server has stopped running but they
+still have a browser tab open. For example, you might have 3 tabs open, and shut
+your server down via one. Or you closed your laptop, your server was
+culled for inactivity, and then you reopen your laptop again! The
+client side code (JupyterLab, Classic Notebook, etc) does not know
+yet that the server is dead, and continues to make some API requests.
+JupyterHub's architecture means that the proxy routes all requests that
+don't go to a running user server to the hub process itself. The hub
+process then explicitly returns a failure response, so the client knows
+that the server is not running anymore. This is used by JupyterLab to
+tell you your server is not running anymore, and offer you the option
+to let you restart it.
+
+Most commonly, you'll see this in reference to the `/api/metrics/v1`
+URL, used by [jupyter-resource-usage](https://github.com/jupyter-server/jupyter-resource-usage).
+
+### Actions you can take
+
+This log message is benign, and there is usually no action for you to take.
+
+## JupyterHub Singleuser Version mismatch
+
+### Example
+
+```
+ jupyterhub version 1.5.0 != jupyterhub-singleuser version 1.3.0. This could cause failure to authenticate and result in redirect loops!
+```
+
+### Cause
+
+JupyterHub requires the `jupyterhub` python package installed inside the image or
+environment the user server starts in. This message indicates that the version of
+the `jupyterhub` package installed inside the user image or environment is not
+the same version as the JupyterHub server itself. This is not necessarily always a
+problem - some version drift is mostly acceptable, and the only two known cases of
+breakage are across the 0.7 and 2.0 version releases. In those cases, issues pop
+up immediately after upgrading your version of JupyterHub, so **always check the JupyterHub
+changelog before upgrading!**. The primary problems this _could_ cause are:
+
+1. Infinite redirect loops after the user server starts
+2. Missing expected environment variables in the user server once it starts
+3. Failure for the started user server to authenticate with the JupyterHub server -
+   note that this is _not_ the same as _user authentication_ failing!
+
+However, for the most part, unless you are seeing these specific issues, the log
+message should be counted as a warning to get the `jupyterhub` package versions
+aligned, rather than as an indicator of an existing problem.
+
+### Actions you can take
+
+Upgrade the version of the `jupyterhub` package in your user environment or image
+so it matches the version of JupyterHub running your JupyterHub server! If you
+are using the [zero-to-jupyterhub](https://z2jh.jupyter.org) helm chart, you can find the appropriate
+version of the `jupyterhub` package to install in your user image [here](https://jupyterhub.github.io/helm-chart/)

docs/source/conf.py

@@ -21,6 +21,7 @@ extensions = [
     'myst_parser',
 ]
 
+myst_heading_anchors = 2
 myst_enable_extensions = [
     'colon_fence',
     'deflist',
@@ -47,7 +48,7 @@ version = '%i.%i' % jupyterhub.version_info[:2]
 # The full version, including alpha/beta/rc tags.
 release = jupyterhub.__version__
 
-language = None
+language = "en"
 exclude_patterns = []
 pygments_style = 'sphinx'
 todo_include_todos = False
@@ -55,13 +56,15 @@ todo_include_todos = False
 # Set the default role so we can use `foo` instead of ``foo``
 default_role = 'literal'
 
-# -- Config -------------------------------------------------------------
-from jupyterhub.app import JupyterHub
-from docutils import nodes
-from sphinx.directives.other import SphinxDirective
 from contextlib import redirect_stdout
 from io import StringIO
 
+from docutils import nodes
+from sphinx.directives.other import SphinxDirective
+
+# -- Config -------------------------------------------------------------
+from jupyterhub.app import JupyterHub
+
 # create a temp instance of JupyterHub just to get the output of the generate-config
 # and help --all commands.
 jupyterhub_app = JupyterHub()

docs/source/contributing/setup.rst

@@ -16,7 +16,7 @@ Install Python
 --------------
 
 JupyterHub is written in the `Python <https://python.org>`_ programming language, and
-requires you have at least version 3.5 installed locally. If you haven’t
+requires you have at least version 3.6 installed locally. If you haven’t
 installed Python before, the recommended way to install it is to use
 `miniconda <https://conda.io/miniconda.html>`_. Remember to get the ‘Python 3’ version,
 and **not** the ‘Python 2’ version!
@@ -24,11 +24,10 @@ and **not** the ‘Python 2’ version!
 Install nodejs
 --------------
 
-``configurable-http-proxy``, the default proxy implementation for
-JupyterHub, is written in Javascript to run on `NodeJS
-<https://nodejs.org/en/>`_. If you have not installed nodejs before, we
-recommend installing it in the ``miniconda`` environment you set up for
-Python. You can do so with ``conda install nodejs``.
+`NodeJS 12+ <https://nodejs.org/en/>`_ is required for building some JavaScript components.
+``configurable-http-proxy``, the default proxy implementation for JupyterHub, is written in Javascript.
+If you have not installed nodejs before, we recommend installing it in the ``miniconda`` environment you set up for Python.
+You can do so with ``conda install nodejs``.
 
 Install git
 -----------
@@ -46,7 +45,7 @@ their effects quickly. You need to do a developer install to make that
 happen.
 
 .. note:: This guide does not attempt to dictate *how* development
-   environements should be isolated since that is a personal preference and can
+   environments should be isolated since that is a personal preference and can
    be achieved in many ways, for example `tox`, `conda`, `docker`, etc. See this
    `forum thread <https://discourse.jupyter.org/t/thoughts-on-using-tox/3497>`_ for
    a more detailed discussion.
@@ -66,7 +65,7 @@ happen.
 
       python -V
 
-   This should return a version number greater than or equal to 3.5.
+   This should return a version number greater than or equal to 3.6.
 
    .. code:: bash
 
@@ -74,12 +73,11 @@ happen.
 
    This should return a version number greater than or equal to 5.0.
 
-3. Install ``configurable-http-proxy``. This is required to run
-   JupyterHub.
+3. Install ``configurable-http-proxy`` (required to run and test the default JupyterHub configuration) and ``yarn`` (required to build some components):
 
    .. code:: bash
 
-      npm install -g configurable-http-proxy
+      npm install -g configurable-http-proxy yarn
 
    If you get an error that says ``Error: EACCES: permission denied``,
    you might need to prefix the command with ``sudo``. If you do not
@@ -87,11 +85,17 @@ happen.
 
    .. code:: bash
 
-      npm install configurable-http-proxy
+      npm install configurable-http-proxy yarn
       export PATH=$PATH:$(pwd)/node_modules/.bin
 
    The second line needs to be run every time you open a new terminal.
 
+   If you are using conda you can instead run:
+
+   .. code:: bash
+
+      conda install configurable-http-proxy yarn
+
 4. Install the python packages required for JupyterHub development.
 
    .. code:: bash
@@ -186,3 +190,4 @@ development updates, with:
 
    python3 setup.py js    # fetch updated client-side js
    python3 setup.py css   # recompile CSS from LESS sources
+   python3 setup.py jsx   # build React admin app

docs/source/contributing/tests.rst

@@ -1,8 +1,8 @@
 .. _contributing/tests:
 
-==================
-Testing JupyterHub
-==================
+===================================
+Testing JupyterHub and linting code
+===================================
 
 Unit test help validate that JupyterHub works the way we think it does,
 and continues to do so when changes occur. They also help communicate
@@ -57,6 +57,50 @@ Running the tests
       
       pytest -v jupyterhub/tests/test_api.py::test_shutdown
 
+   See the `pytest usage documentation <https://pytest.readthedocs.io/en/latest/usage.html>`_ for more details.
+
+Test organisation
+=================
+
+The tests live in ``jupyterhub/tests`` and are organized roughly into:
+
+#. ``test_api.py`` tests the REST API
+#. ``test_pages.py`` tests loading the HTML pages
+
+and other collections of tests for different components.
+When writing a new test, there should usually be a test of
+similar functionality already written and related tests should
+be added nearby.
+
+The fixtures live in ``jupyterhub/tests/conftest.py``. There are
+fixtures that can be used for JupyterHub components, such as:
+
+- ``app``: an instance of JupyterHub with mocked parts
+- ``auth_state_enabled``: enables persisting auth_state (like authentication tokens)
+- ``db``: a sqlite in-memory DB session
+- ``io_loop```: a Tornado event loop
+- ``event_loop``: a new asyncio event loop
+- ``user``: creates a new temporary user
+- ``admin_user``: creates a new temporary admin user
+- single user servers
+  - ``cleanup_after``: allows cleanup of single user servers between tests
+- mocked service
+  - ``MockServiceSpawner``: a spawner that mocks services for testing with a short poll interval
+  - ``mockservice```: mocked service with no external service url
+  - ``mockservice_url``: mocked service with a url to test external services
+
+And fixtures to add functionality or spawning behavior:
+
+- ``admin_access``: grants admin access
+- ``no_patience```: sets slow-spawning timeouts to zero
+- ``slow_spawn``: enables the SlowSpawner (a spawner that takes a few seconds to start)
+- ``never_spawn``: enables the NeverSpawner (a spawner that will never start)
+- ``bad_spawn``: enables the BadSpawner (a spawner that fails immediately)
+- ``slow_bad_spawn``: enables the SlowBadSpawner (a spawner that fails after a short delay)
+
+See the `pytest fixtures documentation <https://pytest.readthedocs.io/en/latest/fixture.html>`_
+for how to use the existing fixtures, and how to create new ones.
+
 
 Troubleshooting Test Failures
 =============================
@@ -66,3 +110,27 @@ All the tests are failing
 
 Make sure you have completed all the steps in :ref:`contributing/setup` successfully, and
 can launch ``jupyterhub`` from the terminal.
+
+
+Code formatting and linting
+===========================
+
+JupyterHub has adopted automatic code formatting and linting.
+As long as your code is valid, the pre-commit hook should take care of how it should look.
+You can invoke the pre-commit hook by hand at any time with:
+
+.. code:: bash
+
+   pre-commit run
+
+which should run any autoformatting on your code and tell you about any errors it couldn't fix automatically.
+You may also install `black integration <https://github.com/psf/black#editor-integration>`_
+into your text editor to format code automatically.
+
+If you have already committed files before running pre-commit you can fix everything using:
+
+.. code:: bash
+
+   pre-commit run --all-files
+
+And committing the changes.

docs/source/getting-started/security-basics.rst

@@ -183,12 +183,6 @@ itself, ``jupyterhub_config.py``, as a binary string:
 
     c.JupyterHub.cookie_secret = bytes.fromhex('64 CHAR HEX STRING')
 
-
-.. important::
-
-   If the cookie secret value changes for the Hub, all single-user notebook
-   servers must also be restarted.
-
 .. _cookies:
 
 Cookies used by JupyterHub authentication

docs/source/index-admin.rst

@@ -10,4 +10,5 @@ well as other information relevant to running your own JupyterHub over time.
 
    troubleshooting
    admin/upgrading
+   admin/log-messages
    changelog

docs/source/rbac/index.md

@@ -1,3 +1,5 @@
+(RBAC)=
+
 # JupyterHub RBAC
 
 Role Based Access Control (RBAC) in JupyterHub serves to provide fine grained control of access to Jupyterhub's API resources.

docs/source/rbac/roles.md

@@ -7,7 +7,7 @@ JupyterHub provides four roles that are available by default:
 ```{admonition} **Default roles**
 - `user` role provides a {ref}`default user scope <default-user-scope-target>` `self` that grants access to the user's own resources.
 - `admin` role contains all available scopes and grants full rights to all actions. This role **cannot be edited**.
-- `token` role provides a {ref}`default token scope <default-token-scope-target>` `all` that resolves to the same permissions as the owner of the token has.
+- `token` role provides a {ref}`default token scope <default-token-scope-target>` `inherit` that resolves to the same permissions as the owner of the token has.
 - `server` role allows for posting activity of "itself" only.
 
 **These roles cannot be deleted.**
@@ -27,7 +27,6 @@ Roles can be assigned to the following entities:
 - Users
 - Services
 - Groups
-- Tokens
 
 An entity can have zero, one, or multiple roles, and there are no restrictions on which roles can be assigned to which entity. Roles can be added to or removed from entities at any time.
 
@@ -41,7 +40,7 @@ Services do not have a default role. Services without roles have no access to th
 A group does not require any role, and has no roles by default. If a user is a member of a group, they automatically inherit any of the group's permissions (see {ref}`resolving-roles-scopes-target` for more details). This is useful for assigning a set of common permissions to several users.
 
 **Tokens** \
-A token’s permissions are evaluated based on their owning entity. Since a token is always issued for a user or service, it can never have more permissions than its owner. If no specific role is requested for a new token, the token is assigned the `token` role.
+A token’s permissions are evaluated based on their owning entity. Since a token is always issued for a user or service, it can never have more permissions than its owner. If no specific scopes are requested for a new token, the token is assigned the scopes of the `token` role.
 
 (define-role-target)=
 

docs/source/rbac/scopes.md

@@ -38,7 +38,7 @@ By adding a scope to an existing role, all role bearers will gain the associated
 Metascopes do not follow the general scope syntax. Instead, a metascope resolves to a set of scopes, which can refer to different resources, based on their owning entity. In JupyterHub, there are currently two metascopes:
 
 1. default user scope `self`, and
-2. default token scope `all`.
+2. default token scope `inherit`.
 
 (default-user-scope-target)=
 
@@ -57,11 +57,11 @@ The `self` scope is only valid for user entities. In other cases (e.g., for serv
 
 ### Default token scope
 
-The token metascope `all` covers the same scopes as the token owner's scopes during requests. For example, if a token owner has roles containing the scopes `read:groups` and `read:users`, the `all` scope resolves to the set of scopes `{read:groups, read:users}`.
+The token metascope `inherit` causes the token to have the same permissions as the token's owner. For example, if a token owner has roles containing the scopes `read:groups` and `read:users`, the `inherit` scope resolves to the set of scopes `{read:groups, read:users}`.
 
-If the token owner has default `user` role, the `all` scope resolves to `self`, which will subsequently be expanded to include all the user-specific scopes (or empty set in the case of services).
+If the token owner has default `user` role, the `inherit` scope resolves to `self`, which will subsequently be expanded to include all the user-specific scopes (or empty set in the case of services).
 
-If the token owner is a member of any group with roles, the group scopes will also be included in resolving the `all` scope.
+If the token owner is a member of any group with roles, the group scopes will also be included in resolving the `inherit` scope.
 
 (horizontal-filtering-target)=
 
@@ -72,13 +72,31 @@ Requested resources are filtered based on the filter of the corresponding scope.
 
 In case a user resource is being accessed, any scopes with _group_ filters will be expanded to filters for each _user_ in those groups.
 
-### `!user` filter
+(self-referencing-filters)=
+
+### Self-referencing filters
+
+There are some 'shortcut' filters,
+which can be applied to all scopes,
+that filter based on the entities associated with the request.
 
 The `!user` filter is a special horizontal filter that strictly refers to the **"owner only"** scopes, where _owner_ is a user entity. The filter resolves internally into `!user=<ownerusername>` ensuring that only the owner's resources may be accessed through the associated scopes.
 
 For example, the `server` role assigned by default to server tokens contains `access:servers!user` and `users:activity!user` scopes. This allows the token to access and post activity of only the servers owned by the token owner.
 
-The filter can be applied to any scope.
+:::{versionadded} 3.0
+`!service` and `!server` filters.
+:::
+
+In addition to `!user`, _tokens_ may have filters `!service`
+or `!server`, which expand similarly to `!service=servicename`
+and `!server=servername`.
+This only applies to tokens issued via the OAuth flow.
+In these cases, the name is the _issuing_ entity (a service or single-user server),
+so that access can be restricted to the issuing service,
+e.g. `access:servers!server` would grant access only to the server that requested the token.
+
+These filters can be applied to any scope.
 
 (vertical-filtering-target)=
 
@@ -114,11 +132,170 @@ There are four exceptions to the general {ref}`scope conventions <scope-conventi
 
 ```
 
+:::{versionadded} 3.0
+The `admin-ui` scope is added to explicitly grant access to the admin page,
+rather than combining `admin:users` and `admin:servers` permissions.
+This means a deployment can enable the admin page with only a subset of functionality enabled.
+
+Note that this means actions to take _via_ the admin UI
+and access _to_ the admin UI are separated.
+For example, it generally doesn't make sense to grant
+`admin-ui` without at least `list:users` for at least some subset of users.
+
+For example:
+
+```python
+c.JupyterHub.load_roles = [
+  {
+    "name": "instructor-data8",
+    "scopes": [
+      # access to the admin page
+      "admin-ui",
+      # list users in the class group
+      "list:users!group=students-data8",
+      # start/stop servers for users in the class
+      "admin:servers!group=students-data8",
+      # access servers for users in the class
+      "access:servers!group=students-data8",
+    ],
+    "group": ["instructors-data8"],
+  }
+]
+```
+
+will grant instructors in the data8 course permission to:
+
+1. view the admin UI
+2. see students in the class (but not all users)
+3. start/stop/access servers for users in the class
+4. but _not_ permission to administer the users themselves (e.g. change their permissions, etc.)
+   :::
+
 ```{Caution}
 Note that only the {ref}`horizontal filtering <horizontal-filtering-target>` can be added to scopes to customize them. \
 Metascopes `self` and `all`, `<resource>`, `<resource>:<subresource>`, `read:<resource>`, `admin:<resource>`, and `access:<resource>` scopes are predefined and cannot be changed otherwise.
 ```
 
+(custom-scopes)=
+
+### Custom scopes
+
+:::{versionadded} 3.0
+:::
+
+JupyterHub 3.0 introduces support for custom scopes.
+Services that use JupyterHub for authentication and want to implement their own granular access may define additional _custom_ scopes and assign them to users with JupyterHub roles.
+
+% Note: keep in sync with pattern/description in jupyterhub/scopes.py
+
+Custom scope names must start with `custom:`
+and contain only lowercase ascii letters, numbers, hyphen, underscore, colon, and asterisk (`-_:*`).
+The part after `custom:` must start with a letter or number.
+Scopes may not end with a hyphen or colon.
+
+The only strict requirement is that a custom scope definition must have a `description`.
+It _may_ also have `subscopes` if you are defining multiple scopes that have a natural hierarchy,
+
+For example:
+
+```python
+c.JupyterHub.custom_scopes = {
+    "custom:myservice:read": {
+        "description": "read-only access to myservice",
+    },
+    "custom:myservice:write": {
+        "description": "write access to myservice",
+        # write permission implies read permission
+        "subscopes": [
+            "custom:myservice:read",
+        ],
+    },
+}
+
+c.JupyterHub.load_roles = [
+    # graders have read-only access to the service
+    {
+        "name": "service-user",
+        "groups": ["graders"],
+        "scopes": [
+            "custom:myservice:read",
+            "access:service!service=myservice",
+        ],
+    },
+    # instructors have read and write access to the service
+    {
+        "name": "service-admin",
+        "groups": ["instructors"],
+        "scopes": [
+            "custom:myservice:write",
+            "access:service!service=myservice",
+        ],
+    },
+]
+```
+
+In the above configuration, two scopes are defined:
+
+- `custom:myservice:read` grants read-only access to the service, and
+- `custom:myservice:write` grants write access to the service
+- write access _implies_ read access via the `subscope`
+
+These custom scopes are assigned to two groups via `roles`:
+
+- users in the group `graders` are granted read access to the service
+- users in the group `instructors` are
+- both are granted _access_ to the service via `access:service!service=myservice`
+
+When the service completes OAuth, it will retrieve the user model from `/hub/api/user`.
+This model includes a `scopes` field which is a list of authorized scope for the request,
+which can be used.
+
+```python
+def require_scope(scope):
+    """decorator to require a scope to perform an action"""
+    def wrapper(func):
+        @functools.wraps(func)
+        def wrapped_func(request):
+            user = fetch_hub_api_user(request.token)
+            if scope not in user["scopes"]:
+                raise HTTP403(f"Requires scope {scope}")
+            else:
+                return func()
+    return wrapper
+
+@require_scope("custom:myservice:read")
+async def read_something(request):
+    ...
+
+@require_scope("custom:myservice:write")
+async def write_something(request):
+    ...
+```
+
+If you use {class}`~.HubOAuthenticated`, this check is performed automatically
+against the `.hub_scopes` attribute of each Handler
+(the default is populated from `$JUPYTERHUB_OAUTH_ACCESS_SCOPES` and usually `access:services!service=myservice`).
+
+:::{versionchanged} 3.0
+The JUPYTERHUB_OAUTH_SCOPES environment variable is deprecated and renamed to JUPYTERHUB_OAUTH_ACCESS_SCOPES,
+to avoid ambiguity with JUPYTERHUB_OAUTH_CLIENT_ALLOWED_SCOPES
+:::
+
+```python
+from tornado import web
+from jupyterhub.services.auth import HubOAuthenticated
+
+class MyHandler(HubOAuthenticated, BaseHandler):
+    hub_scopes = ["custom:myservice:read"]
+
+    @web.authenticated
+    def get(self):
+        ...
+```
+
+Existing scope filters (`!user=`, etc.) may be applied to custom scopes.
+Custom scope _filters_ are NOT supported.
+
 ### Scopes and APIs
 
 The scopes are also listed in the [](../reference/rest-api.rst) documentation. Each API endpoint has a list of scopes which can be used to access the API; if no scopes are listed, the API is not authenticated and can be accessed without any permissions (i.e., no scopes).

docs/source/rbac/tech-implementation.md

@@ -7,11 +7,11 @@ Roles and scopes utilities can be found in `roles.py` and `scopes.py` modules. S
 ```{admonition} **Scope variable nomenclature**
 :class: tip
 - _scopes_ \
-  List of scopes with abbreviations (used in role definitions). E.g., `["users:activity!user"]`.
+  List of scopes that may contain abbreviations (used in role definitions). E.g., `["users:activity!user", "self"]`.
 - _expanded scopes_ \
-  Set of expanded scopes without abbreviations (i.e., resolved metascopes, filters and subscopes). E.g., `{"users:activity!user=charlie", "read:users:activity!user=charlie"}`.
+  Set of fully expanded scopes without abbreviations (i.e., resolved metascopes, filters, and subscopes). E.g., `{"users:activity!user=charlie", "read:users:activity!user=charlie"}`.
 - _parsed scopes_ \
-  Dictionary JSON like format of expanded scopes. E.g., `{"users:activity": {"user": ["charlie"]}, "read:users:activity": {"users": ["charlie"]}}`.
+  Dictionary represenation of expanded scopes. E.g., `{"users:activity": {"user": ["charlie"]}, "read:users:activity": {"users": ["charlie"]}}`.
 - _intersection_ \
   Set of expanded scopes as intersection of 2 expanded scope sets.
 - _identify scopes_ \
@@ -22,27 +22,47 @@ Roles and scopes utilities can be found in `roles.py` and `scopes.py` modules. S
 
 ## Resolving roles and scopes
 
-**Resolving roles** refers to determining which roles a user, service, token, or group has, extracting the list of scopes from each role and combining them into a single set of scopes.
+**Resolving roles** refers to determining which roles a user, service, or group has, extracting the list of scopes from each role and combining them into a single set of scopes.
 
 **Resolving scopes** involves expanding scopes into all their possible subscopes (_expanded scopes_), parsing them into format used for access evaluation (_parsed scopes_) and, if applicable, comparing two sets of scopes (_intersection_). All procedures take into account the scope hierarchy, {ref}`vertical <vertical-filtering-target>` and {ref}`horizontal filtering <horizontal-filtering-target>`, limiting or elevated permissions (`read:<resource>` or `admin:<resource>`, respectively), and metascopes.
 
-Roles and scopes are resolved on several occasions, for example when requesting an API token with specific roles or making an API request. The following sections provide more details.
+Roles and scopes are resolved on several occasions, for example when requesting an API token with specific scopes or making an API request. The following sections provide more details.
 
 (requesting-api-token-target)=
 
-### Requesting API token with specific roles
+### Requesting API token with specific scopes
 
-API tokens grant access to JupyterHub's APIs. The RBAC framework allows for requesting tokens with specific existing roles. To date, it is only possible to add roles to a token through the _POST /users/:name/tokens_ API where the roles can be specified in the token parameters body (see [](../reference/rest-api.rst)).
+:::{versionchanged} 3.0
+API tokens have _scopes_ instead of roles,
+so that their permissions cannot be updated.
 
-RBAC adds several steps into the token issue flow.
+You may still request roles for a token,
+but those roles will be evaluated to the corresponding _scopes_ immediately.
 
-If no roles are requested, the token is issued with the default `token` role (providing the requester is allowed to create the token).
+Prior to 3.0, tokens stored _roles_,
+which meant their scopes were resolved on each request.
+:::
 
-If the token is requested with any roles, the permissions of requesting entity are checked against the requested permissions to ensure the token would not grant its owner additional privileges.
+API tokens grant access to JupyterHub's APIs. The RBAC framework allows for requesting tokens with specific permissions.
 
-If, due to modifications of roles or entities, at API request time a token has any scopes that its owner does not, those scopes are removed. The API request is resolved without additional errors using the scopes _intersection_, but the Hub logs a warning (see {ref}`Figure 2 <api-request-chart>`).
+RBAC is involved in several stages of the OAuth token flow.
 
-Resolving a token's roles (yellow box in {ref}`Figure 1 <token-request-chart>`) corresponds to resolving all the token's owner roles (including the roles associated with their groups) and the token's requested roles into a set of scopes. The two sets are compared (Resolve the scopes box in orange in {ref}`Figure 1 <token-request-chart>`), taking into account the scope hierarchy but, solely for role assignment, omitting any {ref}`horizontal filter <horizontal-filtering-target>` comparison. If the token's scopes are a subset of the token owner's scopes, the token is issued with the requested roles; if not, JupyterHub will raise an error.
+When requesting a token via the tokens API (`/users/:name/tokens`), or the token page (`/hub/token`),
+if no scopes are requested, the token is issued with the permissions stored on the default `token` role
+(providing the requester is allowed to create the token).
+
+OAuth tokens are also requested via OAuth flow
+
+If the token is requested with any scopes, the permissions of requesting entity are checked against the requested permissions to ensure the token would not grant its owner additional privileges.
+
+If, due to modifications of permissions of the token or token owner,
+at API request time a token has any scopes that its owner does not,
+those scopes are removed.
+The API request is resolved without additional errors using the scope _intersection_;
+the Hub logs a warning in this case (see {ref}`Figure 2 <api-request-chart>`).
+
+Resolving a token's scope (yellow box in {ref}`Figure 1 <token-request-chart>`) corresponds to resolving all the token's owner roles (including the roles associated with their groups) and the token's own scopes into a set of scopes. The two sets are compared (Resolve the scopes box in orange in {ref}`Figure 1 <token-request-chart>`), taking into account the scope hierarchy.
+If the token's scopes are a subset of the token owner's scopes, the token is issued with the requested scopes; if not, JupyterHub will raise an error.
 
 {ref}`Figure 1 <token-request-chart>` below illustrates the steps involved. The orange rectangles highlight where in the process the roles and scopes are resolved.
 
@@ -55,9 +75,9 @@ Figure 1. Resolving roles and scopes during API token request
 
 ### Making an API request
 
-With the RBAC framework each authenticated JupyterHub API request is guarded by a scope decorator that specifies which scopes are required to gain the access to the API.
+With the RBAC framework, each authenticated JupyterHub API request is guarded by a scope decorator that specifies which scopes are required to gain the access to the API.
 
-When an API request is performed, the requesting API token's roles are again resolved (yellow box in {ref}`Figure 2 <api-request-chart>`) to ensure the token does not grant more permissions than its owner has at the request time (e.g., due to changing/losing roles).
+When an API request is performed, the requesting API token's scopes are again intersected with its owner's (yellow box in {ref}`Figure 2 <api-request-chart>`) to ensure the token does not grant more permissions than its owner has at the request time (e.g., due to changing/losing roles).
 If the owner's roles do not include some scopes of the token's scopes, only the _intersection_ of the token's and owner's scopes will be used. For example, using a token with scope `users` whose owner's role scope is `read:users:name` will result in only the `read:users:name` scope being passed on. In the case of no _intersection_, an empty set of scopes will be used.
 
 The passed scopes are compared to the scopes required to access the API as follows:

docs/source/rbac/upgrade.md

@@ -49,6 +49,6 @@ API tokens can also be issued to users via API ([_/hub/token_](../reference/urls
 
 ### With RBAC
 
-The RBAC framework allows for granting tokens different levels of permissions via scopes attached to roles. The 'only identify' purpose of the separate OAuth tokens is no longer required. API tokens can be used used for every action, including the login and authentication, for which an API token with no role (i.e., no scope in {ref}`available-scopes-target`) is used.
+The RBAC framework allows for granting tokens different levels of permissions via scopes attached to roles. The 'only identify' purpose of the separate OAuth tokens is no longer required. API tokens can be used for every action, including the login and authentication, for which an API token with no role (i.e., no scope in {ref}`available-scopes-target`) is used.
 
 OAuth tokens are therefore dropped from the Hub upgraded with the RBAC framework.

docs/source/reference/authenticators.md

@@ -1,6 +1,6 @@
 # Authenticators
 
-The [Authenticator][] is the mechanism for authorizing users to use the
+The {class}`.Authenticator` is the mechanism for authorizing users to use the
 Hub and single user notebook servers.
 
 ## The default PAM Authenticator
@@ -137,8 +137,8 @@ via other mechanisms. One such example is using [GitHub OAuth][].
 
 Because the username is passed from the Authenticator to the Spawner,
 a custom Authenticator and Spawner are often used together.
-For example, the Authenticator methods, [pre_spawn_start(user, spawner)][]
-and [post_spawn_stop(user, spawner)][], are hooks that can be used to do
+For example, the Authenticator methods, {meth}`.Authenticator.pre_spawn_start`
+and {meth}`.Authenticator.post_spawn_stop`, are hooks that can be used to do
 auth-related startup (e.g. opening PAM sessions) and cleanup
 (e.g. closing PAM sessions).
 
@@ -223,7 +223,7 @@ If there are multiple keys present, the **first** key is always used to persist
 
 Typically, if `auth_state` is persisted it is desirable to affect the Spawner environment in some way.
 This may mean defining environment variables, placing certificate in the user's home directory, etc.
-The `Authenticator.pre_spawn_start` method can be used to pass information from authenticator state
+The {meth}`Authenticator.pre_spawn_start` method can be used to pass information from authenticator state
 to Spawner environment:
 
 ```python
@@ -247,10 +247,42 @@ class MyAuthenticator(Authenticator):
         spawner.environment['UPSTREAM_TOKEN'] = auth_state['upstream_token']
 ```
 
+(authenticator-groups)=
+
+## Authenticator-managed group membership
+
+:::{versionadded} 2.2
+:::
+
+Some identity providers may have their own concept of group membership that you would like to preserve in JupyterHub.
+This is now possible with `Authenticator.managed_groups`.
+
+You can set the config:
+
+```python
+c.Authenticator.manage_groups = True
+```
+
+to enable this behavior.
+The default is False for Authenticators that ship with JupyterHub,
+but may be True for custom Authenticators.
+Check your Authenticator's documentation for manage_groups support.
+
+If True, {meth}`.Authenticator.authenticate` and {meth}`.Authenticator.refresh_user` may include a field `groups`
+which is a list of group names the user should be a member of:
+
+- Membership will be added for any group in the list
+- Membership in any groups not in the list will be revoked
+- Any groups not already present in the database will be created
+- If `None` is returned, no changes are made to the user's group membership
+
+If authenticator-managed groups are enabled,
+all group-management via the API is disabled.
+
 ## pre_spawn_start and post_spawn_stop hooks
 
-Authenticators uses two hooks, [pre_spawn_start(user, spawner)][] and
-[post_spawn_stop(user, spawner)][] to add pass additional state information
+Authenticators uses two hooks, {meth}`.Authenticator.pre_spawn_start` and
+{meth}`.Authenticator.post_spawn_stop(user, spawner)` to add pass additional state information
 between the authenticator and a spawner. These hooks are typically used auth-related
 startup, i.e. opening a PAM session, and auth-related cleanup, i.e. closing a
 PAM session.
@@ -259,10 +291,7 @@ PAM session.
 
 Beginning with version 0.8, JupyterHub is an OAuth provider.
 
-[authenticator]: https://github.com/jupyterhub/jupyterhub/blob/HEAD/jupyterhub/auth.py
 [pam]: https://en.wikipedia.org/wiki/Pluggable_authentication_module
 [oauth]: https://en.wikipedia.org/wiki/OAuth
 [github oauth]: https://developer.github.com/v3/oauth/
 [oauthenticator]: https://github.com/jupyterhub/oauthenticator
-[pre_spawn_start(user, spawner)]: https://jupyterhub.readthedocs.io/en/latest/api/auth.html#jupyterhub.auth.Authenticator.pre_spawn_start
-[post_spawn_stop(user, spawner)]: https://jupyterhub.readthedocs.io/en/latest/api/auth.html#jupyterhub.auth.Authenticator.post_spawn_stop

docs/source/reference/config-proxy.md

@@ -165,7 +165,7 @@ As with nginx above, you can use [Apache](https://httpd.apache.org) as the rever
 First, we will need to enable the apache modules that we are going to need:
 
 ```bash
-a2enmod ssl rewrite proxy proxy_http proxy_wstunnel
+a2enmod ssl rewrite proxy headers proxy_http proxy_wstunnel
 ```
 
 Our Apache configuration is equivalent to the nginx configuration above:
@@ -188,13 +188,24 @@ Listen 443
 
   ServerName HUB.DOMAIN.TLD
 
+  # enable HTTP/2, if available
+  Protocols h2 http/1.1
+
+  # HTTP Strict Transport Security (mod_headers is required) (63072000 seconds)
+  Header always set Strict-Transport-Security "max-age=63072000"
+
   # configure SSL
   SSLEngine on
   SSLCertificateFile /etc/letsencrypt/live/HUB.DOMAIN.TLD/fullchain.pem
   SSLCertificateKeyFile /etc/letsencrypt/live/HUB.DOMAIN.TLD/privkey.pem
-  SSLProtocol All -SSLv2 -SSLv3
   SSLOpenSSLConfCmd DHParameters /etc/ssl/certs/dhparam.pem
-  SSLCipherSuite EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH
+
+  # intermediate configuration from ssl-config.mozilla.org (2022-03-03)
+  # Please note, that this configuration might be out-dated - please update it accordingly using https://ssl-config.mozilla.org/
+  SSLProtocol             all -SSLv3 -TLSv1 -TLSv1.1
+  SSLCipherSuite          ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+  SSLHonorCipherOrder     off
+  SSLSessionTickets       off
 
   # Use RewriteEngine to handle websocket connection upgrades
   RewriteEngine On
@@ -208,6 +219,7 @@ Listen 443
     # proxy to JupyterHub
     ProxyPass         http://127.0.0.1:8000/
     ProxyPassReverse  http://127.0.0.1:8000/
+    RequestHeader     set "X-Forwarded-Proto" expr=%{REQUEST_SCHEME}
   </Location>
 </VirtualHost>
 ```
@@ -219,8 +231,8 @@ In case of the need to run the jupyterhub under /jhub/ or other location please
 httpd.conf amendments:
 
 ```bash
- RewriteRule /jhub/(.*) ws://127.0.0.1:8000/jhub/$1 [NE,P,L]
- RewriteRule /jhub/(.*) http://127.0.0.1:8000/jhub/$1 [NE,P,L]
+ RewriteRule /jhub/(.*) ws://127.0.0.1:8000/jhub/$1 [P,L]
+ RewriteRule /jhub/(.*) http://127.0.0.1:8000/jhub/$1 [P,L]
 
  ProxyPass /jhub/ http://127.0.0.1:8000/jhub/
  ProxyPassReverse /jhub/  http://127.0.0.1:8000/jhub/

docs/source/reference/rest.md

@@ -113,7 +113,6 @@ c.JupyterHub.load_roles = [
         "scopes": [
             # specify the permissions the token should have
             "admin:users",
-            "admin:services",
         ],
         "services": [
             # assign the service the above permissions

docs/source/reference/services.md

@@ -35,6 +35,8 @@ A Service may have the following properties:
   the service will be added to the proxy at `/services/:name`
 - `api_token: str (default - None)` - For Externally-Managed Services you need to specify
   an API token to perform API requests to the Hub
+- `display: bool (default - True)` - When set to true, display a link to the
+  service's URL under the 'Services' dropdown in user's hub home page.
 
 If a service is also to be managed by the Hub, it has a few extra options:
 
@@ -83,6 +85,7 @@ c.JupyterHub.load_roles = [
             # 'admin:users' # needed if culling idle users as well
         ]
     }
+]
 
 c.JupyterHub.services = [
     {
@@ -113,7 +116,10 @@ JUPYTERHUB_BASE_URL:       Base URL of the Hub (https://mydomain[:port]/)
 JUPYTERHUB_SERVICE_PREFIX: URL path prefix of this service (/services/:service-name/)
 JUPYTERHUB_SERVICE_URL:    Local URL where the service is expected to be listening.
                            Only for proxied web services.
-JUPYTERHUB_OAUTH_SCOPES:   JSON-serialized list of scopes to use for allowing access to the service.
+JUPYTERHUB_OAUTH_SCOPES:   JSON-serialized list of scopes to use for allowing access to the service
+                           (deprecated in 3.0, use JUPYTERHUB_OAUTH_ACCESS_SCOPES).
+JUPYTERHUB_OAUTH_ACCESS_SCOPES: JSON-serialized list of scopes to use for allowing access to the service (new in 3.0).
+JUPYTERHUB_OAUTH_CLIENT_ALLOWED_SCOPES: JSON-serialized list of scopes that can be requested by the oauth client on behalf of users (new in 3.0).
 ```
 
 For the previous 'cull idle' Service example, these environment variables
@@ -208,23 +214,23 @@ can be used by services. You may go beyond this reference implementation and
 create custom hub-authenticating clients and services. We describe the process
 below.
 
-The reference, or base, implementation is the [`HubAuth`][hubauth] class,
+The reference, or base, implementation is the {class}`.HubAuth` class,
 which implements the API requests to the Hub that resolve a token to a User model.
 
 There are two levels of authentication with the Hub:
 
-- [`HubAuth`][hubauth] - the most basic authentication,
+- {class}`.HubAuth` - the most basic authentication,
   for services that should only accept API requests authorized with a token.
 
-- [`HubOAuth`][huboauth] - For services that should use oauth to authenticate with the Hub.
+- {class}`.HubOAuth` - For services that should use oauth to authenticate with the Hub.
   This should be used for any service that serves pages that should be visited with a browser.
 
 To use HubAuth, you must set the `.api_token`, either programmatically when constructing the class,
 or via the `JUPYTERHUB_API_TOKEN` environment variable.
 
 Most of the logic for authentication implementation is found in the
-[`HubAuth.user_for_token`][hubauth.user_for_token]
-methods, which makes a request of the Hub, and returns:
+{meth}`.HubAuth.user_for_token` methods,
+which makes a request of the Hub, and returns:
 
 - None, if no user could be identified, or
 - a dict of the following form:
@@ -245,6 +251,19 @@ action.
 HubAuth also caches the Hub's response for a number of seconds,
 configurable by the `cookie_cache_max_age` setting (default: five minutes).
 
+If your service would like to make further requests _on behalf of users_,
+it should use the token issued by this OAuth process.
+If you are using tornado,
+you can access the token authenticating the current request with {meth}`.HubAuth.get_token`.
+
+:::{versionchanged} 2.2
+
+{meth}`.HubAuth.get_token` adds support for retrieving
+tokens stored in tornado cookies after completion of OAuth.
+Previously, it only retrieved tokens from URL parameters or the Authorization header.
+Passing `get_token(handler, in_cookie=False)` preserves this behavior.
+:::
+
 ### Flask Example
 
 For example, you have a Flask service that returns information about a user.
@@ -360,7 +379,7 @@ The `scopes` field can be used to manage access.
 Note: a user will have access to a service to complete oauth access to the service for the first time.
 Individual permissions may be revoked at any later point without revoking the token,
 in which case the `scopes` field in this model should be checked on each access.
-The default required scopes for access are available from `hub_auth.oauth_scopes` or `$JUPYTERHUB_OAUTH_SCOPES`.
+The default required scopes for access are available from `hub_auth.oauth_scopes` or `$JUPYTERHUB_OAUTH_ACCESS_SCOPES`.
 
 An example of using an Externally-Managed Service and authentication is
 in [nbviewer README][nbviewer example] section on securing the notebook viewer,
@@ -370,11 +389,6 @@ section on securing the notebook viewer.
 
 [requests]: http://docs.python-requests.org/en/master/
 [services_auth]: ../api/services.auth.html
-[hubauth]: ../api/services.auth.html#jupyterhub.services.auth.HubAuth
-[huboauth]: ../api/services.auth.html#jupyterhub.services.auth.HubOAuth
-[hubauth.user_for_token]: ../api/services.auth.html#jupyterhub.services.auth.HubAuth.user_for_token
-[hubauthenticated]: ../api/services.auth.html#jupyterhub.services.auth.HubAuthenticated
-[huboauthenticated]: ../api/services.auth.html#jupyterhub.services.auth.HubOAuthenticated
 [nbviewer example]: https://github.com/jupyter/nbviewer#securing-the-notebook-viewer
 [fastapi example]: https://github.com/jupyterhub/jupyterhub/tree/HEAD/examples/service-fastapi
 [fastapi]: https://fastapi.tiangolo.com

docs/source/reference/spawners.md

@@ -308,6 +308,9 @@ The process environment is returned by `Spawner.get_env`, which specifies the fo
   This is also the OAuth client secret.
 - JUPYTERHUB_CLIENT_ID - the OAuth client ID for authenticating visitors.
 - JUPYTERHUB_OAUTH_CALLBACK_URL - the callback URL to use in oauth, typically `/user/:name/oauth_callback`
+- JUPYTERHUB_OAUTH_ACCESS_SCOPES - the scopes required to access the server (called JUPYTERHUB_OAUTH_SCOPES prior to 3.0)
+- JUPYTERHUB_OAUTH_CLIENT_ALLOWED_SCOPES - the scopes the service is allowed to request.
+  If no scopes are requested explicitly, these scopes will be requested.
 
 Optional environment variables, depending on configuration:
 

docs/source/troubleshooting.md

@@ -275,7 +275,7 @@ where `ssl_cert` is example-chained.crt and ssl_key to your private key.
 
 Then restart JupyterHub.
 
-See also [JupyterHub SSL encryption](./getting-started/security-basics.html#ssl-encryption).
+See also {ref}`ssl-encryption`.
 
 ### Install JupyterHub without a network connection
 
@@ -371,7 +371,7 @@ a JupyterHub deployment. The commands are:
 - System and deployment information
 
 ```bash
-jupyter troubleshooting
+jupyter troubleshoot
 ```
 
 - Kernel information

examples/azuread-with-group-management/jupyterhub_config.py

@@ -0,0 +1,31 @@
+"""sample jupyterhub config file for testing
+
+configures jupyterhub with dummyauthenticator and simplespawner
+to enable testing without administrative privileges.
+"""
+
+c = get_config()  # noqa
+c.Application.log_level = 'DEBUG'
+
+import os
+
+from oauthenticator.azuread import AzureAdOAuthenticator
+
+c.JupyterHub.authenticator_class = AzureAdOAuthenticator
+
+c.AzureAdOAuthenticator.client_id = os.getenv("AAD_CLIENT_ID")
+c.AzureAdOAuthenticator.client_secret = os.getenv("AAD_CLIENT_SECRET")
+c.AzureAdOAuthenticator.oauth_callback_url = os.getenv("AAD_CALLBACK_URL")
+c.AzureAdOAuthenticator.tenant_id = os.getenv("AAD_TENANT_ID")
+c.AzureAdOAuthenticator.username_claim = "email"
+c.AzureAdOAuthenticator.authorize_url = os.getenv("AAD_AUTHORIZE_URL")
+c.AzureAdOAuthenticator.token_url = os.getenv("AAD_TOKEN_URL")
+c.Authenticator.manage_groups = True
+c.Authenticator.refresh_pre_spawn = True
+
+# Optionally set a global password that all users must use
+# c.DummyAuthenticator.password = "your_password"
+
+from jupyterhub.spawner import SimpleLocalProcessSpawner
+
+c.JupyterHub.spawner_class = SimpleLocalProcessSpawner

examples/azuread-with-group-management/requirements.txt

@@ -0,0 +1,2 @@
+oauthenticator
+pyjwt

examples/custom-scopes/grades.py

@@ -0,0 +1,132 @@
+import os
+from functools import wraps
+from html import escape
+from urllib.parse import urlparse
+
+from tornado.httpserver import HTTPServer
+from tornado.ioloop import IOLoop
+from tornado.web import Application, RequestHandler, authenticated
+
+from jupyterhub.services.auth import HubOAuthCallbackHandler, HubOAuthenticated
+from jupyterhub.utils import url_path_join
+
+SCOPE_PREFIX = "custom:grades"
+READ_SCOPE = f"{SCOPE_PREFIX}:read"
+WRITE_SCOPE = f"{SCOPE_PREFIX}:write"
+
+
+def require_scope(scopes):
+    """Decorator to require scopes
+
+    For use if multiple methods on one Handler
+    may want different scopes,
+    so class-level .hub_scopes is insufficient
+    (e.g. read for GET, write for POST).
+    """
+    if isinstance(scopes, str):
+        scopes = [scopes]
+
+    def wrap(method):
+        """The actual decorator"""
+
+        @wraps(method)
+        @authenticated
+        def wrapped(self, *args, **kwargs):
+            self.hub_scopes = scopes
+            return method(self, *args, **kwargs)
+
+        return wrapped
+
+    return wrap
+
+
+class MyGradesHandler(HubOAuthenticated, RequestHandler):
+    # no hub_scopes, anyone with access to this service
+    # will be able to visit this URL
+
+    @authenticated
+    def get(self):
+        self.write("<h1>My grade</h1>")
+        name = self.current_user["name"]
+        grades = self.settings["grades"]
+        self.write(f"<p>My name is: {escape(name)}</p>")
+        if name in grades:
+            self.write(f"<p>My grade is: {escape(str(grades[name]))}</p>")
+        else:
+            self.write("<p>No grade entered</p>")
+        if READ_SCOPE in self.current_user["scopes"]:
+            self.write('<a href="grades/">enter grades</a>')
+
+
+class GradesHandler(HubOAuthenticated, RequestHandler):
+    # default scope for this Handler: read-only
+    hub_scopes = [READ_SCOPE]
+
+    def _render(self):
+        grades = self.settings["grades"]
+        self.write("<h1>All grades</h1>")
+        self.write("<table>")
+        self.write("<tr><th>Student</th><th>Grade</th></tr>")
+        for student, grade in grades.items():
+            qstudent = escape(student)
+            qgrade = escape(str(grade))
+            self.write(
+                f"""
+                <tr>
+                 <td class="student">{qstudent}</td>
+                 <td class="grade">{qgrade}</td>
+                </tr>
+                """
+            )
+        if WRITE_SCOPE in self.current_user["scopes"]:
+            self.write("Enter grade:")
+            self.write(
+                """
+                <form action=. method=POST>
+                    <input name=student placeholder=student></input>
+                    <input kind=number name=grade placeholder=grade></input>
+                    <input type="submit" value="Submit">
+                """
+            )
+
+    @require_scope([READ_SCOPE])
+    async def get(self):
+        self._render()
+
+    # POST requires WRITE_SCOPE instead of READ_SCOPE
+    @require_scope([WRITE_SCOPE])
+    async def post(self):
+        name = self.get_argument("student")
+        grade = self.get_argument("grade")
+        self.settings["grades"][name] = grade
+        self._render()
+
+
+def main():
+    base_url = os.environ['JUPYTERHUB_SERVICE_PREFIX']
+
+    app = Application(
+        [
+            (base_url, MyGradesHandler),
+            (url_path_join(base_url, 'grades/'), GradesHandler),
+            (
+                url_path_join(base_url, 'oauth_callback'),
+                HubOAuthCallbackHandler,
+            ),
+        ],
+        cookie_secret=os.urandom(32),
+        grades={"student": 53},
+    )
+
+    http_server = HTTPServer(app)
+    url = urlparse(os.environ['JUPYTERHUB_SERVICE_URL'])
+
+    http_server.listen(url.port, url.hostname)
+    try:
+        IOLoop.current().start()
+    except KeyboardInterrupt:
+        pass
+
+
+if __name__ == '__main__':
+    main()

examples/custom-scopes/jupyterhub_config.py

@@ -0,0 +1,52 @@
+import sys
+
+c = get_config()  # noqa
+
+c.JupyterHub.services = [
+    {
+        'name': 'grades',
+        'url': 'http://127.0.0.1:10101',
+        'command': [sys.executable, './grades.py'],
+        'oauth_client_allowed_scopes': [
+            'custom:grades:write',
+            'custom:grades:read',
+        ],
+    },
+]
+
+c.JupyterHub.custom_scopes = {
+    "custom:grades:read": {
+        "description": "read-access to all grades",
+    },
+    "custom:grades:write": {
+        "description": "Enter new grades",
+        "subscopes": ["custom:grades:read"],
+    },
+}
+
+c.JupyterHub.load_roles = [
+    {
+        "name": "user",
+        # grant all users access to services
+        "scopes": ["access:services", "self"],
+    },
+    {
+        "name": "grader",
+        # grant graders access to write grades
+        "scopes": ["custom:grades:write"],
+        "users": ["grader"],
+    },
+    {
+        "name": "instructor",
+        # grant instructors access to read, but not write grades
+        "scopes": ["custom:grades:read"],
+        "users": ["instructor"],
+    },
+]
+
+c.JupyterHub.allowed_users = {"instructor", "grader", "student"}
+# dummy spawner and authenticator for testing, don't actually use these!
+c.JupyterHub.authenticator_class = 'dummy'
+c.JupyterHub.spawner_class = 'simple'
+c.JupyterHub.ip = '127.0.0.1'  # let's just run on localhost while dummy auth is enabled
+c.JupyterHub.log_level = 10

examples/external-oauth/whoami-oauth-basic.py

@@ -5,13 +5,10 @@ so all URLs and requests necessary for OAuth with JupyterHub should be in one pl
 """
 import json
 import os
-from urllib.parse import urlencode
-from urllib.parse import urlparse
+from urllib.parse import urlencode, urlparse
 
-from tornado import log
-from tornado import web
-from tornado.httpclient import AsyncHTTPClient
-from tornado.httpclient import HTTPRequest
+from tornado import log, web
+from tornado.httpclient import AsyncHTTPClient, HTTPRequest
 from tornado.httputil import url_concat
 from tornado.ioloop import IOLoop
 

examples/server-api/jupyterhub_config.py

@@ -2,6 +2,8 @@
 # 1. start/stop servers, and
 # 2. access the server API
 
+c = get_config()  # noqa
+
 c.JupyterHub.load_roles = [
     {
         "name": "launcher",

examples/server-api/start-stop-server.py

@@ -16,7 +16,6 @@ import time
 
 import requests
 
-
 log = logging.getLogger(__name__)
 
 

examples/service-announcement/announcement.py

@@ -3,9 +3,7 @@ import datetime
 import json
 import os
 
-from tornado import escape
-from tornado import ioloop
-from tornado import web
+from tornado import escape, ioloop, web
 
 from jupyterhub.services.auth import HubAuthenticated
 

examples/service-fastapi/app/models.py

@@ -1,8 +1,5 @@
 from datetime import datetime
-from typing import Any
-from typing import Dict
-from typing import List
-from typing import Optional
+from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel
 

examples/service-fastapi/app/security.py

@@ -1,9 +1,7 @@
 import json
 import os
 
-from fastapi import HTTPException
-from fastapi import Security
-from fastapi import status
+from fastapi import HTTPException, Security, status
 from fastapi.security import OAuth2AuthorizationCodeBearer
 from fastapi.security.api_key import APIKeyQuery
 

examples/service-fastapi/app/service.py

@@ -1,14 +1,9 @@
 import os
 
-from fastapi import APIRouter
-from fastapi import Depends
-from fastapi import Form
-from fastapi import Request
+from fastapi import APIRouter, Depends, Form, Request
 
 from .client import get_client
-from .models import AuthorizationError
-from .models import HubApiError
-from .models import User
+from .models import AuthorizationError, HubApiError, User
 from .security import get_current_user
 
 # APIRouter prefix cannot end in /

examples/service-whoami-flask/whoami-flask.py

@@ -7,16 +7,10 @@ import os
 import secrets
 from functools import wraps
 
-from flask import Flask
-from flask import make_response
-from flask import redirect
-from flask import request
-from flask import Response
-from flask import session
+from flask import Flask, Response, make_response, redirect, request, session
 
 from jupyterhub.services.auth import HubOAuth
 
-
 prefix = os.environ.get('JUPYTERHUB_SERVICE_PREFIX', '/')
 
 auth = HubOAuth(api_token=os.environ['JUPYTERHUB_API_TOKEN'], cache_max_age=60)

examples/service-whoami/README.md

@@ -26,7 +26,7 @@ After logging in with any username and password, you should see a JSON dump of y
 ```
 
 What is contained in the model will depend on the permissions
-requested in the `oauth_roles` configuration of the service `whoami-oauth` service.
+requested in the `oauth_client_allowed_scopes` configuration of the service `whoami-oauth` service.
 The default is the minimum required for identification and access to the service,
 which will provide the username and current scopes.
 

examples/service-whoami/jupyterhub_config.py

@@ -14,11 +14,11 @@ c.JupyterHub.services = [
         # only requesting access to the service,
         # and identification by name,
         # nothing more.
-        # Specifying 'oauth_roles' as a list of role names
+        # Specifying 'oauth_client_allowed_scopes' as a list of scopes
         # allows requesting more information about users,
         # or the ability to take actions on users' behalf, as required.
-        # The default 'token' role has the full permissions of its owner:
-        # 'oauth_roles': ['token'],
+        # the 'inherit' scope means the full permissions of the owner
+        # 'oauth_client_allowed_scopes': ['inherit'],
     },
 ]
 

examples/service-whoami/whoami-oauth.py

@@ -10,12 +10,9 @@ from urllib.parse import urlparse
 
 from tornado.httpserver import HTTPServer
 from tornado.ioloop import IOLoop
-from tornado.web import Application
-from tornado.web import authenticated
-from tornado.web import RequestHandler
+from tornado.web import Application, RequestHandler, authenticated
 
-from jupyterhub.services.auth import HubOAuthCallbackHandler
-from jupyterhub.services.auth import HubOAuthenticated
+from jupyterhub.services.auth import HubOAuthCallbackHandler, HubOAuthenticated
 from jupyterhub.utils import url_path_join
 
 

examples/service-whoami/whoami.py

@@ -10,9 +10,7 @@ from urllib.parse import urlparse
 
 from tornado.httpserver import HTTPServer
 from tornado.ioloop import IOLoop
-from tornado.web import Application
-from tornado.web import authenticated
-from tornado.web import RequestHandler
+from tornado.web import Application, RequestHandler, authenticated
 
 from jupyterhub.services.auth import HubAuthenticated
 

jsx/build/admin-react.js.LICENSE.txt

@@ -1,47 +0,0 @@
-/*
-object-assign
-(c) Sindre Sorhus
-@license MIT
-*/
-
-/*!
-  Copyright (c) 2017 Jed Watson.
-  Licensed under the MIT License (MIT), see
-  http://jedwatson.github.io/classnames
-*/
-
-/** @license React v0.20.1
- * scheduler.production.min.js
- *
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-/** @license React v16.13.1
- * react-is.production.min.js
- *
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-/** @license React v17.0.1
- * react-dom.production.min.js
- *
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-/** @license React v17.0.1
- * react.production.min.js
- *
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */

jsx/build/index.html

@@ -1,6 +0,0 @@
-<!DOCTYPE html>
-<head></head>
-<body>
-  <div id="admin-react-hook"></div>
-  <script src="admin-react.js"></script>
-</body>

jsx/package.json

@@ -8,7 +8,7 @@
   "scripts": {
     "build": "yarn && webpack",
     "hot": "webpack && webpack-dev-server",
-    "place": "cp -r build/admin-react.js ../share/jupyterhub/static/js/admin-react.js",
+    "place": "cp build/admin-react.js* ../share/jupyterhub/static/js/",
     "test": "jest --verbose",
     "snap": "jest --updateSnapshot",
     "lint": "eslint --ext .jsx --ext .js src/",
@@ -28,43 +28,48 @@
     }
   },
   "dependencies": {
+    "bootstrap": "^4.5.3",
+    "history": "^5.0.0",
+    "lodash.debounce": "^4.0.8",
+    "prop-types": "^15.7.2",
+    "react": "^17.0.1",
+    "react-bootstrap": "^2.1.1",
+    "react-dom": "^17.0.1",
+    "react-icons": "^4.1.0",
+    "react-multi-select-component": "^3.0.7",
+    "react-object-table-viewer": "^1.0.7",
+    "react-redux": "^7.2.2",
+    "react-router": "^5.2.0",
+    "react-router-dom": "^5.2.0",
+    "recompose": "npm:react-recompose@^0.31.2",
+    "redux": "^4.0.5",
+    "regenerator-runtime": "^0.13.9"
+  },
+  "devDependencies": {
     "@babel/core": "^7.12.3",
     "@babel/preset-env": "^7.12.11",
     "@babel/preset-react": "^7.12.10",
     "@testing-library/jest-dom": "^5.15.1",
     "@testing-library/react": "^12.1.2",
     "@testing-library/user-event": "^13.5.0",
-    "babel-loader": "^8.2.1",
-    "bootstrap": "^4.5.3",
-    "css-loader": "^5.0.1",
-    "eslint-plugin-unused-imports": "^1.1.1",
-    "file-loader": "^6.2.0",
-    "history": "^5.0.0",
-    "prop-types": "^15.7.2",
-    "react": "^17.0.1",
-    "react-bootstrap": "^1.4.0",
-    "react-dom": "^17.0.1",
-    "react-icons": "^4.1.0",
-    "react-multi-select-component": "^3.0.7",
-    "react-redux": "^7.2.2",
-    "react-router": "^5.2.0",
-    "react-router-dom": "^5.2.0",
-    "recompose": "^0.30.0",
-    "redux": "^4.0.5",
-    "style-loader": "^2.0.0",
-    "webpack": "^5.6.0",
-    "webpack-cli": "^3.3.4",
-    "webpack-dev-server": "^3.11.0"
-  },
-  "devDependencies": {
+    "@webpack-cli/serve": "^1.7.0",
     "@wojtekmaj/enzyme-adapter-react-17": "^0.6.5",
     "babel-jest": "^26.6.3",
+    "babel-loader": "^8.2.1",
+    "css-loader": "^5.0.1",
     "enzyme": "^3.11.0",
     "eslint": "^7.18.0",
     "eslint-plugin-prettier": "^3.3.1",
     "eslint-plugin-react": "^7.22.0",
+    "eslint-plugin-unused-imports": "^1.1.1",
+    "file-loader": "^6.2.0",
     "identity-obj-proxy": "^3.0.0",
     "jest": "^26.6.3",
-    "prettier": "^2.2.1"
+    "prettier": "^2.2.1",
+    "sinon": "^13.0.1",
+    "style-loader": "^2.0.0",
+    "webpack": "^5.6.0",
+    "webpack-cli": "^4.10.0",
+    "webpack-dev-server": "^4.9.3"
   }
 }

jsx/src/App.jsx

@@ -22,15 +22,16 @@ const store = createStore(reducers, initialState);
 const App = () => {
   useEffect(() => {
     let { limit, user_page, groups_page } = initialState;
-    jhapiRequest(`/users?offset=${user_page * limit}&limit=${limit}`, "GET")
-      .then((data) => data.json())
+    let api = withAPI()().props;
+    api
+      .updateUsers(user_page * limit, limit)
       .then((data) =>
         store.dispatch({ type: "USER_PAGE", value: { data: data, page: 0 } })
       )
       .catch((err) => console.log(err));
 
-    jhapiRequest(`/groups?offset=${groups_page * limit}&limit=${limit}`, "GET")
-      .then((data) => data.json())
+    api
+      .updateGroups(groups_page * limit, limit)
       .then((data) =>
         store.dispatch({ type: "GROUPS_PAGE", value: { data: data, page: 0 } })
       )

jsx/src/Store.js

@@ -1,6 +1,7 @@
 export const initialState = {
   user_data: undefined,
   user_page: 0,
+  name_filter: "",
   groups_data: undefined,
   groups_page: 0,
   limit: window.api_page_limit,
@@ -13,6 +14,7 @@ export const reducers = (state = initialState, action) => {
       return Object.assign({}, state, {
         user_page: action.value.page,
         user_data: action.value.data,
+        name_filter: action.value.name_filter || "",
       });
 
     // Updates the client group model data and stores the page

jsx/src/components/AddUser/AddUser.jsx

@@ -60,7 +60,10 @@ const AddUser = (props) => {
                       placeholder="usernames separated by line"
                       data-testid="user-textarea"
                       onBlur={(e) => {
-                        let split_users = e.target.value.split("\n");
+                        let split_users = e.target.value
+                          .split("\n")
+                          .map((u) => u.trim())
+                          .filter((u) => u.length > 0);
                         setUsers(split_users);
                       }}
                     ></textarea>
@@ -88,17 +91,7 @@ const AddUser = (props) => {
                   data-testid="submit"
                   className="btn btn-primary"
                   onClick={() => {
-                    let filtered_users = users.filter(
-                      (e) =>
-                        e.length > 2 &&
-                        /[!@#$%^&*(),.?":{}|<>]/g.test(e) == false
-                    );
-                    if (filtered_users.length < users.length) {
-                      setUsers(filtered_users);
-                      failRegexEvent();
-                    }
-
-                    addUsers(filtered_users, admin)
+                    addUsers(users, admin)
                       .then((data) =>
                         data.status < 300
                           ? updateUsers(0, limit)

jsx/src/components/AddUser/AddUser.test.js

@@ -70,12 +70,12 @@ test("Removes users when they fail Regex", async () => {
   let textarea = screen.getByTestId("user-textarea");
   let submit = screen.getByTestId("submit");
 
-  fireEvent.blur(textarea, { target: { value: "foo\nbar\n!!*&*" } });
+  fireEvent.blur(textarea, { target: { value: "foo \n bar\na@b.co\n  \n\n" } });
   await act(async () => {
     fireEvent.click(submit);
   });
 
-  expect(callbackSpy).toHaveBeenCalledWith(["foo", "bar"], false);
+  expect(callbackSpy).toHaveBeenCalledWith(["foo", "bar", "a@b.co"], false);
 });
 
 test("Correctly submits admin", async () => {

jsx/src/components/CreateGroup/CreateGroup.jsx

@@ -59,7 +59,7 @@ const CreateGroup = (props) => {
                     value={groupName}
                     placeholder="group name..."
                     onChange={(e) => {
-                      setGroupName(e.target.value);
+                      setGroupName(e.target.value.trim());
                     }}
                   ></input>
                 </div>

jsx/src/components/ServerDashboard/ServerDashboard.jsx

@@ -1,8 +1,19 @@
 import React, { useState } from "react";
+import regeneratorRuntime from "regenerator-runtime";
 import { useSelector, useDispatch } from "react-redux";
 import PropTypes from "prop-types";
 
-import { Button } from "react-bootstrap";
+import {
+  Button,
+  Col,
+  Row,
+  FormControl,
+  Card,
+  CardGroup,
+  Collapse,
+} from "react-bootstrap";
+import ReactObjectTableViewer from "react-object-table-viewer";
+
 import { Link } from "react-router-dom";
 import { FaSort, FaSortUp, FaSortDown } from "react-icons/fa";
 
@@ -10,7 +21,16 @@ import "./server-dashboard.css";
 import { timeSince } from "../../util/timeSince";
 import PaginationFooter from "../PaginationFooter/PaginationFooter";
 
+const AccessServerButton = ({ url }) => (
+  <a href={url || ""}>
+    <button className="btn btn-primary btn-xs" style={{ marginRight: 20 }}>
+      Access Server
+    </button>
+  </a>
+);
+
 const ServerDashboard = (props) => {
+  let base_url = window.base_url || "/";
   // sort methods
   var usernameDesc = (e) => e.sort((a, b) => (a.name > b.name ? 1 : -1)),
     usernameAsc = (e) => e.sort((a, b) => (a.name < b.name ? 1 : -1)),
@@ -29,14 +49,17 @@ const ServerDashboard = (props) => {
 
   var [errorAlert, setErrorAlert] = useState(null);
   var [sortMethod, setSortMethod] = useState(null);
+  var [disabledButtons, setDisabledButtons] = useState({});
+  const [collapseStates, setCollapseStates] = useState({});
 
   var user_data = useSelector((state) => state.user_data),
     user_page = useSelector((state) => state.user_page),
     limit = useSelector((state) => state.limit),
+    name_filter = useSelector((state) => state.name_filter),
     page = parseInt(new URLSearchParams(props.location.search).get("page"));
 
   page = isNaN(page) ? 0 : page;
-  var slice = [page * limit, limit];
+  var slice = [page * limit, limit, name_filter];
 
   const dispatch = useDispatch();
 
@@ -50,12 +73,13 @@ const ServerDashboard = (props) => {
     history,
   } = props;
 
-  var dispatchPageUpdate = (data, page) => {
+  var dispatchPageUpdate = (data, page, name_filter) => {
     dispatch({
       type: "USER_PAGE",
       value: {
         data: data,
         page: page,
+        name_filter: name_filter,
       },
     });
   };
@@ -65,13 +89,258 @@ const ServerDashboard = (props) => {
   }
 
   if (page != user_page) {
-    updateUsers(...slice).then((data) => dispatchPageUpdate(data, page));
+    updateUsers(...slice).then((data) =>
+      dispatchPageUpdate(data, page, name_filter)
+    );
   }
 
+  var debounce = require("lodash.debounce");
+  const handleSearch = debounce(async (event) => {
+    // setNameFilter(event.target.value);
+    updateUsers(page * limit, limit, event.target.value).then((data) =>
+      dispatchPageUpdate(data, page, name_filter)
+    );
+  }, 300);
+
   if (sortMethod != null) {
     user_data = sortMethod(user_data);
   }
 
+  const StopServerButton = ({ serverName, userName }) => {
+    var [isDisabled, setIsDisabled] = useState(false);
+    return (
+      <button
+        className="btn btn-danger btn-xs stop-button"
+        disabled={isDisabled}
+        onClick={() => {
+          setIsDisabled(true);
+          stopServer(userName, serverName)
+            .then((res) => {
+              if (res.status < 300) {
+                updateUsers(...slice)
+                  .then((data) => {
+                    dispatchPageUpdate(data, page, name_filter);
+                  })
+                  .catch(() => {
+                    setIsDisabled(false);
+                    setErrorAlert(`Failed to update users list.`);
+                  });
+              } else {
+                setErrorAlert(`Failed to stop server.`);
+                setIsDisabled(false);
+              }
+              return res;
+            })
+            .catch(() => {
+              setErrorAlert(`Failed to stop server.`);
+              setIsDisabled(false);
+            });
+        }}
+      >
+        Stop Server
+      </button>
+    );
+  };
+
+  const StartServerButton = ({ serverName, userName }) => {
+    var [isDisabled, setIsDisabled] = useState(false);
+    return (
+      <button
+        className="btn btn-success btn-xs start-button"
+        disabled={isDisabled}
+        onClick={() => {
+          setIsDisabled(true);
+          startServer(userName, serverName)
+            .then((res) => {
+              if (res.status < 300) {
+                updateUsers(...slice)
+                  .then((data) => {
+                    dispatchPageUpdate(data, page, name_filter);
+                  })
+                  .catch(() => {
+                    setErrorAlert(`Failed to update users list.`);
+                    setIsDisabled(false);
+                  });
+              } else {
+                setErrorAlert(`Failed to start server.`);
+                setIsDisabled(false);
+              }
+              return res;
+            })
+            .catch(() => {
+              setErrorAlert(`Failed to start server.`);
+              setIsDisabled(false);
+            });
+        }}
+      >
+        Start Server
+      </button>
+    );
+  };
+
+  const EditUserCell = ({ user }) => {
+    return (
+      <td>
+        <button
+          className="btn btn-primary btn-xs"
+          style={{ marginRight: 20 }}
+          onClick={() =>
+            history.push({
+              pathname: "/edit-user",
+              state: {
+                username: user.name,
+                has_admin: user.admin,
+              },
+            })
+          }
+        >
+          Edit User
+        </button>
+      </td>
+    );
+  };
+
+  const ServerRowTable = ({ data }) => {
+    const sortedData = Object.keys(data)
+      .sort()
+      .reduce(function (result, key) {
+        let value = data[key];
+        switch (key) {
+          case "last_activity":
+          case "created":
+          case "started":
+            // format timestamps
+            value = value ? timeSince(value) : value;
+            break;
+        }
+        if (Array.isArray(value)) {
+          // cast arrays (e.g. roles, groups) to string
+          value = value.sort().join(", ");
+        }
+        result[key] = value;
+        return result;
+      }, {});
+    return (
+      <ReactObjectTableViewer
+        className="table-striped table-bordered"
+        style={{
+          padding: "3px 6px",
+          margin: "auto",
+        }}
+        keyStyle={{
+          padding: "4px",
+        }}
+        valueStyle={{
+          padding: "4px",
+        }}
+        data={sortedData}
+      />
+    );
+  };
+
+  const serverRow = (user, server) => {
+    const { servers, ...userNoServers } = user;
+    const serverNameDash = server.name ? `-${server.name}` : "";
+    const userServerName = user.name + serverNameDash;
+    const open = collapseStates[userServerName] || false;
+    return [
+      <tr key={`${userServerName}-row`} className="user-row">
+        <td data-testid="user-row-name">
+          <span>
+            <Button
+              onClick={() =>
+                setCollapseStates({
+                  ...collapseStates,
+                  [userServerName]: !open,
+                })
+              }
+              aria-controls={`${userServerName}-collapse`}
+              aria-expanded={open}
+              data-testid={`${userServerName}-collapse-button`}
+              variant={open ? "secondary" : "primary"}
+              size="sm"
+            >
+              <span className="caret"></span>
+            </Button>{" "}
+          </span>
+          <span data-testid={`user-name-div-${userServerName}`}>
+            {user.name}
+          </span>
+        </td>
+        <td data-testid="user-row-admin">{user.admin ? "admin" : ""}</td>
+
+        <td data-testid="user-row-server">
+          <p className="text-secondary">{server.name}</p>
+        </td>
+        <td data-testid="user-row-last-activity">
+          {server.last_activity ? timeSince(server.last_activity) : "Never"}
+        </td>
+        <td data-testid="user-row-server-activity">
+          {server.started ? (
+            // Stop Single-user server
+            <>
+              <StopServerButton serverName={server.name} userName={user.name} />
+              <AccessServerButton url={server.url} />
+            </>
+          ) : (
+            // Start Single-user server
+            <>
+              <StartServerButton
+                serverName={server.name}
+                userName={user.name}
+                style={{ marginRight: 20 }}
+              />
+              <a
+                href={`${base_url}spawn/${user.name}${
+                  server.name ? "/" + server.name : ""
+                }`}
+              >
+                <button
+                  className="btn btn-secondary btn-xs"
+                  style={{ marginRight: 20 }}
+                >
+                  Spawn Page
+                </button>
+              </a>
+            </>
+          )}
+        </td>
+        <EditUserCell user={user} />
+      </tr>,
+      <tr>
+        <td
+          colSpan={6}
+          style={{ padding: 0 }}
+          data-testid={`${userServerName}-td`}
+        >
+          <Collapse in={open} data-testid={`${userServerName}-collapse`}>
+            <CardGroup
+              id={`${userServerName}-card-group`}
+              style={{ width: "100%", margin: "0 auto", float: "none" }}
+            >
+              <Card style={{ width: "100%", padding: 3, margin: "0 auto" }}>
+                <Card.Title>User</Card.Title>
+                <ServerRowTable data={userNoServers} />
+              </Card>
+              <Card style={{ width: "100%", padding: 3, margin: "0 auto" }}>
+                <Card.Title>Server</Card.Title>
+                <ServerRowTable data={server} />
+              </Card>
+            </CardGroup>
+          </Collapse>
+        </td>
+      </tr>,
+    ];
+  };
+
+  let servers = user_data.flatMap((user) => {
+    let userServers = Object.values({
+      "": user.server || {},
+      ...(user.servers || {}),
+    });
+    return userServers.map((server) => [user, server]);
+  });
+
   return (
     <div className="container" data-testid="container">
       {errorAlert != null ? (
@@ -92,11 +361,24 @@ const ServerDashboard = (props) => {
       ) : (
         <></>
       )}
-      <div className="manage-groups" style={{ float: "right", margin: "20px" }}>
-        <Link to="/groups">{"> Manage Groups"}</Link>
-      </div>
       <div className="server-dashboard-container">
-        <table className="table table-striped table-bordered table-hover">
+        <Row>
+          <Col md={4}>
+            <FormControl
+              type="text"
+              name="user_search"
+              placeholder="Search users"
+              aria-label="user-search"
+              defaultValue={name_filter}
+              onChange={handleSearch}
+            />
+          </Col>
+
+          <Col md="auto" style={{ float: "right", margin: 15 }}>
+            <Link to="/groups">{"> Manage Groups"}</Link>
+          </Col>
+        </Row>
+        <table className="table table-bordered table-hover">
           <thead className="admin-table-head">
             <tr>
               <th id="user-header">
@@ -115,6 +397,14 @@ const ServerDashboard = (props) => {
                   testid="admin-sort"
                 />
               </th>
+              <th id="server-header">
+                Server{" "}
+                <SortHandler
+                  sorts={{ asc: usernameAsc, desc: usernameDesc }}
+                  callback={(method) => setSortMethod(() => method)}
+                  testid="server-sort"
+                />
+              </th>
               <th id="last-activity-header">
                 Last Activity{" "}
                 <SortHandler
@@ -167,7 +457,7 @@ const ServerDashboard = (props) => {
                       .then((res) => {
                         updateUsers(...slice)
                           .then((data) => {
-                            dispatchPageUpdate(data, page);
+                            dispatchPageUpdate(data, page, name_filter);
                           })
                           .catch(() =>
                             setErrorAlert(`Failed to update users list.`)
@@ -203,7 +493,7 @@ const ServerDashboard = (props) => {
                       .then((res) => {
                         updateUsers(...slice)
                           .then((data) => {
-                            dispatchPageUpdate(data, page);
+                            dispatchPageUpdate(data, page, name_filter);
                           })
                           .catch(() =>
                             setErrorAlert(`Failed to update users list.`)
@@ -227,88 +517,7 @@ const ServerDashboard = (props) => {
                 </Button>
               </td>
             </tr>
-            {user_data.map((e, i) => (
-              <tr key={i + "row"} className="user-row">
-                <td data-testid="user-row-name">{e.name}</td>
-                <td data-testid="user-row-admin">{e.admin ? "admin" : ""}</td>
-                <td data-testid="user-row-last-activity">
-                  {e.last_activity ? timeSince(e.last_activity) : "Never"}
-                </td>
-                <td data-testid="user-row-server-activity">
-                  {e.server != null ? (
-                    // Stop Single-user server
-                    <button
-                      className="btn btn-danger btn-xs stop-button"
-                      onClick={() =>
-                        stopServer(e.name)
-                          .then((res) => {
-                            if (res.status < 300) {
-                              updateUsers(...slice)
-                                .then((data) => {
-                                  dispatchPageUpdate(data, page);
-                                })
-                                .catch(() =>
-                                  setErrorAlert(`Failed to update users list.`)
-                                );
-                            } else {
-                              setErrorAlert(`Failed to stop server.`);
-                            }
-                            return res;
-                          })
-                          .catch(() => setErrorAlert(`Failed to stop server.`))
-                      }
-                    >
-                      Stop Server
-                    </button>
-                  ) : (
-                    // Start Single-user server
-                    <button
-                      className="btn btn-primary btn-xs start-button"
-                      onClick={() =>
-                        startServer(e.name)
-                          .then((res) => {
-                            if (res.status < 300) {
-                              updateUsers(...slice)
-                                .then((data) => {
-                                  dispatchPageUpdate(data, page);
-                                })
-                                .catch(() =>
-                                  setErrorAlert(`Failed to update users list.`)
-                                );
-                            } else {
-                              setErrorAlert(`Failed to start server.`);
-                            }
-                            return res;
-                          })
-                          .catch(() => {
-                            setErrorAlert(`Failed to start server.`);
-                          })
-                      }
-                    >
-                      Start Server
-                    </button>
-                  )}
-                </td>
-                <td>
-                  {/* Edit User */}
-                  <button
-                    className="btn btn-primary btn-xs"
-                    style={{ marginRight: 20 }}
-                    onClick={() =>
-                      history.push({
-                        pathname: "/edit-user",
-                        state: {
-                          username: e.name,
-                          has_admin: e.admin,
-                        },
-                      })
-                    }
-                  >
-                    edit user
-                  </button>
-                </td>
-              </tr>
-            ))}
+            {servers.flatMap(([user, server]) => serverRow(user, server))}
           </tbody>
         </table>
         <PaginationFooter

jsx/src/components/ServerDashboard/ServerDashboard.test.js

@@ -1,6 +1,7 @@
 import React from "react";
 import "@testing-library/jest-dom";
 import { act } from "react-dom/test-utils";
+import userEvent from "@testing-library/user-event";
 import { render, screen, fireEvent } from "@testing-library/react";
 import { HashRouter, Switch } from "react-router-dom";
 import { Provider, useSelector } from "react-redux";
@@ -9,6 +10,9 @@ import { createStore } from "redux";
 import regeneratorRuntime from "regenerator-runtime";
 
 import ServerDashboard from "./ServerDashboard";
+import * as sinon from "sinon";
+
+let clock;
 
 jest.mock("react-redux", () => ({
   ...jest.requireActual("react-redux"),
@@ -45,6 +49,7 @@ var mockAppState = () => ({
 });
 
 beforeEach(() => {
+  clock = sinon.useFakeTimers();
   useSelector.mockImplementation((callback) => {
     return callback(mockAppState());
   });
@@ -52,6 +57,7 @@ beforeEach(() => {
 
 afterEach(() => {
   useSelector.mockClear();
+  clock.restore();
 });
 
 test("Renders", async () => {
@@ -71,8 +77,8 @@ test("Renders users from props.user_data into table", async () => {
     render(serverDashboardJsx(callbackSpy));
   });
 
-  let foo = screen.getByText("foo");
-  let bar = screen.getByText("bar");
+  let foo = screen.getByTestId("user-name-div-foo");
+  let bar = screen.getByTestId("user-name-div-bar");
 
   expect(foo).toBeVisible();
   expect(bar).toBeVisible();
@@ -92,6 +98,18 @@ test("Renders correctly the status of a single-user server", async () => {
   expect(stop).toBeVisible();
 });
 
+test("Renders spawn page link", async () => {
+  let callbackSpy = mockAsync();
+
+  await act(async () => {
+    render(serverDashboardJsx(callbackSpy));
+  });
+
+  let link = screen.getByText("Spawn Page").closest("a");
+  let url = new URL(link.href);
+  expect(url.pathname).toEqual("/spawn/bar");
+});
+
 test("Invokes the startServer event on button click", async () => {
   let callbackSpy = mockAsync();
 
@@ -151,12 +169,12 @@ test("Sorts according to username", async () => {
   fireEvent.click(handler);
 
   let first = screen.getAllByTestId("user-row-name")[0];
-  expect(first.textContent).toBe("bar");
+  expect(first.textContent).toContain("bar");
 
   fireEvent.click(handler);
 
   first = screen.getAllByTestId("user-row-name")[0];
-  expect(first.textContent).toBe("foo");
+  expect(first.textContent).toContain("foo");
 });
 
 test("Sorts according to admin", async () => {
@@ -189,12 +207,12 @@ test("Sorts according to last activity", async () => {
   fireEvent.click(handler);
 
   let first = screen.getAllByTestId("user-row-name")[0];
-  expect(first.textContent).toBe("foo");
+  expect(first.textContent).toContain("foo");
 
   fireEvent.click(handler);
 
   first = screen.getAllByTestId("user-row-name")[0];
-  expect(first.textContent).toBe("bar");
+  expect(first.textContent).toContain("bar");
 });
 
 test("Sorts according to server status (running/not running)", async () => {
@@ -208,12 +226,53 @@ test("Sorts according to server status (running/not running)", async () => {
   fireEvent.click(handler);
 
   let first = screen.getAllByTestId("user-row-name")[0];
-  expect(first.textContent).toBe("foo");
+  expect(first.textContent).toContain("foo");
 
   fireEvent.click(handler);
 
   first = screen.getAllByTestId("user-row-name")[0];
-  expect(first.textContent).toBe("bar");
+  expect(first.textContent).toContain("bar");
+});
+
+test("Shows server details with button click", async () => {
+  let callbackSpy = mockAsync();
+
+  await act(async () => {
+    render(serverDashboardJsx(callbackSpy));
+  });
+  let button = screen.getByTestId("foo-collapse-button");
+  let collapse = screen.getByTestId("foo-collapse");
+  let collapseBar = screen.getByTestId("bar-collapse");
+
+  // expect().toBeVisible does not work here with collapse.
+  expect(collapse).toHaveClass("collapse");
+  expect(collapse).not.toHaveClass("show");
+  expect(collapseBar).not.toHaveClass("show");
+
+  await act(async () => {
+    fireEvent.click(button);
+  });
+  clock.tick(400);
+
+  expect(collapse).toHaveClass("collapse show");
+  expect(collapseBar).not.toHaveClass("show");
+
+  await act(async () => {
+    fireEvent.click(button);
+  });
+  clock.tick(400);
+
+  expect(collapse).toHaveClass("collapse");
+  expect(collapse).not.toHaveClass("show");
+  expect(collapseBar).not.toHaveClass("show");
+
+  await act(async () => {
+    fireEvent.click(button);
+  });
+  clock.tick(400);
+
+  expect(collapse).toHaveClass("collapse show");
+  expect(collapseBar).not.toHaveClass("show");
 });
 
 test("Renders nothing if required data is not available", async () => {
@@ -435,3 +494,42 @@ test("Shows a UI error dialogue when stop user server returns an improper status
 
   expect(errorDialog).toBeVisible();
 });
+
+test("Search for user calls updateUsers with name filter", async () => {
+  let spy = mockAsync();
+  let mockUpdateUsers = jest.fn((offset, limit, name_filter) => {
+    return Promise.resolve([]);
+  });
+  await act(async () => {
+    render(
+      <Provider store={createStore(() => {}, {})}>
+        <HashRouter>
+          <Switch>
+            <ServerDashboard
+              updateUsers={mockUpdateUsers}
+              shutdownHub={spy}
+              startServer={spy}
+              stopServer={spy}
+              startAll={spy}
+              stopAll={spy}
+            />
+          </Switch>
+        </HashRouter>
+      </Provider>
+    );
+  });
+
+  let search = screen.getByLabelText("user-search");
+
+  userEvent.type(search, "a");
+  expect(search.value).toEqual("a");
+  clock.tick(400);
+  expect(mockUpdateUsers.mock.calls[1][2]).toEqual("a");
+  expect(mockUpdateUsers.mock.calls).toHaveLength(2);
+
+  userEvent.type(search, "b");
+  expect(search.value).toEqual("ab");
+  clock.tick(400);
+  expect(mockUpdateUsers.mock.calls[2][2]).toEqual("ab");
+  expect(mockUpdateUsers.mock.calls).toHaveLength(3);
+});

jsx/src/util/jhapiUtil.js

@@ -1,5 +1,5 @@
 export const jhapiRequest = (endpoint, method, data) => {
-  let base_url = window.base_url,
+  let base_url = window.base_url || "/",
     api_url = `${base_url}hub/api`;
   return fetch(api_url + endpoint, {
     method: method,

jsx/src/util/withAPI.js

@@ -2,17 +2,22 @@ import { withProps } from "recompose";
 import { jhapiRequest } from "./jhapiUtil";
 
 const withAPI = withProps(() => ({
-  updateUsers: (offset, limit) =>
-    jhapiRequest(`/users?offset=${offset}&limit=${limit}`, "GET").then((data) =>
-      data.json()
-    ),
+  updateUsers: (offset, limit, name_filter) =>
+    jhapiRequest(
+      `/users?include_stopped_servers&offset=${offset}&limit=${limit}&name_filter=${
+        name_filter || ""
+      }`,
+      "GET"
+    ).then((data) => data.json()),
   updateGroups: (offset, limit) =>
     jhapiRequest(`/groups?offset=${offset}&limit=${limit}`, "GET").then(
       (data) => data.json()
     ),
   shutdownHub: () => jhapiRequest("/shutdown", "POST"),
-  startServer: (name) => jhapiRequest("/users/" + name + "/server", "POST"),
-  stopServer: (name) => jhapiRequest("/users/" + name + "/server", "DELETE"),
+  startServer: (name, serverName = "") =>
+    jhapiRequest("/users/" + name + "/servers/" + (serverName || ""), "POST"),
+  stopServer: (name, serverName = "") =>
+    jhapiRequest("/users/" + name + "/servers/" + (serverName || ""), "DELETE"),
   startAll: (names) =>
     names.map((e) => jhapiRequest("/users/" + e + "/server", "POST")),
   stopAll: (names) =>

jsx/webpack.config.js

@@ -1,6 +1,5 @@
 const webpack = require("webpack");
 const path = require("path");
-const express = require("express");
 
 module.exports = {
   entry: path.resolve(__dirname, "src", "App.jsx"),
@@ -34,16 +33,19 @@ module.exports = {
   },
   plugins: [new webpack.HotModuleReplacementPlugin()],
   devServer: {
-    contentBase: path.resolve(__dirname, "build"),
+    static: {
+      directory: path.resolve(__dirname, "build"),
+    },
     port: 9000,
-    before: (app, server) => {
+    onBeforeSetupMiddleware: (devServer) => {
+      const app = devServer.app;
+
       var user_data = JSON.parse(
         '[{"kind":"user","name":"foo","admin":true,"groups":[],"server":"/user/foo/","pending":null,"created":"2020-12-07T18:46:27.112695Z","last_activity":"2020-12-07T21:00:33.336354Z","servers":{"":{"name":"","last_activity":"2020-12-07T20:58:02.437408Z","started":"2020-12-07T20:58:01.508266Z","pending":null,"ready":true,"state":{"pid":28085},"url":"/user/foo/","user_options":{},"progress_url":"/hub/api/users/foo/server/progress"}}},{"kind":"user","name":"bar","admin":false,"groups":[],"server":null,"pending":null,"created":"2020-12-07T18:46:27.115528Z","last_activity":"2020-12-07T20:43:51.013613Z","servers":{}}]'
       );
       var group_data = JSON.parse(
         '[{"kind":"group","name":"testgroup","users":[]}, {"kind":"group","name":"testgroup2","users":["foo", "bar"]}]'
       );
-      app.use(express.json());
 
       // get user_data
       app.get("/hub/api/users", (req, res) => {

jupyterhub/__init__.py

@@ -1,2 +1 @@
-from ._version import __version__
-from ._version import version_info
+from ._version import __version__, version_info

jupyterhub/_data.py

@@ -4,7 +4,7 @@
 def get_data_files():
     """Walk up until we find share/jupyterhub"""
     import sys
-    from os.path import join, abspath, dirname, exists, split
+    from os.path import abspath, dirname, exists, join, split
 
     path = abspath(dirname(__file__))
     starting_points = [path]

jupyterhub/_memoize.py

@@ -0,0 +1,154 @@
+"""Utilities for memoization
+
+Note: a memoized function should always return an _immutable_
+result to avoid later modifications polluting cached results.
+"""
+from collections import OrderedDict
+from functools import wraps
+
+
+class DoNotCache:
+    """Wrapper to return a result without caching it.
+
+    In a function decorated with `@lru_cache_key`:
+
+        return DoNotCache(result)
+
+    is equivalent to:
+
+        return result  # but don't cache it!
+    """
+
+    def __init__(self, result):
+        self.result = result
+
+
+class LRUCache:
+    """A simple Least-Recently-Used (LRU) cache with a max size"""
+
+    def __init__(self, maxsize=1024):
+        self._cache = OrderedDict()
+        self.maxsize = maxsize
+
+    def __contains__(self, key):
+        return key in self._cache
+
+    def get(self, key, default=None):
+        """Get an item from the cache"""
+        if key in self._cache:
+            # cache hit, bump to front of the queue for LRU
+            result = self._cache[key]
+            self._cache.move_to_end(key)
+            return result
+        return default
+
+    def set(self, key, value):
+        """Store an entry in the cache
+
+        Purges oldest entry if cache is full
+        """
+        self._cache[key] = value
+        # cache is full, purge oldest entry
+        if len(self._cache) > self.maxsize:
+            self._cache.popitem(last=False)
+
+    __getitem__ = get
+    __setitem__ = set
+
+
+def lru_cache_key(key_func, maxsize=1024):
+    """Like functools.lru_cache, but takes a custom key function,
+    as seen in sorted(key=func).
+
+    Useful for non-hashable arguments which have a known hashable equivalent (e.g. sets, lists),
+    or mutable objects where only immutable fields might be used
+    (e.g. User, where only username affects output).
+
+    For safety: Cached results should always be immutable,
+    such as using `frozenset` instead of mutable `set`.
+
+    Example:
+
+        @lru_cache_key(lambda user: user.name)
+        def func_user(user):
+            # output only varies by name
+
+    Args:
+        key (callable):
+            Should have the same signature as the decorated function.
+            Returns a hashable key to use in the cache
+        maxsize (int):
+            The maximum size of the cache.
+    """
+
+    def cache_func(func):
+        cache = LRUCache(maxsize=maxsize)
+        # the actual decorated function:
+        @wraps(func)
+        def cached(*args, **kwargs):
+            cache_key = key_func(*args, **kwargs)
+            if cache_key in cache:
+                # cache hit
+                return cache[cache_key]
+            else:
+                # cache miss, call function and cache result
+                result = func(*args, **kwargs)
+                if isinstance(result, DoNotCache):
+                    # DoNotCache prevents caching
+                    result = result.result
+                else:
+                    cache[cache_key] = result
+            return result
+
+        return cached
+
+    return cache_func
+
+
+class FrozenDict(dict):
+    """A frozen dictionary subclass
+
+    Immutable and hashable, so it can be used as a cache key
+
+    Values will be frozen with `.freeze(value)`
+    and must be hashable after freezing.
+
+    Not rigorous, but enough for our purposes.
+    """
+
+    _hash = None
+
+    def __init__(self, d):
+        dict_set = dict.__setitem__
+        for key, value in d.items():
+            dict.__setitem__(self, key, self._freeze(value))
+
+    def _freeze(self, item):
+        """Make values of a dict hashable
+        - list, set -> frozenset
+        - dict -> recursive _FrozenDict
+        - anything else: assumed hashable
+        """
+        if isinstance(item, FrozenDict):
+            return item
+        elif isinstance(item, list):
+            return tuple(self._freeze(e) for e in item)
+        elif isinstance(item, set):
+            return frozenset(item)
+        elif isinstance(item, dict):
+            return FrozenDict(item)
+        else:
+            # any other type is assumed hashable
+            return item
+
+    def __setitem__(self, key):
+        raise RuntimeError("Cannot modify frozen {type(self).__name__}")
+
+    def update(self, other):
+        raise RuntimeError("Cannot modify frozen {type(self).__name__}")
+
+    def __hash__(self):
+        """Cache hash because we are immutable"""
+        if self._hash is None:
+            self._hash = hash(tuple((key, value) for key, value in self.items()))
+        return self._hash

jupyterhub/_version.py

@@ -2,7 +2,7 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 # version_info updated by running `tbump`
-version_info = (2, 1, 0, "", "")
+version_info = (3, 0, 0, "b1", "")
 
 # pep 440 version: no dot before beta/rc, but before .dev
 # 0.1.0rc1

jupyterhub/alembic/env.py

@@ -3,17 +3,16 @@ import sys
 from logging.config import fileConfig
 
 from alembic import context
-from sqlalchemy import engine_from_config
-from sqlalchemy import pool
+from sqlalchemy import engine_from_config, pool
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
-
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 if 'jupyterhub' in sys.modules:
     from traitlets.config import MultipleInstanceError
+
     from jupyterhub.app import JupyterHub
 
     app = None
@@ -42,6 +41,16 @@ target_metadata = orm.Base.metadata
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 
+# pass these to context.configure(**config_opts)
+common_config_opts = dict(
+    # target_metadata for autogenerate
+    target_metadata=target_metadata,
+    # transaction per migration to ensure
+    # each migration is 'complete' before running the next one
+    # (e.g. dropped tables)
+    transaction_per_migration=True,
+)
+
 
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
@@ -56,14 +65,15 @@ def run_migrations_offline():
 
     """
     connectable = config.attributes.get('connection', None)
+    config_opts = {}
+    config_opts.update(common_config_opts)
+    config_opts["literal_binds"] = True
 
     if connectable is None:
-        url = config.get_main_option("sqlalchemy.url")
-        context.configure(url=url, target_metadata=target_metadata, literal_binds=True)
+        config_opts["url"] = config.get_main_option("sqlalchemy.url")
     else:
-        context.configure(
-            connection=connectable, target_metadata=target_metadata, literal_binds=True
-        )
+        config_opts["connection"] = connectable
+    context.configure(**config_opts)
 
     with context.begin_transaction():
         context.run_migrations()
@@ -77,6 +87,8 @@ def run_migrations_online():
 
     """
     connectable = config.attributes.get('connection', None)
+    config_opts = {}
+    config_opts.update(common_config_opts)
 
     if connectable is None:
         connectable = engine_from_config(
@@ -86,7 +98,10 @@ def run_migrations_online():
         )
 
     with connectable.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata)
+        context.configure(
+            connection=connection,
+            **common_config_opts,
+        )
 
         with context.begin_transaction():
             context.run_migrations()

jupyterhub/alembic/versions/19c0846f6344_base_revision_for_0_5.py

@@ -11,8 +11,8 @@ down_revision = None
 branch_labels = None
 depends_on = None
 
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
 
 
 def upgrade():

jupyterhub/alembic/versions/1cebaf56856c_session_id.py

@@ -15,8 +15,8 @@ import logging
 
 logger = logging.getLogger('alembic')
 
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
 
 tables = ('oauth_access_tokens', 'oauth_codes')
 

jupyterhub/alembic/versions/3ec6993fe20c_encrypted_auth_state.py

@@ -22,8 +22,9 @@ import logging
 
 logger = logging.getLogger('alembic')
 
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
+
 from jupyterhub.orm import JSONDict
 
 

jupyterhub/alembic/versions/4dc2d5a8c53c_user_options.py

@@ -11,8 +11,9 @@ down_revision = '896818069c98'
 branch_labels = None
 depends_on = None
 
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
+
 from jupyterhub.orm import JSONDict
 
 

jupyterhub/alembic/versions/56cc5a70207e_token_tracking.py

@@ -11,11 +11,11 @@ down_revision = '1cebaf56856c'
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 import logging
 
+import sqlalchemy as sa
+from alembic import op
+
 logger = logging.getLogger('alembic')
 
 

jupyterhub/alembic/versions/651f5419b74d_api_token_scopes.py

@@ -0,0 +1,115 @@
+"""api_token_scopes
+
+Revision ID: 651f5419b74d
+Revises: 833da8570507
+Create Date: 2022-02-28 12:42:55.149046
+
+"""
+# revision identifiers, used by Alembic.
+revision = '651f5419b74d'
+down_revision = '833da8570507'
+branch_labels = None
+depends_on = None
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import Column, ForeignKey, Table
+from sqlalchemy.orm import relationship
+from sqlalchemy.orm.session import Session
+
+from jupyterhub import orm, roles, scopes
+
+
+def upgrade():
+    c = op.get_bind()
+
+    tables = sa.inspect(c.engine).get_table_names()
+
+    # oauth codes are short lived, no need to upgrade them
+    if 'oauth_code_role_map' in tables:
+        op.drop_table('oauth_code_role_map')
+
+    if 'oauth_codes' in tables:
+        op.add_column('oauth_codes', sa.Column('scopes', orm.JSONList(), nullable=True))
+
+    if 'api_tokens' in tables:
+        # may not be present,
+        # e.g. upgrade from 1.x, token table dropped
+        # in which case no migration to do
+
+        # define new scopes column on API tokens
+        op.add_column('api_tokens', sa.Column('scopes', orm.JSONList(), nullable=True))
+
+        if 'api_token_role_map' in tables:
+            # redefine the to-be-removed api_token->role relationship
+            # so we can run a query on it for the migration
+            token_role_map = Table(
+                "api_token_role_map",
+                orm.Base.metadata,
+                Column(
+                    'api_token_id',
+                    ForeignKey('api_tokens.id', ondelete='CASCADE'),
+                    primary_key=True,
+                ),
+                Column(
+                    'role_id',
+                    ForeignKey('roles.id', ondelete='CASCADE'),
+                    primary_key=True,
+                ),
+                extend_existing=True,
+            )
+            orm.APIToken.roles = relationship('Role', secondary='api_token_role_map')
+
+            # tokens have roles, evaluate to scopes
+            db = Session(bind=c)
+            for token in db.query(orm.APIToken):
+                token.scopes = list(roles.roles_to_scopes(token.roles))
+            db.commit()
+            # drop token-role relationship
+            op.drop_table('api_token_role_map')
+
+    if 'oauth_clients' in tables:
+        # define new scopes column on API tokens
+        op.add_column(
+            'oauth_clients', sa.Column('allowed_scopes', orm.JSONList(), nullable=True)
+        )
+
+        if 'oauth_client_role_map' in tables:
+            # redefine the to-be-removed api_token->role relationship
+            # so we can run a query on it for the migration
+            client_role_map = Table(
+                "oauth_client_role_map",
+                orm.Base.metadata,
+                Column(
+                    'oauth_client_id',
+                    ForeignKey('oauth_clients.id', ondelete='CASCADE'),
+                    primary_key=True,
+                ),
+                Column(
+                    'role_id',
+                    ForeignKey('roles.id', ondelete='CASCADE'),
+                    primary_key=True,
+                ),
+                extend_existing=True,
+            )
+            orm.OAuthClient.allowed_roles = relationship(
+                'Role', secondary='oauth_client_role_map'
+            )
+
+            # oauth clients have allowed_roles, evaluate to allowed_scopes
+            db = Session(bind=c)
+            for oauth_client in db.query(orm.OAuthClient):
+                allowed_scopes = set(roles.roles_to_scopes(oauth_client.allowed_roles))
+                allowed_scopes.update(scopes.access_scopes(oauth_client))
+                oauth_client.allowed_scopes = sorted(allowed_scopes)
+            db.commit()
+            # drop token-role relationship
+            op.drop_table('oauth_client_role_map')
+
+
+def downgrade():
+    # cannot map permissions from scopes back to roles
+    # drop whole api token table (revokes all tokens), which will be recreated on hub start
+    op.drop_table('api_tokens')
+    op.drop_table('oauth_clients')
+    op.drop_table('oauth_codes')

jupyterhub/alembic/versions/833da8570507_rbac.py

@@ -12,12 +12,11 @@ down_revision = '4dc2d5a8c53c'
 branch_labels = None
 depends_on = None
 
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
 
 from jupyterhub import orm
 
-
 naming_convention = orm.meta.naming_convention
 
 

jupyterhub/alembic/versions/896818069c98_token_expires.py

@@ -11,8 +11,8 @@ down_revision = 'd68c98b66cd4'
 branch_labels = None
 depends_on = None
 
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
 
 
 def upgrade():

jupyterhub/alembic/versions/99a28a4418e1_user_created.py

@@ -12,11 +12,11 @@ branch_labels = None
 depends_on = None
 
 
-from alembic import op
-import sqlalchemy as sa
-
 from datetime import datetime
 
+import sqlalchemy as sa
+from alembic import op
+
 
 def upgrade():
     op.add_column('users', sa.Column('created', sa.DateTime, nullable=True))

jupyterhub/alembic/versions/af4cbdb2d13c_services.py

@@ -11,8 +11,8 @@ down_revision = 'eeb276e51423'
 branch_labels = None
 depends_on = None
 
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
 
 
 def upgrade():

jupyterhub/alembic/versions/d68c98b66cd4_client_description.py

@@ -11,8 +11,8 @@ down_revision = '99a28a4418e1'
 branch_labels = None
 depends_on = None
 
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
 
 
 def upgrade():

jupyterhub/alembic/versions/eeb276e51423_auth_state.py

@@ -12,8 +12,9 @@ down_revision = '19c0846f6344'
 branch_labels = None
 depends_on = None
 
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
+
 from jupyterhub.orm import JSONDict
 
 

jupyterhub/apihandlers/__init__.py

@@ -1,9 +1,4 @@
-from . import auth
-from . import groups
-from . import hub
-from . import proxy
-from . import services
-from . import users
+from . import auth, groups, hub, proxy, services, users
 from .base import *
 
 default_handlers = []

jupyterhub/apihandlers/auth.py

@@ -1,25 +1,16 @@
 """Authorization handlers"""
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
-import itertools
 import json
 from datetime import datetime
-from urllib.parse import parse_qsl
-from urllib.parse import quote
-from urllib.parse import urlencode
-from urllib.parse import urlparse
-from urllib.parse import urlunparse
+from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse
 
 from oauthlib import oauth2
 from tornado import web
 
-from .. import orm
-from .. import roles
-from .. import scopes
-from ..utils import get_browser_protocol
-from ..utils import token_authenticated
-from .base import APIHandler
-from .base import BaseHandler
+from .. import orm, roles, scopes
+from ..utils import get_browser_protocol, token_authenticated
+from .base import APIHandler, BaseHandler
 
 
 class TokenAPIHandler(APIHandler):
@@ -38,7 +29,7 @@ class TokenAPIHandler(APIHandler):
         if owner:
             # having a token means we should be able to read the owner's model
             # (this is the only thing this handler is for)
-            self.expanded_scopes.update(scopes.identify_scopes(owner))
+            self.expanded_scopes |= scopes.identify_scopes(owner)
             self.parsed_scopes = scopes.parse_scopes(self.expanded_scopes)
 
         # record activity whenever we see a token
@@ -180,7 +171,7 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler):
             raise
         self.send_oauth_response(headers, body, status)
 
-    def needs_oauth_confirm(self, user, oauth_client, roles):
+    def needs_oauth_confirm(self, user, oauth_client, requested_scopes):
         """Return whether the given oauth client needs to prompt for access for the given user
 
         Checks list for oauth clients that don't need confirmation
@@ -211,20 +202,20 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler):
             user_id=user.id,
             client_id=oauth_client.identifier,
         )
-        authorized_roles = set()
+        authorized_scopes = set()
         for token in existing_tokens:
-            authorized_roles.update({role.name for role in token.roles})
+            authorized_scopes.update(token.scopes)
 
-        if authorized_roles:
-            if set(roles).issubset(authorized_roles):
+        if authorized_scopes:
+            if set(requested_scopes).issubset(authorized_scopes):
                 self.log.debug(
-                    f"User {user.name} has already authorized {oauth_client.identifier} for roles {roles}"
+                    f"User {user.name} has already authorized {oauth_client.identifier} for scopes {requested_scopes}"
                 )
                 return False
             else:
                 self.log.debug(
                     f"User {user.name} has authorized {oauth_client.identifier}"
-                    f" for roles {authorized_roles}, confirming additonal roles {roles}"
+                    f" for scopes {authorized_scopes}, confirming additional scopes {requested_scopes}"
                 )
         # default: require confirmation
         return True
@@ -251,7 +242,7 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler):
         uri, http_method, body, headers = self.extract_oauth_params()
         try:
             (
-                role_names,
+                requested_scopes,
                 credentials,
             ) = self.oauth_provider.validate_authorization_request(
                 uri, http_method, body, headers
@@ -283,26 +274,51 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler):
                 raise web.HTTPError(
                     403, f"You do not have permission to access {client.description}"
                 )
-            if not self.needs_oauth_confirm(self.current_user, client, role_names):
+
+            # subset 'raw scopes' to those held by authenticating user
+            requested_scopes = set(requested_scopes)
+            user = self.current_user
+            # raw, _not_ expanded scopes
+            user_scopes = roles.roles_to_scopes(roles.get_roles_for(user.orm_user))
+            # these are some scopes the user may not have
+            # in 'raw' form, but definitely have at this point
+            # make sure they are here, because we are computing the
+            # 'raw' scope intersection,
+            # rather than the expanded_scope intersection
+
+            required_scopes = {*scopes.identify_scopes(), *scopes.access_scopes(client)}
+            user_scopes |= {"inherit", *required_scopes}
+
+            allowed_scopes = requested_scopes.intersection(user_scopes)
+            excluded_scopes = requested_scopes.difference(user_scopes)
+            # TODO: compute lower-level intersection of remaining _expanded_ scopes
+            # (e.g. user has admin:users, requesting read:users!group=x)
+
+            if excluded_scopes:
+                self.log.warning(
+                    f"Service {client.description} requested scopes {','.join(requested_scopes)}"
+                    f" for user {self.current_user.name},"
+                    f" granting only {','.join(allowed_scopes) or '[]'}."
+                )
+
+            if not self.needs_oauth_confirm(self.current_user, client, allowed_scopes):
                 self.log.debug(
                     "Skipping oauth confirmation for %s accessing %s",
                     self.current_user,
                     client.description,
                 )
                 # this is the pre-1.0 behavior for all oauth
-                self._complete_login(uri, headers, role_names, credentials)
+                self._complete_login(uri, headers, allowed_scopes, credentials)
                 return
 
-            # resolve roles to scopes for authorization page
-            raw_scopes = set()
-            if role_names:
-                role_objects = (
-                    self.db.query(orm.Role).filter(orm.Role.name.in_(role_names)).all()
-                )
-                raw_scopes = set(
-                    itertools.chain(*(role.scopes for role in role_objects))
-                )
-            if not raw_scopes:
+            # discard 'required' scopes from description
+            # no need to describe the ability to access itself
+            scopes_to_describe = allowed_scopes.difference(required_scopes)
+
+            if not scopes_to_describe:
+                # TODO: describe all scopes?
+                # Not right now, because the no-scope default 'identify' text
+                # is clearer than what we produce for those scopes individually
                 scope_descriptions = [
                     {
                         "scope": None,
@@ -312,8 +328,8 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler):
                         "filter": "",
                     }
                 ]
-            elif 'inherit' in raw_scopes:
-                raw_scopes = ['inherit']
+            elif 'inherit' in scopes_to_describe:
+                allowed_scopes = scopes_to_describe = ['inherit']
                 scope_descriptions = [
                     {
                         "scope": "inherit",
@@ -325,7 +341,7 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler):
                 ]
             else:
                 scope_descriptions = scopes.describe_raw_scopes(
-                    raw_scopes,
+                    scopes_to_describe,
                     username=self.current_user.name,
                 )
             # Render oauth 'Authorize application...' page
@@ -334,7 +350,7 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler):
                 await self.render_template(
                     "oauth.html",
                     auth_state=auth_state,
-                    role_names=role_names,
+                    allowed_scopes=allowed_scopes,
                     scope_descriptions=scope_descriptions,
                     oauth_client=client,
                 )
@@ -381,6 +397,10 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler):
         # The scopes the user actually authorized, i.e. checkboxes
         # that were selected.
         scopes = self.get_arguments('scopes')
+        if scopes == []:
+            # avoid triggering default scopes (provider selects default scopes when scopes is falsy)
+            # when an explicit empty list is authorized
+            scopes = ["identify"]
         # credentials we need in the validator
         credentials = self.add_credentials()
 

jupyterhub/apihandlers/base.py

@@ -4,19 +4,15 @@
 import json
 from functools import lru_cache
 from http.client import responses
-from urllib.parse import parse_qs
-from urllib.parse import urlencode
-from urllib.parse import urlparse
-from urllib.parse import urlunparse
+from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 
 from sqlalchemy.exc import SQLAlchemyError
 from tornado import web
 
 from .. import orm
 from ..handlers import BaseHandler
-from ..utils import get_browser_protocol
-from ..utils import isoformat
-from ..utils import url_path_join
+from ..scopes import get_scopes_for
+from ..utils import get_browser_protocol, isoformat, url_escape_path, url_path_join
 
 PAGINATION_MEDIA_TYPE = "application/jupyterhub-pagination+json"
 
@@ -191,22 +187,44 @@ class APIHandler(BaseHandler):
             json.dumps({'status': status_code, 'message': message or status_message})
         )
 
-    def server_model(self, spawner):
+    def server_model(self, spawner, *, user=None):
         """Get the JSON model for a Spawner
-        Assume server permission already granted"""
+        Assume server permission already granted
+        """
+        if isinstance(spawner, orm.Spawner):
+            # if an orm.Spawner is passed,
+            # create a model for a stopped Spawner
+            # not all info is available without the higher-level Spawner wrapper
+            orm_spawner = spawner
+            pending = None
+            ready = False
+            stopped = True
+            user = user
+            if user is None:
+                raise RuntimeError("Must specify User with orm.Spawner")
+            state = orm_spawner.state
+        else:
+            orm_spawner = spawner.orm_spawner
+            pending = spawner.pending
+            ready = spawner.ready
+            user = spawner.user
+            stopped = not spawner.active
+            state = spawner.get_state()
+
         model = {
-            'name': spawner.name,
-            'last_activity': isoformat(spawner.orm_spawner.last_activity),
-            'started': isoformat(spawner.orm_spawner.started),
-            'pending': spawner.pending,
-            'ready': spawner.ready,
-            'url': url_path_join(spawner.user.url, spawner.name, '/'),
+            'name': orm_spawner.name,
+            'last_activity': isoformat(orm_spawner.last_activity),
+            'started': isoformat(orm_spawner.started),
+            'pending': pending,
+            'ready': ready,
+            'stopped': stopped,
+            'url': url_path_join(user.url, url_escape_path(spawner.name), '/'),
             'user_options': spawner.user_options,
-            'progress_url': spawner._progress_url,
+            'progress_url': user.progress_url(spawner.name),
         }
         scope_filter = self.get_scope_filter('admin:server_state')
         if scope_filter(spawner, kind='server'):
-            model['state'] = spawner.get_state()
+            model['state'] = state
         return model
 
     def token_model(self, token):
@@ -224,7 +242,9 @@ class APIHandler(BaseHandler):
             owner_key: owner,
             'id': token.api_id,
             'kind': 'api_token',
-            'roles': [r.name for r in token.roles],
+            # deprecated field, but leave it present.
+            'roles': [],
+            'scopes': list(get_scopes_for(token)),
             'created': isoformat(token.created),
             'last_activity': isoformat(token.last_activity),
             'expires_at': isoformat(token.expires_at),
@@ -250,10 +270,22 @@ class APIHandler(BaseHandler):
             keys.update(allowed_keys)
         return model
 
+    _include_stopped_servers = None
+
+    @property
+    def include_stopped_servers(self):
+        """Whether stopped servers should be included in user models"""
+        if self._include_stopped_servers is None:
+            self._include_stopped_servers = self.get_argument(
+                "include_stopped_servers", "0"
+            ).lower() not in {"0", "false"}
+        return self._include_stopped_servers
+
     def user_model(self, user):
         """Get the JSON model for a User object"""
         if isinstance(user, orm.User):
             user = self.users[user.id]
+        include_stopped_servers = self.include_stopped_servers
         model = {
             'kind': 'user',
             'name': user.name,
@@ -293,18 +325,29 @@ class APIHandler(BaseHandler):
             if '' in user.spawners and 'pending' in allowed_keys:
                 model['pending'] = user.spawners[''].pending
 
-            servers = model['servers'] = {}
+            servers = {}
             scope_filter = self.get_scope_filter('read:servers')
             for name, spawner in user.spawners.items():
                 # include 'active' servers, not just ready
                 # (this includes pending events)
-                if spawner.active and scope_filter(spawner, kind='server'):
+                if (spawner.active or include_stopped_servers) and scope_filter(
+                    spawner, kind='server'
+                ):
                     servers[name] = self.server_model(spawner)
-            if not servers and 'servers' not in allowed_keys:
+
+            if include_stopped_servers:
+                # add any stopped servers in the db
+                seen = set(servers.keys())
+                for name, orm_spawner in user.orm_spawners.items():
+                    if name not in seen and scope_filter(orm_spawner, kind='server'):
+                        servers[name] = self.server_model(orm_spawner, user=user)
+
+            if "servers" in allowed_keys or servers:
                 # omit servers if no access
                 # leave present and empty
                 # if request has access to read servers in general
-                model.pop('servers')
+                model["servers"] = servers
+
         return model
 
     def group_model(self, group):

jupyterhub/apihandlers/groups.py

@@ -6,8 +6,7 @@ import json
 from tornado import web
 
 from .. import orm
-from ..scopes import needs_scope
-from ..scopes import Scope
+from ..scopes import Scope, needs_scope
 from .base import APIHandler
 
 
@@ -33,6 +32,11 @@ class _GroupAPIHandler(APIHandler):
             raise web.HTTPError(404, "No such group: %s", group_name)
         return group
 
+    def check_authenticator_managed_groups(self):
+        """Raise error on group-management APIs if Authenticator is managing groups"""
+        if self.authenticator.manage_groups:
+            raise web.HTTPError(400, "Group management via API is disabled")
+
 
 class GroupListAPIHandler(_GroupAPIHandler):
     @needs_scope('list:groups')
@@ -45,7 +49,7 @@ class GroupListAPIHandler(_GroupAPIHandler):
                 # the only valid filter is group=...
                 # don't expand invalid !server=x to all groups!
                 self.log.warning(
-                    "Invalid filter on list:group for {self.current_user}: {sub_scope}"
+                    f"Invalid filter on list:group for {self.current_user}: {sub_scope}"
                 )
                 raise web.HTTPError(403)
             query = query.filter(orm.Group.name.in_(sub_scope['group']))
@@ -68,6 +72,9 @@ class GroupListAPIHandler(_GroupAPIHandler):
     @needs_scope('admin:groups')
     async def post(self):
         """POST creates Multiple groups"""
+
+        self.check_authenticator_managed_groups()
+
         model = self.get_json_body()
         if not model or not isinstance(model, dict) or not model.get('groups'):
             raise web.HTTPError(400, "Must specify at least one group to create")
@@ -106,6 +113,7 @@ class GroupAPIHandler(_GroupAPIHandler):
     @needs_scope('admin:groups')
     async def post(self, group_name):
         """POST creates a group by name"""
+        self.check_authenticator_managed_groups()
         model = self.get_json_body()
         if model is None:
             model = {}
@@ -132,6 +140,7 @@ class GroupAPIHandler(_GroupAPIHandler):
     @needs_scope('delete:groups')
     def delete(self, group_name):
         """Delete a group by name"""
+        self.check_authenticator_managed_groups()
         group = self.find_group(group_name)
         self.log.info("Deleting group %s", group_name)
         self.db.delete(group)
@@ -145,6 +154,7 @@ class GroupUsersAPIHandler(_GroupAPIHandler):
     @needs_scope('groups')
     def post(self, group_name):
         """POST adds users to a group"""
+        self.check_authenticator_managed_groups()
         group = self.find_group(group_name)
         data = self.get_json_body()
         self._check_group_model(data)
@@ -163,6 +173,7 @@ class GroupUsersAPIHandler(_GroupAPIHandler):
     @needs_scope('groups')
     async def delete(self, group_name):
         """DELETE removes users from a group"""
+        self.check_authenticator_managed_groups()
         group = self.find_group(group_name)
         data = self.get_json_body()
         self._check_group_model(data)

jupyterhub/apihandlers/hub.py

@@ -47,9 +47,8 @@ class ShutdownAPIHandler(APIHandler):
         self.set_status(202)
         self.finish(json.dumps({"message": "Shutting down Hub"}))
 
-        # stop the eventloop, which will trigger cleanup
-        loop = IOLoop.current()
-        loop.add_callback(loop.stop)
+        # instruct the app to stop, which will trigger cleanup
+        app.stop()
 
 
 class RootAPIHandler(APIHandler):

jupyterhub/apihandlers/proxy.py

@@ -26,7 +26,7 @@ class ProxyAPIHandler(APIHandler):
         else:
             routes = {}
             end = offset + limit
-            for i, key in sorted(all_routes.keys()):
+            for i, key in enumerate(sorted(all_routes.keys())):
                 if i < offset:
                     continue
                 elif i >= end:

jupyterhub/apihandlers/services.py

@@ -6,8 +6,7 @@ Currently GET-only, no actions can be taken to modify services.
 # Distributed under the terms of the Modified BSD License.
 import json
 
-from ..scopes import needs_scope
-from ..scopes import Scope
+from ..scopes import Scope, needs_scope
 from .base import APIHandler
 
 

jupyterhub/apihandlers/users.py

@@ -3,26 +3,25 @@
 # Distributed under the terms of the Modified BSD License.
 import asyncio
 import json
-from datetime import datetime
-from datetime import timedelta
-from datetime import timezone
+from datetime import datetime, timedelta, timezone
 
 from async_generator import aclosing
 from dateutil.parser import parse as parse_date
-from sqlalchemy import func
-from sqlalchemy import or_
+from sqlalchemy import func, or_
 from tornado import web
 from tornado.iostream import StreamClosedError
 
-from .. import orm
-from .. import scopes
+from .. import orm, scopes
 from ..roles import assign_default_roles
 from ..scopes import needs_scope
 from ..user import User
-from ..utils import isoformat
-from ..utils import iterate_until
-from ..utils import maybe_future
-from ..utils import url_path_join
+from ..utils import (
+    isoformat,
+    iterate_until,
+    maybe_future,
+    url_escape_path,
+    url_path_join,
+)
 from .base import APIHandler
 
 
@@ -51,7 +50,7 @@ class SelfAPIHandler(APIHandler):
         for scope in identify_scopes:
             if scope not in self.expanded_scopes:
                 _added_scopes.add(scope)
-                self.expanded_scopes.add(scope)
+                self.expanded_scopes |= {scope}
         if _added_scopes:
             # re-parse with new scopes
             self.parsed_scopes = scopes.parse_scopes(self.expanded_scopes)
@@ -84,6 +83,7 @@ class UserListAPIHandler(APIHandler):
     @needs_scope('list:users')
     def get(self):
         state_filter = self.get_argument("state", None)
+        name_filter = self.get_argument("name_filter", None)
         offset, limit = self.get_api_pagination()
 
         # post_filter
@@ -130,7 +130,7 @@ class UserListAPIHandler(APIHandler):
             if not set(sub_scope).issubset({'group', 'user'}):
                 # don't expand invalid !server=x filter to all users!
                 self.log.warning(
-                    "Invalid filter on list:user for {self.current_user}: {sub_scope}"
+                    f"Invalid filter on list:user for {self.current_user}: {sub_scope}"
                 )
                 raise web.HTTPError(403)
             filters = []
@@ -148,6 +148,9 @@ class UserListAPIHandler(APIHandler):
             else:
                 query = query.filter(or_(*filters))
 
+        if name_filter:
+            query = query.filter(orm.User.name.ilike(f'%{name_filter}%'))
+
         full_query = query
         query = query.order_by(orm.User.id.asc()).offset(offset).limit(limit)
 
@@ -402,21 +405,18 @@ class UserTokenListAPIHandler(APIHandler):
             if requester is not user:
                 note += f" by {kind} {requester.name}"
 
-        token_roles = body.get('roles')
+        token_roles = body.get("roles")
+        token_scopes = body.get("scopes")
+
         try:
             api_token = user.new_api_token(
                 note=note,
                 expires_in=body.get('expires_in', None),
                 roles=token_roles,
+                scopes=token_scopes,
             )
-        except KeyError:
-            raise web.HTTPError(404, "Requested roles %r not found" % token_roles)
-        except ValueError:
-            raise web.HTTPError(
-                403,
-                "Requested roles %r cannot have higher permissions than the token owner"
-                % token_roles,
-            )
+        except ValueError as e:
+            raise web.HTTPError(400, str(e))
         if requester is not user:
             self.log.info(
                 "%s %s requested API token for %s",
@@ -515,7 +515,7 @@ class UserServerAPIHandler(APIHandler):
                             user_name, self.named_server_limit_per_user
                         ),
                     )
-        spawner = user.spawners[server_name]
+        spawner = user.get_spawner(server_name, replace_failed=True)
         pending = spawner.pending
         if pending == 'spawn':
             self.set_header('Content-Type', 'text/plain')
@@ -691,7 +691,7 @@ class SpawnProgressAPIHandler(APIHandler):
         # - spawner not running at all
         # - spawner failed
         # - spawner pending start (what we expect)
-        url = url_path_join(user.url, server_name, '/')
+        url = url_path_join(user.url, url_escape_path(server_name), '/')
         ready_event = {
             'progress': 100,
             'ready': True,

jupyterhub/app.py

@@ -11,106 +11,85 @@ import re
 import secrets
 import signal
 import socket
+import ssl
 import sys
 import time
 from concurrent.futures import ThreadPoolExecutor
-from datetime import datetime
-from datetime import timedelta
-from datetime import timezone
-from functools import partial
+from datetime import datetime, timedelta, timezone
 from getpass import getuser
-from glob import glob
-from itertools import chain
 from operator import itemgetter
 from textwrap import dedent
-from urllib.parse import unquote
-from urllib.parse import urlparse
-from urllib.parse import urlunparse
+from urllib.parse import unquote, urlparse, urlunparse
 
 if sys.version_info[:2] < (3, 3):
     raise ValueError("Python < 3.3 not supported: %s" % sys.version)
 
-# For compatibility with python versions 3.6 or earlier.
-# asyncio.Task.all_tasks() is fully moved to asyncio.all_tasks() starting with 3.9. Also applies to current_task.
-try:
-    asyncio_all_tasks = asyncio.all_tasks
-    asyncio_current_task = asyncio.current_task
-except AttributeError as e:
-    asyncio_all_tasks = asyncio.Task.all_tasks
-    asyncio_current_task = asyncio.Task.current_task
-
-from dateutil.parser import parse as parse_date
-from jinja2 import Environment, FileSystemLoader, PrefixLoader, ChoiceLoader
-from sqlalchemy.exc import OperationalError, SQLAlchemyError
-
-from tornado.httpclient import AsyncHTTPClient
 import tornado.httpserver
-from tornado.ioloop import IOLoop, PeriodicCallback
-from tornado.log import app_log, access_log, gen_log
 import tornado.options
+from dateutil.parser import parse as parse_date
+from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader
+from jupyter_telemetry.eventlog import EventLog
+from sqlalchemy.exc import OperationalError, SQLAlchemyError
 from tornado import gen, web
-
+from tornado.httpclient import AsyncHTTPClient
+from tornado.ioloop import IOLoop, PeriodicCallback
+from tornado.log import access_log, app_log, gen_log
 from traitlets import (
-    Unicode,
-    Integer,
-    Dict,
-    TraitError,
-    List,
-    Bool,
     Any,
-    Tuple,
-    Type,
-    Set,
-    Instance,
+    Bool,
     Bytes,
+    Dict,
     Float,
+    Instance,
+    Integer,
+    List,
+    Set,
+    Tuple,
+    Unicode,
     Union,
-    observe,
     default,
+    observe,
     validate,
 )
 from traitlets.config import Application, Configurable, catch_config_error
 
-from jupyter_telemetry.eventlog import EventLog
-
 here = os.path.dirname(__file__)
 
 import jupyterhub
-from . import handlers, apihandlers
-from .handlers.static import CacheControlStaticFilesHandler, LogoHandler
-from .services.service import Service
 
-from . import crypto
-from . import dbutil, orm
-from . import roles
-from .user import UserDict
-from .oauth.provider import make_provider
+from . import apihandlers, crypto, dbutil, handlers, orm, roles, scopes
 from ._data import DATA_FILES_PATH
-from .log import CoroutineLogFormatter, log_request
-from .proxy import Proxy, ConfigurableHTTPProxy
-from .traitlets import URLPrefix, Command, EntryPointType, Callable
-from .utils import (
-    AnyTimeoutError,
-    catch_db_error,
-    maybe_future,
-    url_path_join,
-    print_stacks,
-    print_ps_info,
-    make_ssl_context,
-)
-from .metrics import HUB_STARTUP_DURATION_SECONDS
-from .metrics import INIT_SPAWNERS_DURATION_SECONDS
-from .metrics import RUNNING_SERVERS
-from .metrics import TOTAL_USERS
 
 # classes for config
 from .auth import Authenticator, PAMAuthenticator
 from .crypto import CryptKeeper
-from .spawner import Spawner, LocalProcessSpawner
-from .objects import Hub, Server
 
 # For faking stats
 from .emptyclass import EmptyClass
+from .handlers.static import CacheControlStaticFilesHandler, LogoHandler
+from .log import CoroutineLogFormatter, log_request
+from .metrics import (
+    HUB_STARTUP_DURATION_SECONDS,
+    INIT_SPAWNERS_DURATION_SECONDS,
+    RUNNING_SERVERS,
+    TOTAL_USERS,
+)
+from .oauth.provider import make_provider
+from .objects import Hub, Server
+from .proxy import ConfigurableHTTPProxy, Proxy
+from .services.service import Service
+from .spawner import LocalProcessSpawner, Spawner
+from .traitlets import Callable, Command, EntryPointType, URLPrefix
+from .user import UserDict
+from .utils import (
+    AnyTimeoutError,
+    catch_db_error,
+    make_ssl_context,
+    maybe_future,
+    print_ps_info,
+    print_stacks,
+    url_path_join,
+)
 
 common_aliases = {
     'log-level': 'Application.log_level',
@@ -353,6 +332,29 @@ class JupyterHub(Application):
         """,
     ).tag(config=True)
 
+    custom_scopes = Dict(
+        key_trait=Unicode(),
+        value_trait=Dict(
+            key_trait=Unicode(),
+        ),
+        help="""Custom scopes to define.
+
+        For use when defining custom roles,
+        to grant users granular permissions
+
+        All custom scopes must have a description,
+        and must start with the prefix `custom:`.
+
+        For example::
+
+            custom_scopes = {
+                "custom:jupyter_server:read": {
+                    "description": "read-only access to a single-user server",
+                },
+            }
+        """,
+    ).tag(config=True)
+
     config_file = Unicode('jupyterhub_config.py', help="The config file to load").tag(
         config=True
     )
@@ -701,11 +703,14 @@ class JupyterHub(Application):
         """,
     ).tag(config=True)
 
-    def _subdomain_host_changed(self, name, old, new):
+    @validate("subdomain_host")
+    def _validate_subdomain_host(self, proposal):
+        new = proposal.value
         if new and '://' not in new:
             # host should include '://'
             # if not specified, assume https: You have to be really explicit about HTTP!
-            self.subdomain_host = 'https://' + new
+            new = 'https://' + new
+        return new
 
     domain = Unicode(help="domain name, e.g. 'example.com' (excludes protocol, port)")
 
@@ -1119,7 +1124,7 @@ class JupyterHub(Application):
 
     @default('authenticator')
     def _authenticator_default(self):
-        return self.authenticator_class(parent=self, db=self.db)
+        return self.authenticator_class(parent=self, _deprecated_db_session=self.db)
 
     implicit_spawn_seconds = Float(
         0,
@@ -1307,11 +1312,14 @@ class JupyterHub(Application):
 
     admin_access = Bool(
         False,
-        help="""Grant admin users permission to access single-user servers.
+        help="""DEPRECATED since version 2.0.0.
 
-        Users should be properly informed if this is enabled.
+        The default admin role has full permissions, use custom RBAC scopes instead to
+        create restricted administrator roles.
+        https://jupyterhub.readthedocs.io/en/stable/rbac/index.html
         """,
     ).tag(config=True)
+
     admin_users = Set(
         help="""DEPRECATED since version 0.7.2, use Authenticator.admin_users instead."""
     ).tag(config=True)
@@ -1689,7 +1697,9 @@ class JupyterHub(Application):
             for authority, files in self.internal_ssl_authorities.items():
                 if files:
                     self.log.info("Adding CA for %s", authority)
-                    certipy.store.add_record(authority, is_ca=True, files=files)
+                    certipy.store.add_record(
+                        authority, is_ca=True, files=files, overwrite=True
+                    )
 
             self.internal_trust_bundles = certipy.trust_from_graph(
                 self.internal_ssl_components_trust
@@ -2001,6 +2011,9 @@ class JupyterHub(Application):
     async def init_groups(self):
         """Load predefined groups into the database"""
         db = self.db
+
+        if self.authenticator.manage_groups and self.load_groups:
+            raise ValueError("Group management has been offloaded to the authenticator")
         for name, usernames in self.load_groups.items():
             group = orm.Group.find(db, name)
             if group is None:
@@ -2015,7 +2028,10 @@ class JupyterHub(Application):
         db.commit()
 
     async def init_role_creation(self):
-        """Load default and predefined roles into the database"""
+        """Load default and user-defined roles and scopes into the database"""
+        if self.custom_scopes:
+            self.log.info(f"Defining {len(self.custom_scopes)} custom scopes.")
+            scopes.define_custom_scopes(self.custom_scopes)
         self.log.debug('Loading roles into database')
         default_roles = roles.get_default_roles()
         config_role_names = [r['name'] for r in self.load_roles]
@@ -2052,23 +2068,6 @@ class JupyterHub(Application):
 
         # make sure we load any default roles not overridden
         init_roles = list(default_roles_dict.values()) + init_roles
-        if roles_with_new_permissions:
-            unauthorized_oauth_tokens = (
-                self.db.query(orm.APIToken)
-                .filter(
-                    orm.APIToken.roles.any(
-                        orm.Role.name.in_(roles_with_new_permissions)
-                    )
-                )
-                .filter(orm.APIToken.client_id != 'jupyterhub')
-            )
-            for token in unauthorized_oauth_tokens:
-                self.log.warning(
-                    "Deleting OAuth token %s; one of its roles obtained new permissions that were not authorized by user"
-                    % token
-                )
-                self.db.delete(token)
-            self.db.commit()
 
         init_role_names = [r['name'] for r in init_roles]
         if (
@@ -2204,9 +2203,6 @@ class JupyterHub(Application):
             for kind in kinds:
                 roles.check_for_default_roles(db, kind)
 
-            # check tokens for default roles
-            roles.check_for_default_roles(db, bearer='tokens')
-
     async def _add_tokens(self, token_dict, kind):
         """Add tokens for users or services to the database"""
         if kind == 'user':
@@ -2371,21 +2367,34 @@ class JupyterHub(Application):
                 service.orm.server = None
 
             if service.oauth_available:
-                allowed_roles = []
+                allowed_scopes = set()
+                if service.oauth_client_allowed_scopes:
+                    allowed_scopes.update(service.oauth_client_allowed_scopes)
                 if service.oauth_roles:
-                    allowed_roles = list(
-                        self.db.query(orm.Role).filter(
-                            orm.Role.name.in_(service.oauth_roles)
+                    if not allowed_scopes:
+                        # DEPRECATED? It's still convenient and valid,
+                        # e.g. 'admin'
+                        allowed_roles = list(
+                            self.db.query(orm.Role).filter(
+                                orm.Role.name.in_(service.oauth_roles)
+                            )
+                        )
+                        allowed_scopes.update(roles.roles_to_scopes(allowed_roles))
+                    else:
+                        self.log.warning(
+                            f"Ignoring oauth_roles for {service.name}: {service.oauth_roles},"
+                            f" using oauth_client_allowed_scopes={allowed_scopes}."
                         )
-                    )
                 oauth_client = self.oauth_provider.add_client(
                     client_id=service.oauth_client_id,
                     client_secret=service.api_token,
                     redirect_uri=service.oauth_redirect_uri,
-                    allowed_roles=allowed_roles,
                     description="JupyterHub service %s" % service.name,
                 )
                 service.orm.oauth_client = oauth_client
+                # add access-scopes, derived from OAuthClient itself
+                allowed_scopes.update(scopes.access_scopes(oauth_client))
+                oauth_client.allowed_scopes = sorted(allowed_scopes)
             else:
                 if service.oauth_client:
                     self.db.delete(service.oauth_client)
@@ -3055,7 +3064,7 @@ class JupyterHub(Application):
             self.internal_ssl_key,
             self.internal_ssl_cert,
             cafile=self.internal_ssl_ca,
-            check_hostname=False,
+            purpose=ssl.Purpose.CLIENT_AUTH,
         )
 
         # start the webserver
@@ -3147,7 +3156,12 @@ class JupyterHub(Application):
             self.last_activity_callback = pc
             pc.start()
 
-        self.log.info("JupyterHub is now running at %s", self.proxy.public_url)
+        if self.proxy.should_start:
+            self.log.info("JupyterHub is now running at %s", self.proxy.public_url)
+        else:
+            self.log.info(
+                "JupyterHub is now running, internal Hub API at %s", self.hub.url
+            )
         # Use atexit for Windows, it doesn't have signal handling support
         if _mswindows:
             atexit.register(self.atexit)
@@ -3227,16 +3241,18 @@ class JupyterHub(Application):
         self._atexit_ran = True
         self._init_asyncio_patch()
         # run the cleanup step (in a new loop, because the interrupted one is unclean)
-        asyncio.set_event_loop(asyncio.new_event_loop())
-        IOLoop.clear_current()
-        loop = IOLoop()
-        loop.make_current()
-        loop.run_sync(self.cleanup)
+        asyncio.run(self.cleanup())
 
-    async def shutdown_cancel_tasks(self, sig):
+    async def shutdown_cancel_tasks(self, sig=None):
         """Cancel all other tasks of the event loop and initiate cleanup"""
-        self.log.critical("Received signal %s, initiating shutdown...", sig.name)
-        tasks = [t for t in asyncio_all_tasks() if t is not asyncio_current_task()]
+        if sig is None:
+            self.log.critical("Initiating shutdown...")
+        else:
+            self.log.critical("Received signal %s, initiating shutdown...", sig.name)
+
+        await self.cleanup()
+
+        tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
 
         if tasks:
             self.log.debug("Cancelling pending tasks")
@@ -3249,10 +3265,9 @@ class JupyterHub(Application):
             except StopAsyncIteration as e:
                 self.log.error("Caught StopAsyncIteration Exception", exc_info=True)
 
-            tasks = [t for t in asyncio_all_tasks()]
+            tasks = [t for t in asyncio.all_tasks()]
             for t in tasks:
                 self.log.debug("Task status: %s", t)
-        await self.cleanup()
         asyncio.get_event_loop().stop()
 
     def stop(self):
@@ -3260,7 +3275,7 @@ class JupyterHub(Application):
             return
         if self.http_server:
             self.http_server.stop()
-        self.io_loop.add_callback(self.io_loop.stop)
+        self.io_loop.add_callback(self.shutdown_cancel_tasks)
 
     async def start_show_config(self):
         """Async wrapper around base start_show_config method"""
@@ -3286,16 +3301,19 @@ class JupyterHub(Application):
     def launch_instance(cls, argv=None):
         self = cls.instance()
         self._init_asyncio_patch()
-        loop = IOLoop.current()
-        task = asyncio.ensure_future(self.launch_instance_async(argv))
+        loop = IOLoop(make_current=False)
+
+        try:
+            loop.run_sync(self.launch_instance_async, argv)
+        except Exception:
+            loop.close()
+            raise
+
         try:
             loop.start()
         except KeyboardInterrupt:
             print("\nInterrupted")
         finally:
-            if task.done():
-                # re-raise exceptions in launch_instance_async
-                task.result()
             loop.stop()
             loop.close()
 

jupyterhub/auth.py

@@ -9,9 +9,8 @@ import warnings
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 from shutil import which
-from subprocess import PIPE
-from subprocess import Popen
-from subprocess import STDOUT
+from subprocess import PIPE, STDOUT, Popen
+from textwrap import dedent
 
 try:
     import pamela
@@ -20,13 +19,12 @@ except Exception as e:
     _pamela_error = e
 
 from tornado.concurrent import run_on_executor
-
+from traitlets import Any, Bool, Dict, Integer, Set, Unicode, default, observe
 from traitlets.config import LoggingConfigurable
-from traitlets import Bool, Integer, Set, Unicode, Dict, Any, default, observe
 
 from .handlers.login import LoginHandler
-from .utils import maybe_future, url_path_join
 from .traitlets import Command
+from .utils import maybe_future, url_path_join
 
 
 class Authenticator(LoggingConfigurable):
@@ -34,6 +32,23 @@ class Authenticator(LoggingConfigurable):
 
     db = Any()
 
+    @default("db")
+    def _deprecated_db(self):
+        self.log.warning(
+            dedent(
+                """
+                The shared database session at Authenticator.db is deprecated, and will be removed.
+                Please manage your own database and connections.
+
+                Contact JupyterHub at https://github.com/jupyterhub/jupyterhub/issues/3700
+                if you have questions or ideas about direct database needs for your Authenticator.
+                """
+            ),
+        )
+        return self._deprecated_db_session
+
+    _deprecated_db_session = Any()
+
     enable_auth_state = Bool(
         False,
         config=True,
@@ -582,9 +597,13 @@ class Authenticator(LoggingConfigurable):
                 or None if Authentication failed.
 
                 The Authenticator may return a dict instead, which MUST have a
-                key `name` holding the username, and MAY have two optional keys
-                set: `auth_state`, a dictionary of of auth state that will be
-                persisted; and `admin`, the admin setting value for the user.
+                key `name` holding the username, and MAY have additional keys:
+
+                - `auth_state`, a dictionary of of auth state that will be
+                  persisted;
+                - `admin`, the admin setting value for the user
+                - `groups`, the list of group names the user should be a member of,
+                  if Authenticator.manage_groups is True.
         """
 
     def pre_spawn_start(self, user, spawner):
@@ -635,6 +654,19 @@ class Authenticator(LoggingConfigurable):
         """
         self.allowed_users.discard(user.name)
 
+    manage_groups = Bool(
+        False,
+        config=True,
+        help="""Let authenticator manage user groups
+
+        If True, Authenticator.authenticate and/or .refresh_user
+        may return a list of group names in the 'groups' field,
+        which will be assigned to the user.
+
+        All group-assignment APIs are disabled if this is True.
+        """,
+    )
+
     auto_login = Bool(
         False,
         config=True,
@@ -800,7 +832,7 @@ class LocalAuthenticator(Authenticator):
             raise ValueError("I don't know how to create users on OS X")
         elif which('pw'):
             # Probably BSD
-            return ['pw', 'useradd', '-m']
+            return ['pw', 'useradd', '-m', '-n']
         else:
             # This appears to be the Linux non-interactive adduser command:
             return ['adduser', '-q', '--gecos', '""', '--disabled-password']
@@ -958,16 +990,24 @@ class PAMAuthenticator(LocalAuthenticator):
     ).tag(config=True)
 
     open_sessions = Bool(
-        True,
+        False,
         help="""
         Whether to open a new PAM session when spawners are started.
 
-        This may trigger things like mounting shared filsystems,
-        loading credentials, etc. depending on system configuration,
-        but it does not always work.
+        This may trigger things like mounting shared filesystems,
+        loading credentials, etc. depending on system configuration.
+
+        The lifecycle of PAM sessions is not correct,
+        so many PAM session configurations will not work.
 
         If any errors are encountered when opening/closing PAM sessions,
         this is automatically set to False.
+
+        .. versionchanged:: 2.2
+
+            Due to longstanding problems in the session lifecycle,
+            this is now disabled by default.
+            You may opt-in to opening sessions by setting this to True.
         """,
     ).tag(config=True)
 

jupyterhub/crypto.py

@@ -4,18 +4,12 @@ import os
 from binascii import a2b_hex
 from concurrent.futures import ThreadPoolExecutor
 
-from traitlets import Any
-from traitlets import default
-from traitlets import Integer
-from traitlets import List
-from traitlets import observe
-from traitlets import validate
-from traitlets.config import Config
-from traitlets.config import SingletonConfigurable
+from traitlets import Any, Integer, List, default, observe, validate
+from traitlets.config import Config, SingletonConfigurable
 
 try:
     import cryptography
-    from cryptography.fernet import Fernet, MultiFernet, InvalidToken
+    from cryptography.fernet import Fernet, InvalidToken, MultiFernet
 except ImportError:
     cryptography = None
 

jupyterhub/handlers/__init__.py

@@ -1,7 +1,4 @@
-from . import base
-from . import login
-from . import metrics
-from . import pages
+from . import base, login, metrics, pages
 from .base import *
 from .login import *
 

jupyterhub/handlers/base.py

@@ -9,50 +9,44 @@ import random
 import re
 import time
 import uuid
-from datetime import datetime
-from datetime import timedelta
+from datetime import datetime, timedelta
 from http.client import responses
-from urllib.parse import parse_qs
-from urllib.parse import parse_qsl
-from urllib.parse import urlencode
-from urllib.parse import urlparse
-from urllib.parse import urlunparse
+from urllib.parse import parse_qs, parse_qsl, urlencode, urlparse, urlunparse
 
 from jinja2 import TemplateNotFound
 from sqlalchemy.exc import SQLAlchemyError
-from tornado import gen
-from tornado import web
-from tornado.httputil import HTTPHeaders
-from tornado.httputil import url_concat
+from tornado import gen, web
+from tornado.httputil import HTTPHeaders, url_concat
 from tornado.ioloop import IOLoop
 from tornado.log import app_log
-from tornado.web import addslash
-from tornado.web import RequestHandler
+from tornado.web import RequestHandler, addslash
 
-from .. import __version__
-from .. import orm
-from .. import roles
-from .. import scopes
-from ..metrics import PROXY_ADD_DURATION_SECONDS
-from ..metrics import PROXY_DELETE_DURATION_SECONDS
-from ..metrics import ProxyDeleteStatus
-from ..metrics import RUNNING_SERVERS
-from ..metrics import SERVER_POLL_DURATION_SECONDS
-from ..metrics import SERVER_SPAWN_DURATION_SECONDS
-from ..metrics import SERVER_STOP_DURATION_SECONDS
-from ..metrics import ServerPollStatus
-from ..metrics import ServerSpawnStatus
-from ..metrics import ServerStopStatus
-from ..metrics import TOTAL_USERS
+from .. import __version__, orm, roles, scopes
+from ..metrics import (
+    PROXY_ADD_DURATION_SECONDS,
+    PROXY_DELETE_DURATION_SECONDS,
+    RUNNING_SERVERS,
+    SERVER_POLL_DURATION_SECONDS,
+    SERVER_SPAWN_DURATION_SECONDS,
+    SERVER_STOP_DURATION_SECONDS,
+    TOTAL_USERS,
+    ProxyDeleteStatus,
+    ServerPollStatus,
+    ServerSpawnStatus,
+    ServerStopStatus,
+)
 from ..objects import Server
 from ..scopes import needs_scope
 from ..spawner import LocalProcessSpawner
 from ..user import User
-from ..utils import AnyTimeoutError
-from ..utils import get_accepted_mimetype
-from ..utils import get_browser_protocol
-from ..utils import maybe_future
-from ..utils import url_path_join
+from ..utils import (
+    AnyTimeoutError,
+    get_accepted_mimetype,
+    get_browser_protocol,
+    maybe_future,
+    url_escape_path,
+    url_path_join,
+)
 
 # pattern for the authentication token header
 auth_header_pat = re.compile(r'^(?:token|bearer)\s+([^\s]+)$', flags=re.IGNORECASE)
@@ -526,10 +520,16 @@ class BaseHandler(RequestHandler):
             path=url_path_join(self.base_url, 'services'),
             **kwargs,
         )
-        # clear tornado cookie
+        # clear_cookie only accepts a subset of set_cookie's kwargs
+        clear_xsrf_cookie_kwargs = {
+            key: value
+            for key, value in self.settings.get('xsrf_cookie_kwargs', {}).items()
+            if key in {"path", "domain"}
+        }
+
         self.clear_cookie(
             '_xsrf',
-            **self.settings.get('xsrf_cookie_kwargs', {}),
+            **clear_xsrf_cookie_kwargs,
         )
         # Reset _jupyterhub_user
         self._jupyterhub_user = None
@@ -636,29 +636,32 @@ class BaseHandler(RequestHandler):
         next_url = next_url.replace('\\', '%5C')
         proto = get_browser_protocol(self.request)
         host = self.request.host
+        if next_url.startswith("///"):
+            # strip more than 2 leading // down to 2
+            # because urlparse treats that as empty netloc,
+            # whereas browsers treat more than two leading // the same as //,
+            # so netloc is the first non-/ bit
+            next_url = "//" + next_url.lstrip("/")
+        parsed_next_url = urlparse(next_url)
 
         if (next_url + '/').startswith((f'{proto}://{host}/', f'//{host}/',)) or (
             self.subdomain_host
-            and urlparse(next_url).netloc
-            and ("." + urlparse(next_url).netloc).endswith(
+            and parsed_next_url.netloc
+            and ("." + parsed_next_url.netloc).endswith(
                 "." + urlparse(self.subdomain_host).netloc
             )
         ):
             # treat absolute URLs for our host as absolute paths:
-            # below, redirects that aren't strictly paths
-            parsed = urlparse(next_url)
-            next_url = parsed.path
-            if parsed.query:
-                next_url = next_url + '?' + parsed.query
-            if parsed.fragment:
-                next_url = next_url + '#' + parsed.fragment
+            # below, redirects that aren't strictly paths are rejected
+            next_url = parsed_next_url.path
+            if parsed_next_url.query:
+                next_url = next_url + '?' + parsed_next_url.query
+            if parsed_next_url.fragment:
+                next_url = next_url + '#' + parsed_next_url.fragment
+            parsed_next_url = urlparse(next_url)
 
         # if it still has host info, it didn't match our above check for *this* host
-        if next_url and (
-            '://' in next_url
-            or next_url.startswith('//')
-            or not next_url.startswith('/')
-        ):
+        if next_url and (parsed_next_url.netloc or not next_url.startswith('/')):
             self.log.warning("Disallowing redirect outside JupyterHub: %r", next_url)
             next_url = ''
 
@@ -774,13 +777,22 @@ class BaseHandler(RequestHandler):
         # always ensure default roles ('user', 'admin' if admin) are assigned
         # after a successful login
         roles.assign_default_roles(self.db, entity=user)
+
+        # apply authenticator-managed groups
+        if self.authenticator.manage_groups:
+            group_names = authenticated.get("groups")
+            if group_names is not None:
+                user.sync_groups(group_names)
+
         # always set auth_state and commit,
         # because there could be key-rotation or clearing of previous values
         # going on.
         if not self.authenticator.enable_auth_state:
             # auth_state is not enabled. Force None.
             auth_state = None
+
         await user.save_auth_state(auth_state)
+
         return user
 
     async def login_user(self, data=None):
@@ -794,6 +806,7 @@ class BaseHandler(RequestHandler):
             self.set_login_cookie(user)
             self.statsd.incr('login.success')
             self.statsd.timing('login.authenticate.success', auth_timer.ms)
+
             self.log.info("User logged in: %s", user.name)
             user._auth_refreshed = time.monotonic()
             return user
@@ -843,6 +856,12 @@ class BaseHandler(RequestHandler):
         user_server_name = user.name
 
         if server_name:
+            if '/' in server_name:
+                error_message = (
+                    f"Invalid server_name (may not contain '/'): {server_name}"
+                )
+                self.log.error(error_message)
+                raise web.HTTPError(400, error_message)
             user_server_name = f'{user.name}:{server_name}'
 
         if server_name in user.spawners and user.spawners[server_name].pending:
@@ -1501,6 +1520,7 @@ class UserUrlHandler(BaseHandler):
                 server_name = ''
         else:
             server_name = ''
+        escaped_server_name = url_escape_path(server_name)
         spawner = user.spawners[server_name]
 
         if spawner.ready:
@@ -1510,20 +1530,19 @@ class UserUrlHandler(BaseHandler):
 
         # if request is expecting JSON, assume it's an API request and fail with 503
         # because it won't like the redirect to the pending page
-        if (
-            get_accepted_mimetype(
-                self.request.headers.get('Accept', ''),
-                choices=['application/json', 'text/html'],
-            )
-            == 'application/json'
-            or 'api' in user_path.split('/')
-        ):
+        if get_accepted_mimetype(
+            self.request.headers.get('Accept', ''),
+            choices=['application/json', 'text/html'],
+        ) == 'application/json' or 'api' in user_path.split('/'):
             self._fail_api_request(user_name, server_name)
             return
 
         pending_url = url_concat(
             url_path_join(
-                self.hub.base_url, 'spawn-pending', user.escaped_name, server_name
+                self.hub.base_url,
+                'spawn-pending',
+                user.escaped_name,
+                escaped_server_name,
             ),
             {'next': self.request.uri},
         )
@@ -1537,7 +1556,9 @@ class UserUrlHandler(BaseHandler):
         # page *in* the server is not found, we return a 424 instead of a 404.
         # We allow retaining the old behavior to support older JupyterLab versions
         spawn_url = url_concat(
-            url_path_join(self.hub.base_url, "spawn", user.escaped_name, server_name),
+            url_path_join(
+                self.hub.base_url, "spawn", user.escaped_name, escaped_server_name
+            ),
             {"next": self.request.uri},
         )
         self.set_status(
@@ -1599,7 +1620,7 @@ class UserUrlHandler(BaseHandler):
         if redirects:
             self.log.warning("Redirect loop detected on %s", self.request.uri)
             # add capped exponential backoff where cap is 10s
-            await asyncio.sleep(min(1 * (2 ** redirects), 10))
+            await asyncio.sleep(min(1 * (2**redirects), 10))
             # rewrite target url with new `redirects` query value
             url_parts = urlparse(target)
             query_parts = parse_qs(url_parts.query)

jupyterhub/handlers/metrics.py

@@ -1,7 +1,5 @@
 """Handlers for serving prometheus metrics"""
-from prometheus_client import CONTENT_TYPE_LATEST
-from prometheus_client import generate_latest
-from prometheus_client import REGISTRY
+from prometheus_client import CONTENT_TYPE_LATEST, REGISTRY, generate_latest
 
 from ..utils import metrics_authentication
 from .base import BaseHandler

jupyterhub/handlers/pages.py

@@ -12,11 +12,9 @@ from tornado import web
 from tornado.httputil import url_concat
 
 from .. import __version__
-from ..metrics import SERVER_POLL_DURATION_SECONDS
-from ..metrics import ServerPollStatus
+from ..metrics import SERVER_POLL_DURATION_SECONDS, ServerPollStatus
 from ..scopes import needs_scope
-from ..utils import maybe_future
-from ..utils import url_path_join
+from ..utils import maybe_future, url_escape_path, url_path_join
 from .base import BaseHandler
 
 
@@ -151,7 +149,7 @@ class SpawnHandler(BaseHandler):
             self.redirect(url)
             return
 
-        spawner = user.spawners[server_name]
+        spawner = user.get_spawner(server_name, replace_failed=True)
 
         pending_url = self._get_pending_url(user, server_name)
 
@@ -237,7 +235,7 @@ class SpawnHandler(BaseHandler):
             if user is None:
                 raise web.HTTPError(404, "No such user: %s" % for_user)
 
-        spawner = user.spawners[server_name]
+        spawner = user.get_spawner(server_name, replace_failed=True)
 
         if spawner.ready:
             raise web.HTTPError(400, "%s is already running" % (spawner._log_name))
@@ -255,7 +253,7 @@ class SpawnHandler(BaseHandler):
             self.log.debug(
                 "Triggering spawn with supplied form options for %s", spawner._log_name
             )
-            options = await maybe_future(spawner.options_from_form(form_options))
+            options = await maybe_future(spawner.run_options_from_form(form_options))
             pending_url = self._get_pending_url(user, server_name)
             return await self._wrap_spawn_single_user(
                 user, server_name, spawner, pending_url, options
@@ -270,15 +268,6 @@ class SpawnHandler(BaseHandler):
             )
             self.finish(form)
             return
-        if current_user is user:
-            self.set_login_cookie(user)
-        next_url = self.get_next_url(
-            user,
-            default=url_path_join(
-                self.hub.base_url, "spawn-pending", user.escaped_name, server_name
-            ),
-        )
-        self.redirect(next_url)
 
     def _get_pending_url(self, user, server_name):
         # resolve `?next=...`, falling back on the spawn-pending url
@@ -286,7 +275,10 @@ class SpawnHandler(BaseHandler):
         # which may get handled by the default server if they aren't ready yet
 
         pending_url = url_path_join(
-            self.hub.base_url, "spawn-pending", user.escaped_name, server_name
+            self.hub.base_url,
+            "spawn-pending",
+            user.escaped_name,
+            url_escape_path(server_name),
         )
 
         pending_url = self.append_query_parameters(pending_url, exclude=['next'])
@@ -355,6 +347,7 @@ class SpawnPendingHandler(BaseHandler):
         if server_name and server_name not in user.spawners:
             raise web.HTTPError(404, f"{user.name} has no such server {server_name}")
 
+        escaped_server_name = url_escape_path(server_name)
         spawner = user.spawners[server_name]
 
         if spawner.ready:
@@ -369,19 +362,15 @@ class SpawnPendingHandler(BaseHandler):
         auth_state = await user.get_auth_state()
 
         # First, check for previous failure.
-        if (
-            not spawner.active
-            and spawner._spawn_future
-            and spawner._spawn_future.done()
-            and spawner._spawn_future.exception()
-        ):
-            # Condition: spawner not active and _spawn_future exists and contains an Exception
+        if not spawner.active and spawner._failed:
+            # Condition: spawner not active and last spawn failed
+            # (failure is available as spawner._spawn_future.exception()).
             # Implicit spawn on /user/:name is not allowed if the user's last spawn failed.
             # We should point the user to Home if the most recent spawn failed.
             exc = spawner._spawn_future.exception()
             self.log.error("Previous spawn for %s failed: %s", spawner._log_name, exc)
             spawn_url = url_path_join(
-                self.hub.base_url, "spawn", user.escaped_name, server_name
+                self.hub.base_url, "spawn", user.escaped_name, escaped_server_name
             )
             self.set_status(500)
             html = await self.render_template(
@@ -434,7 +423,7 @@ class SpawnPendingHandler(BaseHandler):
         # serving the expected page
         if status is not None:
             spawn_url = url_path_join(
-                self.hub.base_url, "spawn", user.escaped_name, server_name
+                self.hub.base_url, "spawn", user.escaped_name, escaped_server_name
             )
             html = await self.render_template(
                 "not_running.html",
@@ -460,15 +449,14 @@ class AdminHandler(BaseHandler):
     @web.authenticated
     # stacked decorators: all scopes must be present
     # note: keep in sync with admin link condition in page.html
-    @needs_scope('admin:users')
-    @needs_scope('admin:servers')
+    @needs_scope('admin-ui')
     async def get(self):
         auth_state = await self.current_user.get_auth_state()
         html = await self.render_template(
             'admin.html',
             current_user=self.current_user,
             auth_state=auth_state,
-            admin_access=self.settings.get('admin_access', False),
+            admin_access=True,
             allow_named_servers=self.allow_named_servers,
             named_server_limit_per_user=self.named_server_limit_per_user,
             server_version=f'{__version__} {self.version_hash}',
@@ -502,7 +490,7 @@ class TokenPageHandler(BaseHandler):
                 continue
             if not token.client_id:
                 # token should have been deleted when client was deleted
-                self.log.warning("Deleting stale oauth token {token}")
+                self.log.warning(f"Deleting stale oauth token {token}")
                 self.db.delete(token)
                 self.db.commit()
                 continue

jupyterhub/log.py

@@ -6,13 +6,10 @@ import logging
 import traceback
 from functools import partial
 from http.cookies import SimpleCookie
-from urllib.parse import urlparse
-from urllib.parse import urlunparse
+from urllib.parse import urlparse, urlunparse
 
-from tornado.log import access_log
-from tornado.log import LogFormatter
-from tornado.web import HTTPError
-from tornado.web import StaticFileHandler
+from tornado.log import LogFormatter, access_log
+from tornado.web import HTTPError, StaticFileHandler
 
 from .handlers.pages import HealthCheckHandler
 from .metrics import prometheus_log_method

jupyterhub/metrics.py

@@ -21,8 +21,7 @@ them manually here.
 """
 from enum import Enum
 
-from prometheus_client import Gauge
-from prometheus_client import Histogram
+from prometheus_client import Gauge, Histogram
 
 REQUEST_DURATION_SECONDS = Histogram(
     'jupyterhub_request_duration_seconds',