Merge main into query-performance

.github/workflows/release.yml

@@ -150,7 +150,7 @@ jobs:
           branchRegex: ^\w[\w-.]*$
 
       - name: Build and push jupyterhub
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        uses: docker/build-push-action@v4
         with:
           context: .
           platforms: linux/amd64,linux/arm64
@@ -171,7 +171,7 @@ jobs:
           branchRegex: ^\w[\w-.]*$
 
       - name: Build and push jupyterhub-onbuild
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        uses: docker/build-push-action@v4
         with:
           build-args: |
             BASE_IMAGE=${{ fromJson(steps.jupyterhubtags.outputs.tags)[0] }}
@@ -192,7 +192,7 @@ jobs:
           branchRegex: ^\w[\w-.]*$
 
       - name: Build and push jupyterhub-demo
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        uses: docker/build-push-action@v4
         with:
           build-args: |
             BASE_IMAGE=${{ fromJson(steps.onbuildtags.outputs.tags)[0] }}
@@ -216,7 +216,7 @@ jobs:
           branchRegex: ^\w[\w-.]*$
 
       - name: Build and push jupyterhub/singleuser
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        uses: docker/build-push-action@v4
         with:
           build-args: |
             JUPYTERHUB_VERSION=${{ github.ref_type == 'tag' && github.ref_name || format('git:{0}', github.sha) }}

.pre-commit-config.yaml

@@ -16,7 +16,7 @@ ci:
 repos:
   # Autoformat: Python code, syntax patterns are modernized
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.4.0
+    rev: v3.10.1
     hooks:
       - id: pyupgrade
         args:
@@ -24,7 +24,7 @@ repos:
 
   # Autoformat: Python code
   - repo: https://github.com/PyCQA/autoflake
-    rev: v2.1.1
+    rev: v2.2.0
     hooks:
       - id: autoflake
         # args ref: https://github.com/PyCQA/autoflake#advanced-usage
@@ -39,13 +39,13 @@ repos:
 
   # Autoformat: Python code
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 23.7.0
     hooks:
       - id: black
 
   # Autoformat: markdown, yaml, javascript (see the file .prettierignore)
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.0.0-alpha.9-for-vscode
+    rev: v3.0.0
     hooks:
       - id: prettier
 
@@ -61,6 +61,6 @@ repos:
 
   # Linting: Python code (see the file .flake8)
   - repo: https://github.com/PyCQA/flake8
-    rev: "6.0.0"
+    rev: "6.1.0"
     hooks:
       - id: flake8

ci/init-db.sh

@@ -21,7 +21,7 @@ fi
 # Configure a set of databases in the database server for upgrade tests
 # this list must be in sync with versions in test_db.py:test_upgrade
 set -x
-for SUFFIX in '' _upgrade_110 _upgrade_122 _upgrade_130 _upgrade_150 _upgrade_211; do
+for SUFFIX in '' _upgrade_110 _upgrade_122 _upgrade_130 _upgrade_150 _upgrade_211 _upgrade_311; do
     $SQL_CLIENT "DROP DATABASE jupyterhub${SUFFIX};" 2>/dev/null || true
     $SQL_CLIENT "CREATE DATABASE jupyterhub${SUFFIX} ${EXTRA_CREATE_DATABASE_ARGS:-};"
 done

docs/source/_static/rest-api.yml

@@ -1498,6 +1498,9 @@ components:
             read:groups: Read group models.
             read:groups:name: Read group names.
             delete:groups: Delete groups.
+            admin:services:
+              Create, read, update, delete services, not including services
+              defined from config files.
             list:services: List services, including at least their names.
             read:services: Read service models.
             read:services:name: Read service names.

docs/source/contributing/setup.md

@@ -173,3 +173,46 @@ python3 setup.py js    # fetch updated client-side js
 python3 setup.py css   # recompile CSS from LESS sources
 python3 setup.py jsx   # build React admin app
 ```
+
+### Failed to bind XXX to `http://127.0.0.1:<port>/<path>`
+
+This error can happen when there's already an application or a service using this
+port.
+
+Use the following command to find out which service is using this port.
+
+```bash
+lsof -P -i TCP:<port> -sTCP:LISTEN
+```
+
+If nothing shows up, it likely means there's a system service that uses it but
+your current user cannot list it. Reuse the same command with sudo.
+
+```bash
+sudo lsof -P -i TCP:<port> -sTCP:LISTEN
+```
+
+Depending on the result of the above commands, the most simple solution is to
+configure JupyterHub to use a different port for the service that is failing.
+
+As an example, the following is a frequently seen issue:
+
+`Failed to bind hub to http://127.0.0.1:8081/hub/`
+
+Using the procedure described above, start with:
+
+```bash
+lsof -P -i TCP:8081 -sTCP:LISTEN
+```
+
+and if nothing shows up:
+
+```bash
+sudo lsof -P -i TCP:8081 -sTCP:LISTEN
+```
+
+Finally, depending on your findings, you can apply the following change and start JupyterHub again:
+
+```python
+c.JupyterHub.hub_port = 9081 # Or any other free port
+```

docs/source/explanation/database.md

@@ -145,7 +145,7 @@ There is additional configuration required for MySQL that is not needed for Post
 
 For example, to connect to a postgres database with psycopg2:
 
-1. install psycopg2: `pip instal psycopg2` (or `psycopg2-binary` to avoid compilation, which is [not recommended for production][psycopg2-binary])
+1. install psycopg2: `pip install psycopg2` (or `psycopg2-binary` to avoid compilation, which is [not recommended for production][psycopg2-binary])
 2. set authentication via environment variables `PGUSER` and `PGPASSWORD`
 3. configure [](JupyterHub.db_url):
 

docs/source/howto/configuration/config-user-env.md

@@ -45,7 +45,7 @@ additional packages.
 
 ## Configuring Jupyter and IPython
 
-[Jupyter](https://jupyter-notebook.readthedocs.io/en/stable/config_overview.html)
+[Jupyter](https://jupyter-notebook.readthedocs.io/en/stable/configuring/config_overview.html)
 and [IPython](https://ipython.readthedocs.io/en/stable/development/config.html)
 have their own configuration systems.
 
@@ -212,13 +212,31 @@ By default, the single-user server launches JupyterLab,
 which is based on [Jupyter Server][].
 
 This is the default server when running JupyterHub ≥ 2.0.
-To switch to using the legacy Jupyter Notebook server, you can set the `JUPYTERHUB_SINGLEUSER_APP` environment variable
+To switch to using the legacy Jupyter Notebook server (notebook < 7.0), you can set the `JUPYTERHUB_SINGLEUSER_APP` environment variable
 (in the single-user environment) to:
 
 ```bash
 export JUPYTERHUB_SINGLEUSER_APP='notebook.notebookapp.NotebookApp'
 ```
 
+:::{note}
+
+```
+JUPYTERHUB_SINGLEUSER_APP='notebook.notebookapp.NotebookApp'
+```
+
+is only valid for notebook < 7. notebook v7 is based on jupyter-server,
+and the default jupyter-server application must be used.
+Selecting the new notebook UI is no longer a matter of selecting the server app to launch,
+but only the default URL for users to visit.
+To use notebook v7 with JupyterHub, leave the default singleuser app config alone (or specify `JUPYTERHUB_SINGLEUSER_APP=jupyter-server`) and set the default _URL_ for user servers:
+
+```python
+c.Spawner.default_url = '/tree/'
+```
+
+:::
+
 [jupyter server]: https://jupyter-server.readthedocs.io
 [jupyter notebook]: https://jupyter-notebook.readthedocs.io
 

docs/source/howto/rest.md

@@ -24,6 +24,7 @@ such as:
 
 - Checking which users are active
 - Adding or removing users
+- Adding or removing services
 - Stopping or starting single user notebook servers
 - Authenticating services
 - Communicating with an individual Jupyter server's REST API

docs/source/reference/services.md

@@ -174,6 +174,47 @@ c.JupyterHub.services = [
 In this case, the `url` field will be passed along to the Service as
 `JUPYTERHUB_SERVICE_URL`.
 
+## Adding or removing services at runtime
+
+Only externally-managed services can be added at runtime by using JupyterHub’s REST API.
+
+### Add a new service
+
+To add a new service, send a POST request to this endpoint
+
+```
+POST /hub/api/services/:servicename
+```
+
+**Required scope: `admin:services`**
+
+**Payload**: The payload should contain the definition of the service to be created. The endpoint supports the same properties as externally-managed services defined in the config file.
+
+**Possible responses**
+
+- `201 Created`: The service and related objects are created (and started in case of a Hub-managed one) successfully.
+- `400 Bad Request`: The payload is invalid or JupyterHub can not create the service.
+- `409 Conflict`: The service with the same name already exists.
+
+### Remove an existing service
+
+To remove an existing service, send a DELETE request to this endpoint
+
+```
+DELETE /hub/api/services/:servicename
+```
+
+**Required scope: `admin:services`**
+
+**Payload**: `None`
+
+**Possible responses**
+
+- `200 OK`: The service and related objects are removed (and stopped in case of a Hub-managed one) successfully.
+- `400 Bad Request`: JupyterHub can not remove the service.
+- `404 Not Found`: The requested service does not exist.
+- `405 Not Allowed`: The requested service is created from the config file, it can not be removed at runtime.
+
 ## Writing your own Services
 
 When writing your own services, you have a few decisions to make (in addition

docs/source/tutorial/getting-started/authenticators-users-basics.md

@@ -22,6 +22,21 @@ started.
 If this configuration value is not set, then **all authenticated users will be allowed into your hub**.
 ```
 
+## One Time Passwords ( request_otp )
+
+By setting `request_otp` to true, the login screen will show and additional password input field
+to accept an OTP:
+
+```python
+c.Authenticator.request_otp = True
+```
+
+By default, the prompt label is `OTP:`, but this can be changed by setting `otp_prompt`:
+
+```python
+c.Authenticator.otp_prompt = 'Google Authenticator:'
+```
+
 ## Configure admins (`admin_users`)
 
 ```{note}

jsx/src/components/ServerDashboard/ServerDashboard.jsx

@@ -21,14 +21,6 @@ import "./server-dashboard.css";
 import { timeSince } from "../../util/timeSince";
 import PaginationFooter from "../PaginationFooter/PaginationFooter";
 
-const AccessServerButton = ({ url }) => (
-  <a href={url || ""}>
-    <button className="btn btn-primary btn-xs" style={{ marginRight: 20 }}>
-      Access Server
-    </button>
-  </a>
-);
-
 const RowListItem = ({ text }) => (
   <span className="server-dashboard-row-list-item">{text}</span>
 );
@@ -56,7 +48,6 @@ const ServerDashboard = (props) => {
 
   var [errorAlert, setErrorAlert] = useState(null);
   var [sortMethod, setSortMethod] = useState(null);
-  var [disabledButtons, setDisabledButtons] = useState({});
   var [collapseStates, setCollapseStates] = useState({});
 
   var user_data = useSelector((state) => state.user_data),
@@ -128,15 +119,15 @@ const ServerDashboard = (props) => {
     user_data = sortMethod(user_data);
   }
 
-  const StopServerButton = ({ serverName, userName }) => {
+  const ServerButton = ({ server, user, action, name, extraClass }) => {
     var [isDisabled, setIsDisabled] = useState(false);
     return (
       <button
-        className="btn btn-danger btn-xs stop-button"
-        disabled={isDisabled}
+        className={`btn btn-xs ${extraClass}`}
+        disabled={isDisabled || server.pending}
         onClick={() => {
           setIsDisabled(true);
-          stopServer(userName, serverName)
+          action(user.name, server.name)
             .then((res) => {
               if (res.status < 300) {
                 updateUsers(...slice)
@@ -152,103 +143,87 @@ const ServerDashboard = (props) => {
                     setErrorAlert(`Failed to update users list.`);
                   });
               } else {
-                setErrorAlert(`Failed to stop server.`);
+                setErrorAlert(`Failed to ${name.toLowerCase()}.`);
                 setIsDisabled(false);
               }
               return res;
             })
             .catch(() => {
-              setErrorAlert(`Failed to stop server.`);
+              setErrorAlert(`Failed to ${name.toLowerCase()}.`);
               setIsDisabled(false);
             });
         }}
       >
-        Stop Server
+        {name}
       </button>
     );
   };
 
-  const DeleteServerButton = ({ serverName, userName }) => {
-    if (serverName === "") {
+  const StopServerButton = ({ server, user }) => {
+    if (!server.ready) {
+      return null;
+    }
+    return ServerButton({
+      server,
+      user,
+      action: stopServer,
+      name: "Stop Server",
+      extraClass: "btn-danger stop-button",
+    });
+  };
+  const DeleteServerButton = ({ server, user }) => {
+    if (server.name === "") {
+      // It's not possible to delete unnamed servers
+      return null;
+    }
+    if (server.ready || server.pending) {
+      return null;
+    }
+    return ServerButton({
+      server,
+      user,
+      action: deleteServer,
+      name: "Delete Server",
+      extraClass: "btn-danger stop-button",
+    });
+  };
+
+  const StartServerButton = ({ server, user }) => {
+    if (server.ready) {
+      return null;
+    }
+    return ServerButton({
+      server,
+      user,
+      action: startServer,
+      name: server.pending ? "Server is pending" : "Start Server",
+      extraClass: "btn-success start-button",
+    });
+  };
+
+  const SpawnPageButton = ({ server, user }) => {
+    if (server.ready) {
       return null;
     }
-    var [isDisabled, setIsDisabled] = useState(false);
     return (
-      <button
-        className="btn btn-danger btn-xs stop-button"
-        // It's not possible to delete unnamed servers
-        disabled={isDisabled}
-        onClick={() => {
-          setIsDisabled(true);
-          deleteServer(userName, serverName)
-            .then((res) => {
-              if (res.status < 300) {
-                updateUsers(...slice)
-                  .then((data) => {
-                    dispatchPageUpdate(
-                      data.items,
-                      data._pagination,
-                      name_filter,
-                    );
-                  })
-                  .catch(() => {
-                    setIsDisabled(false);
-                    setErrorAlert(`Failed to update users list.`);
-                  });
-              } else {
-                setErrorAlert(`Failed to delete server.`);
-                setIsDisabled(false);
-              }
-              return res;
-            })
-            .catch(() => {
-              setErrorAlert(`Failed to delete server.`);
-              setIsDisabled(false);
-            });
-        }}
+      <a
+        href={`${base_url}spawn/${user.name}${
+          server.name ? "/" + server.name : ""
+        }`}
       >
-        Delete Server
-      </button>
+        <button className="btn btn-light btn-xs">Spawn Page</button>
+      </a>
     );
   };
 
-  const StartServerButton = ({ serverName, userName }) => {
-    var [isDisabled, setIsDisabled] = useState(false);
+  const AccessServerButton = ({ server }) => {
+    if (!server.ready) {
+      return null;
+    }
     return (
-      <button
-        className="btn btn-success btn-xs start-button"
-        disabled={isDisabled}
-        onClick={() => {
-          setIsDisabled(true);
-          startServer(userName, serverName)
-            .then((res) => {
-              if (res.status < 300) {
-                updateUsers(...slice)
-                  .then((data) => {
-                    dispatchPageUpdate(
-                      data.items,
-                      data._pagination,
-                      name_filter,
-                    );
-                  })
-                  .catch(() => {
-                    setErrorAlert(`Failed to update users list.`);
-                    setIsDisabled(false);
-                  });
-              } else {
-                setErrorAlert(`Failed to start server.`);
-                setIsDisabled(false);
-              }
-              return res;
-            })
-            .catch(() => {
-              setErrorAlert(`Failed to start server.`);
-              setIsDisabled(false);
-            });
-        }}
-      >
-        Start Server
-      </button>
+      <a href={server.url || ""}>
+        <button className="btn btn-primary btn-xs">Access Server</button>
+      </a>
     );
   };
 
@@ -358,43 +333,16 @@ const ServerDashboard = (props) => {
         <td data-testid="user-row-last-activity">
           {server.last_activity ? timeSince(server.last_activity) : "Never"}
         </td>
-        <td data-testid="user-row-server-activity">
-          {server.ready ? (
-            // Stop Single-user server
-            <>
-              <StopServerButton serverName={server.name} userName={user.name} />
-              <AccessServerButton url={server.url} />
-            </>
-          ) : (
-            // Start Single-user server
-            <>
-              <StartServerButton
-                serverName={server.name}
-                userName={user.name}
-                style={{ marginRight: 20 }}
-              />
-              <DeleteServerButton
-                serverName={server.name}
-                userName={user.name}
-              />
-              <a
-                href={`${base_url}spawn/${user.name}${
-                  server.name ? "/" + server.name : ""
-                }`}
-              >
-                <button
-                  className="btn btn-secondary btn-xs"
-                  style={{ marginRight: 20 }}
-                >
-                  Spawn Page
-                </button>
-              </a>
-            </>
-          )}
+        <td data-testid="user-row-server-activity" className="actions">
+          <StartServerButton server={server} user={user} />
+          <StopServerButton server={server} user={user} />
+          <DeleteServerButton server={server} user={user} />
+          <AccessServerButton server={server} />
+          <SpawnPageButton server={server} user={user} />
         </td>
         <EditUserCell user={user} />
       </tr>,
-      <tr>
+      <tr key={`${userServerName}-detail`}>
         <td
           colSpan={6}
           style={{ padding: 0 }}
@@ -514,9 +462,11 @@ const ServerDashboard = (props) => {
           <tbody>
             <tr className="noborder">
               <td>
-                <Button variant="light" className="add-users-button">
-                  <Link to="/add-users">Add Users</Link>
-                </Button>
+                <Link to="/add-users">
+                  <Button variant="light" className="add-users-button">
+                    Add Users
+                  </Button>
+                </Link>
               </td>
               <td></td>
               <td></td>
@@ -611,6 +561,7 @@ const ServerDashboard = (props) => {
                   Shutdown Hub
                 </Button>
               </td>
+              <td></td>
             </tr>
             {servers.flatMap(([user, server]) => serverRow(user, server))}
           </tbody>

jsx/src/components/ServerDashboard/ServerDashboard.test.js

@@ -2,7 +2,13 @@ import React from "react";
 import "@testing-library/jest-dom";
 import { act } from "react-dom/test-utils";
 import userEvent from "@testing-library/user-event";
-import { render, screen, fireEvent, getByText } from "@testing-library/react";
+import {
+  render,
+  screen,
+  fireEvent,
+  getByText,
+  getAllByRole,
+} from "@testing-library/react";
 import { HashRouter, Switch } from "react-router-dom";
 import { Provider, useSelector } from "react-redux";
 import { createStore } from "redux";
@@ -697,3 +703,59 @@ test("Server delete button exists for named servers", async () => {
     expect(delete_button).toBeEnabled();
   }
 });
+
+test("Start server and confirm pending state", async () => {
+  let spy = mockAsync();
+
+  let mockStartServer = jest.fn(() => {
+    return new Promise(async (resolve) =>
+      clock.setTimeout(() => {
+        resolve({ status: 200 });
+      }, 100),
+    );
+  });
+
+  let mockUpdateUsers = jest.fn(() => Promise.resolve(mockAppState()));
+
+  await act(async () => {
+    render(
+      <Provider store={createStore(mockReducers, {})}>
+        <HashRouter>
+          <Switch>
+            <ServerDashboard
+              updateUsers={mockUpdateUsers}
+              shutdownHub={spy}
+              startServer={mockStartServer}
+              stopServer={spy}
+              startAll={spy}
+              stopAll={spy}
+            />
+          </Switch>
+        </HashRouter>
+      </Provider>,
+    );
+  });
+
+  let actions = screen.getAllByTestId("user-row-server-activity")[1];
+  let buttons = getAllByRole(actions, "button");
+
+  expect(buttons.length).toBe(2);
+  expect(buttons[0].textContent).toBe("Start Server");
+  expect(buttons[1].textContent).toBe("Spawn Page");
+
+  await act(async () => {
+    fireEvent.click(buttons[0]);
+  });
+  expect(mockUpdateUsers.mock.calls).toHaveLength(1);
+
+  expect(buttons.length).toBe(2);
+  expect(buttons[0].textContent).toBe("Start Server");
+  expect(buttons[0]).toBeDisabled();
+  expect(buttons[1].textContent).toBe("Spawn Page");
+  expect(buttons[1]).toBeEnabled();
+
+  await act(async () => {
+    await clock.tick(100);
+  });
+  expect(mockUpdateUsers.mock.calls).toHaveLength(2);
+});

jsx/src/components/ServerDashboard/server-dashboard.css

@@ -7,7 +7,7 @@
   margin-left: auto;
 }
 
-.server-dashboard-container .add-users-button {
+.server-dashboard-container .btn-light {
   border: 1px solid #ddd;
 }
 
@@ -38,3 +38,11 @@ tr.noborder > td {
   border: 1px solid #ddd;
   border-radius: 2px;
 }
+
+.table > tbody > tr.user-row > td {
+  vertical-align: inherit;
+}
+
+.user-row .actions > * {
+  margin-right: 5px;
+}

jsx/testing/group.json

@@ -0,0 +1,7 @@
+{
+  "items": [
+    { "kind": "group", "name": "testgroup", "users": [] },
+    { "kind": "group", "name": "testgroup2", "users": ["foo", "bar"] }
+  ],
+  "_pagination": { "offset": 0, "limit": 50, "total": 2, "next": null }
+}

jsx/testing/index.html

@@ -0,0 +1,21 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+
+    <title>JupyterHub</title>
+    <meta http-equiv="X-UA-Compatible" content="chrome=1" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+
+    <link rel="stylesheet" href="/static/css/style.min.css" type="text/css" />
+  </head>
+  <body>
+    <div id="react-admin-hook">
+      <script id="jupyterhub-admin-config">
+        window.api_page_limit = parseInt("50");
+        window.base_url = "/";
+      </script>
+      <script src="admin-react.js"></script>
+    </div>
+  </body>
+</html>

jsx/testing/user.json

@@ -0,0 +1,142 @@
+{
+  "items": [
+    {
+      "last_activity": "2022-08-04T23:01:40.770831Z",
+      "groups": [],
+      "created": "2022-08-04T23:01:15.074531Z",
+      "roles": ["user"],
+      "auth_state": null,
+      "pending": null,
+      "kind": "user",
+      "server": null,
+      "name": "userA",
+      "servers": {
+        "": {
+          "name": "",
+          "last_activity": "2022-08-04T23:01:40.770831Z",
+          "started": null,
+          "pending": null,
+          "ready": false,
+          "stopped": true,
+          "url": "/user/usera/",
+          "user_options": null,
+          "progress_url": "/hub/api/users/usera/server/progress",
+          "state": {}
+        }
+      }
+    },
+    {
+      "last_activity": "2022-08-05T16:43:44.442068Z",
+      "groups": [],
+      "admin": true,
+      "created": "2022-08-04T23:01:27.819148Z",
+      "roles": ["user", "admin"],
+      "auth_state": null,
+      "pending": null,
+      "kind": "user",
+      "server": null,
+      "name": "userB",
+      "servers": {
+        "": {
+          "name": "",
+          "last_activity": "2022-08-05T16:43:44.442068Z",
+          "started": null,
+          "pending": null,
+          "ready": true,
+          "stopped": false,
+          "url": "/user/userb/",
+          "user_options": null,
+          "progress_url": "/hub/api/users/userb/server/progress",
+          "state": {}
+        }
+      }
+    },
+    {
+      "last_activity": "2022-08-05T16:43:44.442068Z",
+      "groups": [],
+      "created": "2022-08-04T23:01:27.819148Z",
+      "roles": ["user"],
+      "auth_state": null,
+      "pending": "spawn",
+      "kind": "user",
+      "server": null,
+      "name": "userC",
+      "servers": {
+        "": {
+          "last_activity": "2023-06-11T16:22:02.228468Z",
+          "name": "",
+          "pending": "spawn",
+          "progress_url": "/hub/api/users/userc/server/progress",
+          "ready": false,
+          "started": "2023-06-11T16:22:02.228468Z",
+          "state": { "pid": 68137 },
+          "stopped": false,
+          "url": "/user/userc/",
+          "user_options": {}
+        }
+      }
+    },
+    {
+      "last_activity": "2023-06-11T15:19:49.786502Z",
+      "groups": [],
+      "created": "2023-05-14T09:05:48.996574Z",
+      "roles": ["user"],
+      "auth_state": null,
+      "pending": null,
+      "kind": "user",
+      "server": "/user/userD/",
+      "name": "userD",
+      "servers": {
+        "": {
+          "name": "",
+          "last_activity": "2023-06-11T13:39:27.017000Z",
+          "started": "2023-06-11T13:39:24.679829Z",
+          "pending": null,
+          "ready": true,
+          "stopped": false,
+          "url": "/user/userd/",
+          "user_options": {},
+          "progress_url": "/hub/api/users/userd/server/progress",
+          "state": { "pid": 41517 }
+        },
+        "serverA": {
+          "name": "serverA",
+          "last_activity": "2023-05-14T13:59:06.931642Z",
+          "started": null,
+          "pending": null,
+          "ready": false,
+          "stopped": true,
+          "url": "/user/userd/servera/",
+          "user_options": null,
+          "progress_url": "/hub/api/users/userd/servers/servera/progress",
+          "state": {}
+        },
+        "serverB": {
+          "last_activity": "2023-06-11T16:22:02.228468Z",
+          "name": "serverB",
+          "pending": "spawn",
+          "progress_url": "/hub/api/users/userb/servers/serverb/progress",
+          "ready": false,
+          "started": "2023-06-11T16:22:02.228468Z",
+          "state": { "pid": 68137 },
+          "stopped": false,
+          "url": "/user/userd/serverb/",
+          "user_options": {}
+        },
+        "serverC": {
+          "name": "serverC",
+          "last_activity": "2023-05-14T13:59:06.931642Z",
+          "started": null,
+          "pending": null,
+          "ready": true,
+          "stopped": false,
+          "url": "/user/userd/serverc/",
+          "user_options": null,
+          "progress_url": "/hub/api/users/userd/servers/serverc/progress",
+          "state": {}
+        }
+      }
+    }
+  ],
+  "_pagination": { "offset": 0, "limit": 50, "total": 4, "next": null }
+}

jsx/webpack.config.js

@@ -1,5 +1,7 @@
 const webpack = require("webpack");
 const path = require("path");
+const user_json = require("./testing/user.json");
+const group_json = require("./testing/group.json");
 
 module.exports = {
   entry: path.resolve(__dirname, "src", "App.jsx"),
@@ -33,31 +35,21 @@ module.exports = {
   },
   plugins: [new webpack.HotModuleReplacementPlugin()],
   devServer: {
-    static: {
-      directory: path.resolve(__dirname, "build"),
+    client: {
+      overlay: false,
     },
+    static: ["build", "testing", "../share/jupyterhub"],
     port: 9000,
     onBeforeSetupMiddleware: (devServer) => {
       const app = devServer.app;
 
-      var user_data = JSON.parse(
-        '[{"kind":"user","name":"foo","admin":true,"groups":[],"server":"/user/foo/","pending":null,"created":"2020-12-07T18:46:27.112695Z","last_activity":"2020-12-07T21:00:33.336354Z","servers":{"":{"name":"","last_activity":"2020-12-07T20:58:02.437408Z","started":"2020-12-07T20:58:01.508266Z","pending":null,"ready":true,"state":{"pid":28085},"url":"/user/foo/","user_options":{},"progress_url":"/hub/api/users/foo/server/progress"}}},{"kind":"user","name":"bar","admin":false,"groups":[],"server":null,"pending":null,"created":"2020-12-07T18:46:27.115528Z","last_activity":"2020-12-07T20:43:51.013613Z","servers":{}}]',
-      );
-      var group_data = JSON.parse(
-        '[{"kind":"group","name":"testgroup","users":[]}, {"kind":"group","name":"testgroup2","users":["foo", "bar"]}]',
-      );
-
       // get user_data
       app.get("/hub/api/users", (req, res) => {
-        res
-          .set("Content-Type", "application/json")
-          .send(JSON.stringify(user_data));
+        res.set("Content-Type", "application/json").send(user_json);
       });
       // get group_data
       app.get("/hub/api/groups", (req, res) => {
-        res
-          .set("Content-Type", "application/json")
-          .send(JSON.stringify(group_data));
+        res.set("Content-Type", "application/json").send(group_json);
       });
       // add users to group
       app.post("/hub/api/groups/*/users", (req, res) => {

jupyterhub/alembic/versions/3c2384c5aae1_add_from_config_column_to_the_services_.py

@@ -0,0 +1,51 @@
+"""Add from_config column to the services table
+
+Revision ID: 3c2384c5aae1
+Revises: 0eee8c825d24
+Create Date: 2023-02-27 16:22:26.196231
+"""
+
+# revision identifiers, used by Alembic.
+revision = '3c2384c5aae1'
+down_revision = '0eee8c825d24'
+branch_labels = None
+depends_on = None
+
+import sqlalchemy as sa
+from alembic import op
+
+from jupyterhub.orm import JSONDict, JSONList
+
+COL_DATA = [
+    {'name': 'url', 'type': sa.Unicode(length=2047)},
+    {'name': 'oauth_client_allowed_scopes', 'type': JSONDict()},
+    {'name': 'info', 'type': JSONDict()},
+    {'name': 'display', 'type': sa.Boolean},
+    {'name': 'oauth_no_confirm', 'type': sa.Boolean},
+    {'name': 'command', 'type': JSONList()},
+    {'name': 'cwd', 'type': sa.Unicode(length=2047)},
+    {'name': 'environment', 'type': JSONDict()},
+    {'name': 'user', 'type': sa.Unicode(255)},
+]
+
+
+def upgrade():
+    engine = op.get_bind().engine
+    tables = sa.inspect(engine).get_table_names()
+    if 'services' in tables:
+        op.add_column(
+            'services',
+            sa.Column('from_config', sa.Boolean, default=True),
+        )
+        op.execute('UPDATE services SET from_config = true')
+        for item in COL_DATA:
+            op.add_column(
+                'services',
+                sa.Column(item['name'], item['type'], nullable=True),
+            )
+
+
+def downgrade():
+    op.drop_column('services', sa.Column('from_config'))
+    for item in COL_DATA:
+        op.drop_column('services', sa.Column(item['name']))

jupyterhub/alembic/versions/651f5419b74d_api_token_scopes.py

@@ -13,16 +13,37 @@ depends_on = None
 
 import sqlalchemy as sa
 from alembic import op
-from sqlalchemy import Column, ForeignKey, Table
+from sqlalchemy import Column, ForeignKey, Table, text
 from sqlalchemy.orm import raiseload, relationship, selectinload
 from sqlalchemy.orm.session import Session
 
-from jupyterhub import orm, roles, scopes
+from jupyterhub import orm, roles
+
+
+def access_scopes(oauth_client: orm.OAuthClient, db: Session):
+    """Return scope(s) required to access an oauth client
+    This is a clone of `scopes.access_scopes` without using
+    the `orm.Service`
+    """
+    scopes = set()
+    if oauth_client.identifier == "jupyterhub":
+        return frozenset()
+    spawner = oauth_client.spawner
+    if spawner:
+        scopes.add(f"access:servers!server={spawner.user.name}/{spawner.name}")
+    else:
+        statement = "SELECT * FROM services WHERE oauth_client_id = :identifier"
+        service = db.execute(
+            text(statement), {"identifier": oauth_client.identifier}
+        ).fetchall()
+        if len(service) > 0:
+            scopes.add(f"access:services!service={service[0].name}")
+
+    return frozenset(scopes)
 
 
 def upgrade():
     c = op.get_bind()
-
     tables = sa.inspect(c.engine).get_table_names()
 
     # oauth codes are short lived, no need to upgrade them
@@ -103,7 +124,7 @@ def upgrade():
             db = Session(bind=c)
             for oauth_client in db.query(orm.OAuthClient):
                 allowed_scopes = set(roles.roles_to_scopes(oauth_client.allowed_roles))
-                allowed_scopes.update(scopes.access_scopes(oauth_client))
+                allowed_scopes.update(access_scopes(oauth_client, db))
                 oauth_client.allowed_scopes = sorted(allowed_scopes)
             db.commit()
             # drop token-role relationship

jupyterhub/apihandlers/base.py

@@ -421,6 +421,23 @@ class APIHandler(BaseHandler):
 
     _group_model_types = {'name': str, 'users': list, 'roles': list}
 
+    _service_model_types = {
+        'name': str,
+        'admin': bool,
+        'url': str,
+        'oauth_client_allowed_scopes': list,
+        'api_token': str,
+        'info': dict,
+        'display': bool,
+        'oauth_no_confirm': bool,
+        'command': list,
+        'cwd': str,
+        'environment': dict,
+        'user': str,
+        'oauth_client_id': str,
+        'oauth_redirect_uri': str,
+    }
+
     def _check_model(self, model, model_types, name):
         """Check a model provided by a REST API request
 
@@ -459,6 +476,15 @@ class APIHandler(BaseHandler):
                     400, ("group names must be str, not %r", type(groupname))
                 )
 
+    def _check_service_model(self, model):
+        """Check a request-provided service model from a REST API"""
+        self._check_model(model, self._service_model_types, 'service')
+        service_name = model.get('name')
+        if not isinstance(service_name, str):
+            raise web.HTTPError(
+                400, ("Service name must be str, not %r", type(service_name))
+            )
+
     def get_api_pagination(self):
         default_limit = self.settings["api_page_default_limit"]
         max_limit = self.settings["api_page_max_limit"]

jupyterhub/apihandlers/services.py

@@ -5,8 +5,14 @@ Currently GET-only, no actions can be taken to modify services.
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 import json
+from typing import Optional, Tuple
 
-from ..scopes import Scope, needs_scope
+from tornado import web
+
+from .. import orm
+from ..roles import get_default_roles
+from ..scopes import Scope, _check_token_scopes, needs_scope
+from ..services.service import Service
 from .base import APIHandler
 
 
@@ -25,9 +31,171 @@ class ServiceListAPIHandler(APIHandler):
 class ServiceAPIHandler(APIHandler):
     @needs_scope('read:services', 'read:services:name', 'read:roles:services')
     def get(self, service_name):
+        if service_name not in self.services:
+            raise web.HTTPError(404, f"No such service: {service_name}")
         service = self.services[service_name]
         self.write(json.dumps(self.service_model(service)))
 
+    def _check_service_scopes(self, spec: dict):
+        user = self.current_user
+        requested_scopes = []
+        if spec.get('admin'):
+            default_roles = get_default_roles()
+            admin_scopes = [
+                role['scopes'] for role in default_roles if role['name'] == 'admin'
+            ]
+            requested_scopes.extend(admin_scopes[0])
+
+        requested_client_scope = spec.get('oauth_client_allowed_scopes')
+        if requested_client_scope is not None:
+            requested_scopes.extend(requested_client_scope)
+        if len(requested_scopes) > 0:
+            try:
+                _check_token_scopes(requested_scopes, user, None)
+            except ValueError as e:
+                raise web.HTTPError(400, str(e))
+
+    async def add_service(self, spec: dict) -> Service:
+        """Add a new service and related objects to the database
+
+        Args:
+            spec (dict): The service specification
+
+        Raises:
+            web.HTTPError: Raise if the service is not created
+
+        Returns:
+            Service: Returns the service instance.
+
+        """
+
+        self._check_service_model(spec)
+        self._check_service_scopes(spec)
+
+        service_name = spec["name"]
+        managed = bool(spec.get('command'))
+        if managed:
+            msg = f"Can not create managed service {service_name} at runtime"
+            self.log.error(msg, exc_info=True)
+            raise web.HTTPError(400, msg)
+        try:
+            new_service = self.service_from_spec(spec)
+        except Exception:
+            msg = f"Failed to create service {service_name}"
+            self.log.error(msg, exc_info=True)
+            raise web.HTTPError(400, msg)
+
+        if new_service is None:
+            raise web.HTTPError(400, f"Failed to create service {service_name}")
+
+        if new_service.api_token:
+            # Add api token to database
+            await self.app._add_tokens(
+                {new_service.api_token: new_service.name}, kind='service'
+            )
+        if new_service.url:
+            # Start polling for external service
+            service_status = await self.app.start_service(service_name, new_service)
+            if not service_status:
+                self.log.error(
+                    'Failed to start service %s',
+                    service_name,
+                    exc_info=True,
+                )
+
+        if new_service.oauth_no_confirm:
+            oauth_no_confirm_list = self.settings.get('oauth_no_confirm_list')
+            msg = f"Allowing service {new_service.name} to complete OAuth without confirmation on an authorization web page"
+            self.log.warning(msg)
+            oauth_no_confirm_list.add(new_service.oauth_client_id)
+
+        return new_service
+
+    @needs_scope('admin:services')
+    async def post(self, service_name: str):
+        data = self.get_json_body()
+        service, _ = self.find_service(service_name)
+
+        if service is not None:
+            raise web.HTTPError(409, f"Service {service_name} already exists")
+
+        if not data or not isinstance(data, dict):
+            raise web.HTTPError(400, "Invalid service data")
+
+        data['name'] = service_name
+        new_service = await self.add_service(data)
+        self.write(json.dumps(self.service_model(new_service)))
+        self.set_status(201)
+
+    @needs_scope('admin:services')
+    async def delete(self, service_name: str):
+        service, orm_service = self.find_service(service_name)
+
+        if service is None:
+            raise web.HTTPError(404, f"Service {service_name} does not exist")
+
+        if service.from_config:
+            raise web.HTTPError(
+                405, f"Service {service_name} is not modifiable at runtime"
+            )
+        try:
+            await self.remove_service(service, orm_service)
+            self.services.pop(service_name)
+        except Exception:
+            msg = f"Failed to remove service {service_name}"
+            self.log.error(msg, exc_info=True)
+            raise web.HTTPError(400, msg)
+
+        self.set_status(200)
+
+    async def remove_service(self, service: Service, orm_service: orm.Service) -> None:
+        """Remove a service and all related objects from the database.
+
+        Args:
+            service (Service): the service object to be removed
+
+            orm_service (orm.Service): The `orm.Service` object linked
+            with `service`
+        """
+        if service.managed:
+            await service.stop()
+
+        if service.oauth_client:
+            self.oauth_provider.remove_client(service.oauth_client_id)
+
+        if orm_service._server_id is not None:
+            orm_server = (
+                self.db.query(orm.Server).filter_by(id=orm_service._server_id).first()
+            )
+            if orm_server is not None:
+                self.db.delete(orm_server)
+
+        if service.oauth_no_confirm:
+            oauth_no_confirm_list = self.settings.get('oauth_no_confirm_list')
+            oauth_no_confirm_list.discard(service.oauth_client_id)
+
+        self.db.delete(orm_service)
+        self.db.commit()
+
+    def service_from_spec(self, spec) -> Optional[Service]:
+        """Create service from api request"""
+        service = self.app.service_from_spec(spec, from_config=False)
+        self.db.commit()
+        return service
+
+    def find_service(
+        self, name: str
+    ) -> Tuple[Optional[Service], Optional[orm.Service]]:
+        """Get a service by name
+        return None if no such service
+        """
+        orm_service = orm.Service.find(db=self.db, name=name)
+        if orm_service is not None:
+            service = self.services.get(name)
+            return service, orm_service
+
+        return (None, None)
+
 
 default_handlers = [
     (r"/api/services", ServiceListAPIHandler),

jupyterhub/app.py

@@ -21,6 +21,7 @@ from functools import partial
 from getpass import getuser
 from operator import itemgetter
 from textwrap import dedent
+from typing import Optional
 from urllib.parse import unquote, urlparse, urlunparse
 
 if sys.version_info[:2] < (3, 3):
@@ -32,6 +33,7 @@ from dateutil.parser import parse as parse_date
 from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader
 from jupyter_telemetry.eventlog import EventLog
 from sqlalchemy.exc import OperationalError, SQLAlchemyError
+from sqlalchemy.orm import joinedload
 from tornado import gen, web
 from tornado.httpclient import AsyncHTTPClient
 from tornado.ioloop import IOLoop, PeriodicCallback
@@ -2250,22 +2252,53 @@ class JupyterHub(Application):
 
         db.commit()
         if self.authenticator.allowed_users:
-            self.log.debug(
-                f"Assigning {len(self.authenticator.allowed_users)} allowed_users to the user role"
-            )
-            allowed_users = db.query(orm.User).filter(
+            user_role = orm.Role.find(db, "user")
+            self.log.debug("Assigning allowed_users to the user role")
+            # query only those that need the user role _and don't have it_
+            needs_user_role = db.query(orm.User).filter(
                 orm.User.name.in_(self.authenticator.allowed_users)
+                & ~orm.User.roles.any(id=user_role.id)
             )
-            for user in allowed_users:
-                roles.grant_role(db, user, 'user')
+            if self.log.isEnabledFor(logging.DEBUG):
+                # filter on isEnabledFor to skip the extra `count()` query if we aren't going to log it
+                self.log.debug(
+                    f"Assigning {needs_user_role.count()} allowed_users to the user role"
+                )
+
+            for user in needs_user_role:
+                roles.grant_role(db, user, user_role)
+
         admin_role = orm.Role.find(db, 'admin')
         for kind in admin_role_objects:
             Class = orm.get_class(kind)
-            for admin_obj in db.query(Class).filter_by(admin=True):
+
+            # sync obj.admin with admin role
+            # query only those objects that do not match config
+            # to avoid expensive query for no-op updates
+
+            # always: in admin role sets admin = True
+            for is_admin in db.query(Class).filter(
+                (Class.admin == False) & Class.roles.any(id=admin_role.id)
+            ):
+                self.log.info(f"Setting admin=True on {is_admin}")
+                is_admin.admin = True
+
+            # iterate over users with admin=True
+            # who are not in the admin role.
+            for not_admin_obj in db.query(Class).filter(
+                (Class.admin == True) & ~Class.roles.any(id=admin_role.id)
+            ):
                 if has_admin_role_spec[kind]:
-                    admin_obj.admin = admin_role in admin_obj.roles
+                    # role membership specified exactly in config,
+                    # already populated above.
+                    # make sure user.admin matches admin role
+                    # setting .admin=False for anyone no longer in admin role
+                    self.log.warning(f"Removing admin=True from {not_admin_obj}")
+                    not_admin_obj.admin = False
                 else:
-                    roles.grant_role(db, admin_obj, 'admin')
+                    # no admin role membership declared,
+                    # populate admin role from admin attribute (the old way, only additive)
+                    roles.grant_role(db, not_admin_obj, admin_role)
         db.commit()
         # make sure that on hub upgrade, all users, services and tokens have at least one role (update with default)
         if getattr(self, '_rbac_upgrade', False):
@@ -2291,7 +2324,7 @@ class JupyterHub(Application):
                 if not self.authenticator.validate_username(name):
                     raise ValueError("Token user name %r is not valid" % name)
             if kind == 'service':
-                if not any(service["name"] == name for service in self.services):
+                if not any(service_name == name for service_name in self._service_map):
                     self.log.warning(
                         f"service {name} not in services, creating implicitly. It is recommended to register services using services list."
                     )
@@ -2354,8 +2387,20 @@ class JupyterHub(Application):
         )
         pc.start()
 
-    def init_services(self):
-        self._service_map.clear()
+    def service_from_orm(
+        self,
+        orm_service: orm.Service,
+    ) -> Service:
+        """Create the service instance and related objects from
+        ORM data.
+
+        Args:
+            orm_service (orm.Service): The `orm.Service` object
+
+        Returns:
+            Service: the created service
+        """
+
         if self.domain:
             domain = 'services.' + self.domain
             parsed = urlparse(self.subdomain_host)
@@ -2363,118 +2408,208 @@ class JupyterHub(Application):
         else:
             domain = host = ''
 
-        for spec in self.services:
-            if 'name' not in spec:
-                raise ValueError('service spec must have a name: %r' % spec)
-            name = spec['name']
-            # get/create orm
-            orm_service = orm.Service.find(self.db, name=name)
-            if orm_service is None:
-                # not found, create a new one
-                orm_service = orm.Service(name=name)
-                self.db.add(orm_service)
-                if spec.get('admin', False):
-                    self.log.warning(
-                        f"Service {name} sets `admin: True`, which is deprecated in JupyterHub 2.0."
-                        " You can assign now assign roles via `JupyterHub.load_roles` configuration."
-                        " If you specify services in the admin role configuration, "
-                        "the Service admin flag will be ignored."
+        name = orm_service.name
+        service = Service(
+            parent=self,
+            app=self,
+            base_url=self.base_url,
+            db=self.db,
+            orm=orm_service,
+            roles=orm_service.roles,
+            domain=domain,
+            host=host,
+            hub=self.hub,
+        )
+        traits = service.traits(input=True)
+        for key, trait in traits.items():
+            if not trait.metadata.get("in_db", True):
+                continue
+            orm_value = getattr(orm_service, key)
+            if orm_value is not None:
+                setattr(service, key, orm_value)
+
+        if orm_service.oauth_client is not None:
+            service.oauth_client_id = orm_service.oauth_client.identifier
+            service.oauth_redirect_uri = orm_service.oauth_client.redirect_uri
+
+        self._service_map[name] = service
+
+        return service
+
+    def service_from_spec(
+        self,
+        spec: Dict,
+        from_config=True,
+    ) -> Optional[Service]:
+        """Create the service instance and related objects from
+        config data.
+
+        Args:
+            spec (Dict): The spec of service, defined in the config file.
+            from_config (bool, optional): `True` if the service will be created
+            from the config file, `False` if it is created from REST API.
+            Defaults to `True`.
+
+        Returns:
+            Optional[Service]: The created service
+        """
+
+        if self.domain:
+            domain = 'services.' + self.domain
+            parsed = urlparse(self.subdomain_host)
+            host = f'{parsed.scheme}://services.{parsed.netloc}'
+        else:
+            domain = host = ''
+
+        if 'name' not in spec:
+            raise ValueError('service spec must have a name: %r' % spec)
+
+        name = spec['name']
+        # get/create orm
+        orm_service = orm.Service.find(self.db, name=name)
+        if orm_service is None:
+            # not found, create a new one
+            orm_service = orm.Service(name=name, from_config=from_config)
+            self.db.add(orm_service)
+            if spec.get('admin', False):
+                self.log.warning(
+                    f"Service {name} sets `admin: True`, which is deprecated in JupyterHub 2.0."
+                    " You can assign now assign roles via `JupyterHub.load_roles` configuration."
+                    " If you specify services in the admin role configuration, "
+                    "the Service admin flag will be ignored."
+                )
+                roles.update_roles(self.db, entity=orm_service, roles=['admin'])
+        else:
+            # Do nothing if the config file tries to modify a API-base service
+            # or vice versa.
+            if orm_service.from_config != from_config:
+                if from_config:
+                    self.log.error(
+                        f"The service {name} from the config file is trying to modify a runtime-created service with the same name"
                     )
-                    roles.update_roles(self.db, entity=orm_service, roles=['admin'])
-            orm_service.admin = spec.get('admin', False)
-            self.db.commit()
-            service = Service(
-                parent=self,
-                app=self,
-                base_url=self.base_url,
-                db=self.db,
-                orm=orm_service,
-                roles=orm_service.roles,
-                domain=domain,
-                host=host,
-                hub=self.hub,
+                else:
+                    self.log.error(
+                        f"The runtime-created service {name} is trying to modify a config-based service with the same name"
+                    )
+                return
+        orm_service.admin = spec.get('admin', False)
+
+        self.db.commit()
+        service = Service(
+            parent=self,
+            app=self,
+            base_url=self.base_url,
+            db=self.db,
+            orm=orm_service,
+            roles=orm_service.roles,
+            domain=domain,
+            host=host,
+            hub=self.hub,
+        )
+
+        traits = service.traits(input=True)
+        for key, value in spec.items():
+            trait = traits.get(key)
+            if trait is None:
+                raise AttributeError("No such service field: %s" % key)
+            setattr(service, key, value)
+            # also set the value on the orm object
+            # unless it's marked as not in the db
+            # (e.g. on the oauth object)
+            if trait.metadata.get("in_db", True):
+                setattr(orm_service, key, value)
+
+        if service.api_token:
+            self.service_tokens[service.api_token] = service.name
+        elif service.managed:
+            # generate new token
+            # TODO: revoke old tokens?
+            service.api_token = service.orm.new_api_token(note="generated at startup")
+
+        if service.url:
+            parsed = urlparse(service.url)
+            if parsed.scheme not in {"http", "https"}:
+                raise ValueError(
+                    f"Unsupported scheme in URL for service {name}: {service.url}. Must be http[s]"
+                )
+
+            port = None
+            if parsed.port is not None:
+                port = parsed.port
+            elif parsed.scheme == 'http':
+                port = 80
+            elif parsed.scheme == 'https':
+                port = 443
+
+            server = service.orm.server = orm.Server(
+                proto=parsed.scheme,
+                ip=parsed.hostname,
+                port=port,
+                cookie_name=service.oauth_client_id,
+                base_url=service.prefix,
             )
+            self.db.add(server)
+        else:
+            service.orm.server = None
 
-            traits = service.traits(input=True)
-            for key, value in spec.items():
-                if key not in traits:
-                    raise AttributeError("No such service field: %s" % key)
-                setattr(service, key, value)
-
-            if service.api_token:
-                self.service_tokens[service.api_token] = service.name
-            elif service.managed:
-                # generate new token
-                # TODO: revoke old tokens?
-                service.api_token = service.orm.new_api_token(
-                    note="generated at startup"
-                )
-
-            if service.url:
-                parsed = urlparse(service.url)
-                if parsed.port is not None:
-                    port = parsed.port
-                elif parsed.scheme == 'http':
-                    port = 80
-                elif parsed.scheme == 'https':
-                    port = 443
-                server = service.orm.server = orm.Server(
-                    proto=parsed.scheme,
-                    ip=parsed.hostname,
-                    port=port,
-                    cookie_name=service.oauth_client_id,
-                    base_url=service.prefix,
-                )
-                self.db.add(server)
-
-            else:
-                service.orm.server = None
-
-            if service.oauth_available:
-                allowed_scopes = set()
-                if service.oauth_client_allowed_scopes:
-                    allowed_scopes.update(service.oauth_client_allowed_scopes)
-                if service.oauth_roles:
-                    if not allowed_scopes:
-                        # DEPRECATED? It's still convenient and valid,
-                        # e.g. 'admin'
-                        allowed_roles = list(
-                            self.db.query(orm.Role).filter(
-                                orm.Role.name.in_(service.oauth_roles)
-                            )
+        if service.oauth_available:
+            allowed_scopes = set()
+            if service.oauth_client_allowed_scopes:
+                allowed_scopes.update(service.oauth_client_allowed_scopes)
+            if service.oauth_roles:
+                if not allowed_scopes:
+                    # DEPRECATED? It's still convenient and valid,
+                    # e.g. 'admin'
+                    allowed_roles = list(
+                        self.db.query(orm.Role).filter(
+                            orm.Role.name.in_(service.oauth_roles)
                         )
-                        allowed_scopes.update(roles.roles_to_scopes(allowed_roles))
-                    else:
-                        self.log.warning(
-                            f"Ignoring oauth_roles for {service.name}: {service.oauth_roles},"
-                            f" using oauth_client_allowed_scopes={allowed_scopes}."
-                        )
-                oauth_client = self.oauth_provider.add_client(
-                    client_id=service.oauth_client_id,
-                    client_secret=service.api_token,
-                    redirect_uri=service.oauth_redirect_uri,
-                    description="JupyterHub service %s" % service.name,
-                )
-                service.orm.oauth_client = oauth_client
-                # add access-scopes, derived from OAuthClient itself
-                allowed_scopes.update(scopes.access_scopes(oauth_client))
-                oauth_client.allowed_scopes = sorted(allowed_scopes)
+                    )
+                    allowed_scopes.update(roles.roles_to_scopes(allowed_roles))
+                else:
+                    self.log.warning(
+                        f"Ignoring oauth_roles for {service.name}: {service.oauth_roles},"
+                        f" using oauth_client_allowed_scopes={allowed_scopes}."
+                    )
+            oauth_client = self.oauth_provider.add_client(
+                client_id=service.oauth_client_id,
+                client_secret=service.api_token,
+                redirect_uri=service.oauth_redirect_uri,
+                description="JupyterHub service %s" % service.name,
+            )
+            service.orm.oauth_client = oauth_client
+            # add access-scopes, derived from OAuthClient itself
+            allowed_scopes.update(scopes.access_scopes(oauth_client))
+            oauth_client.allowed_scopes = sorted(allowed_scopes)
+        else:
+            if service.oauth_client:
+                self.db.delete(service.oauth_client)
+
+        self._service_map[name] = service
+
+        return service
+
+    def init_services(self):
+        self._service_map.clear()
+        for spec in self.services:
+            self.service_from_spec(spec, from_config=True)
+
+        for service_orm in self.db.query(orm.Service):
+            if service_orm.from_config:
+                # delete config-based services from db
+                # that are not in current config file:
+                if service_orm.name not in self._service_map:
+                    self.db.delete(service_orm)
             else:
-                if service.oauth_client:
-                    self.db.delete(service.oauth_client)
+                self.service_from_orm(service_orm)
 
-            self._service_map[name] = service
-
-        # delete services from db not in service config:
-        for service in self.db.query(orm.Service):
-            if service.name not in self._service_map:
-                self.db.delete(service)
         self.db.commit()
 
     async def check_services_health(self):
         """Check connectivity of all services"""
         for name, service in self._service_map.items():
             if not service.url:
+                # no URL to check, nothing to do
                 continue
             try:
                 await Server.from_orm(service.orm.server).wait_up(timeout=1, http=True)
@@ -2591,19 +2726,22 @@ class JupyterHub(Application):
         # Server objects can be associated with either a Spawner or a Service,
         # we are only interested in the ones associated with a Spawner
         check_futures = []
-        for orm_server in db.query(orm.Server):
-            orm_spawner = orm_server.spawner
-            if not orm_spawner:
-                # check for orphaned Server rows
-                # this shouldn't happen if we've got our sqlachemy right
-                if not orm_server.service:
-                    self.log.warning("deleting orphaned server %s", orm_server)
-                    self.db.delete(orm_server)
-                    self.db.commit()
-                continue
+
+        for orm_user, orm_spawner in (
+            self.db.query(orm.User, orm.Spawner)
+            # join filters out any Users with no Spawners
+            .join(orm.Spawner, orm.User._orm_spawners)
+            # this gets Users with *any* active server
+            .filter(orm.Spawner.server != None)
+            # pre-load relationships to avoid O(N active servers) queries
+            .options(
+                joinedload(orm.User._orm_spawners),
+                joinedload(orm.Spawner.server),
+            )
+        ):
             # instantiate Spawner wrapper and check if it's still alive
             # spawner should be running
-            user = self.users[orm_spawner.user]
+            user = self.users[orm_user]
             spawner = user.spawners[orm_spawner.name]
             self.log.debug("Loading state for %s from db", spawner._log_name)
             # signal that check is pending to avoid race conditions
@@ -2652,7 +2790,6 @@ class JupyterHub(Application):
         for user in self.users.values():
             for spawner in user.spawners.values():
                 oauth_client_ids.add(spawner.oauth_client_id)
-
         for i, oauth_client in enumerate(self.db.query(orm.OAuthClient)):
             if oauth_client.identifier not in oauth_client_ids:
                 self.log.warning("Deleting OAuth client %s", oauth_client.identifier)
@@ -3094,6 +3231,72 @@ class JupyterHub(Application):
 
         await self.proxy.check_routes(self.users, self._service_map, routes)
 
+    async def start_service(
+        self,
+        service_name: str,
+        service: Service,
+        ssl_context: Optional[ssl.SSLContext] = None,
+    ) -> bool:
+        """Start a managed service or poll for external service
+
+        Args:
+            service_name (str): Name of the service.
+            service (Service): The service object.
+
+        Returns:
+            boolean: Returns `True` if the service is started successfully,
+            returns `False` otherwise.
+        """
+        if ssl_context is None:
+            ssl_context = make_ssl_context(
+                self.internal_ssl_key,
+                self.internal_ssl_cert,
+                cafile=self.internal_ssl_ca,
+                purpose=ssl.Purpose.CLIENT_AUTH,
+            )
+
+        msg = f'{service_name} at {service.url}' if service.url else service_name
+        if service.managed:
+            self.log.info("Starting managed service %s", msg)
+            try:
+                await service.start()
+            except Exception as e:
+                self.log.critical(
+                    "Failed to start service %s", service_name, exc_info=True
+                )
+                return False
+        else:
+            self.log.info("Adding external service %s", msg)
+
+        if service.url:
+            tries = 10 if service.managed else 1
+            for i in range(tries):
+                try:
+                    await Server.from_orm(service.orm.server).wait_up(
+                        http=True, timeout=1, ssl_context=ssl_context
+                    )
+                except AnyTimeoutError:
+                    if service.managed:
+                        status = await service.spawner.poll()
+                        if status is not None:
+                            self.log.error(
+                                "Service %s exited with status %s",
+                                service_name,
+                                status,
+                            )
+                            return False
+                else:
+                    return True
+            else:
+                self.log.error(
+                    "Cannot connect to %s service %s at %s. Is it running?",
+                    service.kind,
+                    service_name,
+                    service.url,
+                )
+                return False
+        return True
+
     async def start(self):
         """Start the whole thing"""
         self.io_loop = loop = IOLoop.current()
@@ -3179,55 +3382,29 @@ class JupyterHub(Application):
 
         # start the service(s)
         for service_name, service in self._service_map.items():
-            msg = f'{service_name} at {service.url}' if service.url else service_name
-            if service.managed:
-                self.log.info("Starting managed service %s", msg)
-                try:
-                    await service.start()
-                except Exception as e:
-                    self.log.critical(
-                        "Failed to start service %s", service_name, exc_info=True
-                    )
+            service_ready = await self.start_service(service_name, service, ssl_context)
+            if not service_ready:
+                if service.from_config:
+                    # Stop the application if a config-based service failed to start.
                     self.exit(1)
-            else:
-                self.log.info("Adding external service %s", msg)
-
-            if service.url:
-                tries = 10 if service.managed else 1
-                for i in range(tries):
-                    try:
-                        await Server.from_orm(service.orm.server).wait_up(
-                            http=True, timeout=1, ssl_context=ssl_context
-                        )
-                    except AnyTimeoutError:
-                        if service.managed:
-                            status = await service.spawner.poll()
-                            if status is not None:
-                                self.log.error(
-                                    "Service %s exited with status %s",
-                                    service_name,
-                                    status,
-                                )
-                                break
-                    else:
-                        break
                 else:
+                    # Only warn for database-based service, so that admin can connect
+                    # to hub to remove the service.
                     self.log.error(
-                        "Cannot connect to %s service %s at %s. Is it running?",
-                        service.kind,
+                        "Failed to reach externally managed service %s",
                         service_name,
-                        service.url,
+                        exc_info=True,
                     )
 
         await self.proxy.check_routes(self.users, self._service_map)
 
-        if self.service_check_interval and any(
-            s.url for s in self._service_map.values()
-        ):
-            pc = PeriodicCallback(
+        # Check services health
+        self._check_services_health_callback = None
+        if self.service_check_interval:
+            self._check_services_health_callback = PeriodicCallback(
                 self.check_services_health, 1e3 * self.service_check_interval
             )
-            pc.start()
+            self._check_services_health_callback.start()
 
         if self.last_activity_interval:
             pc = PeriodicCallback(

jupyterhub/auth.py

@@ -157,6 +157,25 @@ class Authenticator(LoggingConfigurable):
         """
     ).tag(config=True)
 
+    otp_prompt = Any(
+        "OTP:",
+        help="""
+        The prompt string for the extra OTP (One Time Password) field.
+
+        .. versionadded:: 5.0
+        """,
+    ).tag(config=True)
+
+    request_otp = Bool(
+        False,
+        config=True,
+        help="""
+        Prompt for OTP (One Time Password) in the login form.
+
+        .. versionadded:: 5.0
+        """,
+    )
+
     _deprecated_aliases = {
         "whitelist": ("allowed_users", "1.2"),
         "blacklist": ("blocked_users", "1.2"),
@@ -485,6 +504,8 @@ class Authenticator(LoggingConfigurable):
          - `authenticate` turns formdata into a username
          - `normalize_username` normalizes the username
          - `check_allowed` checks against the allowed usernames
+         - `check_blocked_users` check against the blocked usernames
+         - `is_admin` check if a user is an admin
 
         .. versionchanged:: 0.8
             return dict instead of username
@@ -603,8 +624,7 @@ class Authenticator(LoggingConfigurable):
                 The Authenticator may return a dict instead, which MUST have a
                 key `name` holding the username, and MAY have additional keys:
 
-                - `auth_state`, a dictionary of of auth state that will be
-                  persisted;
+                - `auth_state`, a dictionary of auth state that will be persisted;
                 - `admin`, the admin setting value for the user
                 - `groups`, the list of group names the user should be a member of,
                   if Authenticator.manage_groups is True.
@@ -1103,9 +1123,16 @@ class PAMAuthenticator(LocalAuthenticator):
         Return None otherwise.
         """
         username = data['username']
+        password = data["password"]
+        if "otp" in data:
+            # OTP given, pass as tuple (requires pamela 1.1)
+            password = (data["password"], data["otp"])
         try:
             pamela.authenticate(
-                username, data['password'], service=self.service, encoding=self.encoding
+                username,
+                password,
+                service=self.service,
+                encoding=self.encoding,
             )
         except pamela.PAMError as e:
             if handler is not None:

jupyterhub/handlers/login.py

@@ -105,6 +105,7 @@ class LoginHandler(BaseHandler):
                     'next': self.get_argument('next', ''),
                 },
             ),
+            "authenticator": self.authenticator,
             "xsrf": self.xsrf_token.decode('ascii'),
         }
         custom_html = Template(

jupyterhub/oauth/provider.py

@@ -668,6 +668,18 @@ class JupyterHubOAuthServer(WebApplicationServer):
         self.db.commit()
         return orm_client
 
+    def remove_client(self, client_id):
+        """Remove a client by its id if it is existed."""
+        orm_client = (
+            self.db.query(orm.OAuthClient).filter_by(identifier=client_id).one_or_none()
+        )
+        if orm_client is not None:
+            self.db.delete(orm_client)
+            self.db.commit()
+            app_log.info("Removed client %s", client_id)
+        else:
+            app_log.warning("No such client %s", client_id)
+
     def fetch_by_client_id(self, client_id):
         """Find a client by its id"""
         client = self.db.query(orm.OAuthClient).filter_by(identifier=client_id).first()

jupyterhub/orm.py

@@ -404,6 +404,26 @@ class Service(Base):
         'Role', secondary='service_role_map', back_populates='services', lazy="selectin"
     )
 
+    url = Column(Unicode(2047), nullable=True)
+
+    oauth_client_allowed_scopes = Column(JSONList, nullable=True)
+
+    info = Column(JSONDict, nullable=True)
+
+    display = Column(Boolean, nullable=True)
+
+    oauth_no_confirm = Column(Boolean, nullable=True)
+
+    command = Column(JSONList, nullable=True)
+
+    cwd = Column(Unicode(4095), nullable=True)
+
+    environment = Column(JSONDict, nullable=True)
+
+    user = Column(Unicode(255), nullable=True)
+
+    from_config = Column(Boolean, default=True)
+
     api_tokens = relationship(
         "APIToken", back_populates="service", cascade="all, delete-orphan"
     )
@@ -426,6 +446,7 @@ class Service(Base):
             ondelete='SET NULL',
         ),
     )
+
     oauth_client = relationship(
         'OAuthClient',
         back_populates="service",

jupyterhub/roles.py

@@ -34,6 +34,7 @@ def get_default_roles():
                 'admin-ui',
                 'admin:users',
                 'admin:servers',
+                'admin:services',
                 'tokens',
                 'admin:groups',
                 'list:services',

jupyterhub/scopes.py

@@ -123,6 +123,10 @@ scope_definitions = {
     'delete:groups': {
         'description': "Delete groups.",
     },
+    'admin:services': {
+        'description': 'Create, read, update, delete services, not including services defined from config files.',
+        'subscopes': ['list:services', 'read:services', 'read:roles:services'],
+    },
     'list:services': {
         'description': 'List services, including at least their names.',
         'subscopes': ['read:services:name'],
@@ -435,7 +439,7 @@ def _expand_self_scope(username):
 
 @lru_cache(maxsize=65535)
 def _expand_scope(scope):
-    """Returns a scope and all all subscopes
+    """Returns a scope and all subscopes
 
     Arguments:
       scope (str): the scope to expand

jupyterhub/services/service.py

@@ -180,9 +180,12 @@ class Service(LoggingConfigurable):
     - user: str
         The name of a system user to become.
         If unspecified, run as the same user as the Hub.
+
     """
 
-    # inputs:
+    # traits tagged with `input=True` are accepted as input from configuration / API
+    # input traits are also persisted to the db UNLESS they are also tagged with `in_db=False`
+
     name = Unicode(
         help="""The name of the service.
 
@@ -205,7 +208,7 @@ class Service(LoggingConfigurable):
 
         DEPRECATED in 3.0: use oauth_client_allowed_scopes
       """
-    ).tag(input=True)
+    ).tag(input=True, in_db=False)
 
     oauth_client_allowed_scopes = List(
         help="""OAuth allowed scopes.
@@ -225,7 +228,7 @@ class Service(LoggingConfigurable):
 
         If unspecified, an API token will be generated for managed services.
         """
-    ).tag(input=True)
+    ).tag(input=True, in_db=False)
 
     info = Dict(
         help="""Provide a place to include miscellaneous information about the service,
@@ -310,7 +313,7 @@ class Service(LoggingConfigurable):
         You shouldn't generally need to change this.
         Default: `service-<name>`
         """
-    ).tag(input=True)
+    ).tag(input=True, in_db=False)
 
     @default('oauth_client_id')
     def _default_client_id(self):
@@ -331,7 +334,7 @@ class Service(LoggingConfigurable):
         You shouldn't generally need to change this.
         Default: `/services/:name/oauth_callback`
         """
-    ).tag(input=True)
+    ).tag(input=True, in_db=False)
 
     @default('oauth_redirect_uri')
     def _default_redirect_uri(self):
@@ -371,6 +374,11 @@ class Service(LoggingConfigurable):
         else:
             return self.server.base_url
 
+    @property
+    def from_config(self):
+        """Is the service defined from config file?"""
+        return self.orm.from_config
+
     def __repr__(self):
         return "<{cls}(name={name}{managed})>".format(
             cls=self.__class__.__name__,

jupyterhub/singleuser/app.py

@@ -6,7 +6,7 @@
 .. versionchanged:: 2.0
 
     Default app changed to launch `jupyter labhub`.
-    Use JUPYTERHUB_SINGLEUSER_APP=notebook.notebookapp.NotebookApp for the legacy 'classic' notebook server.
+    Use JUPYTERHUB_SINGLEUSER_APP='notebook' for the legacy 'classic' notebook server (requires notebook<7).
 """
 import os
 
@@ -27,7 +27,25 @@ JUPYTERHUB_SINGLEUSER_APP = _app_shortcuts.get(
     JUPYTERHUB_SINGLEUSER_APP.replace("_", "-"), JUPYTERHUB_SINGLEUSER_APP
 )
 
+
 if JUPYTERHUB_SINGLEUSER_APP:
+    if JUPYTERHUB_SINGLEUSER_APP in {"notebook", _app_shortcuts["notebook"]}:
+        # better error for notebook v7, which uses jupyter-server
+        # when the legacy notebook server is requested
+        try:
+            from notebook import __version__
+        except ImportError:
+            # will raise later
+            pass
+        else:
+            # check if this failed because of notebook v7
+            _notebook_major_version = int(__version__.split(".", 1)[0])
+            if _notebook_major_version >= 7:
+                raise ImportError(
+                    f"JUPYTERHUB_SINGLEUSER_APP={JUPYTERHUB_SINGLEUSER_APP} is not valid with notebook>=7 (have notebook=={__version__}).\n"
+                    f"Leave $JUPYTERHUB_SINGLEUSER_APP unspecified (or use the default JUPYTERHUB_SINGLEUSER_APP=jupyter-server), "
+                    'and set `c.Spawner.default_url = "/tree"` to make notebook v7 the default UI.'
+                )
     App = import_item(JUPYTERHUB_SINGLEUSER_APP)
 else:
     App = None

jupyterhub/singleuser/extension.py

@@ -483,6 +483,11 @@ class JupyterHubSingleUser(ExtensionApp):
         cfg.answer_yes = True
         self.config.FileContentsManager.delete_to_trash = False
 
+        # load Spawner.notebook_dir configuration, if given
+        root_dir = os.getenv("JUPYTERHUB_ROOT_DIR", None)
+        if root_dir:
+            cfg.root_dir = os.path.expanduser(root_dir)
+
         # load http server config from environment
         url = urlparse(os.environ['JUPYTERHUB_SERVICE_URL'])
         if url.port:
@@ -639,7 +644,7 @@ class JupyterHubSingleUser(ExtensionApp):
     disable_user_config = Bool()
 
     @default("disable_user_config")
-    def _defaut_disable_user_config(self):
+    def _default_disable_user_config(self):
         return _bool_env("JUPYTERHUB_DISABLE_USER_CONFIG")
 
     @classmethod

jupyterhub/spawner.py

@@ -274,8 +274,6 @@ class Spawner(LoggingConfigurable):
     api_token = Unicode()
     oauth_client_id = Unicode()
 
-    oauth_scopes = List(Unicode())
-
     @property
     def oauth_scopes(self):
         warnings.warn(

jupyterhub/tests/browser/test_browser.py

@@ -2,6 +2,7 @@
 
 import json
 import re
+from unittest import mock
 from urllib.parse import parse_qs, urlparse
 
 import pytest
@@ -143,6 +144,53 @@ async def test_login_with_invalid_credantials(app, browser, username, pass_w):
     await expect(browser).to_have_url(re.compile(".*/hub/login"))
 
 
+@pytest.mark.parametrize("request_otp", [True, False])
+async def test_login_otp(request, app, browser, username, request_otp):
+    def _reset():
+        app.authenticator.request_otp = False
+
+    request.addfinalizer(_reset)
+
+    app.authenticator.request_otp = request_otp
+    login_url = url_concat(
+        url_path_join(public_host(app), app.hub.base_url, "login"),
+        {"next": ujoin(public_url(app), "/hub/home/")},
+    )
+    await browser.goto(login_url)
+    # check for otp element
+    otp_label = browser.locator("label[for=otp_input]")
+    otp_input = browser.locator("input#otp_input")
+    if not request_otp:
+        # request_otp is False, no OTP prompt
+        assert await otp_label.count() == 0
+        assert await otp_input.count() == 0
+        return
+
+    await expect(otp_label).to_be_visible()
+    await expect(otp_input).to_be_visible()
+    await expect(otp_label).to_have_text(app.authenticator.otp_prompt)
+
+    # fill it out
+    await browser.get_by_label("Username:").fill(username)
+    await browser.get_by_label("Password:").fill(username)
+    await browser.get_by_label("OTP:").fill("otp!")
+
+    # submit form
+    with mock.patch(
+        "jupyterhub.tests.mocking.mock_authenticate",
+        spec=True,
+        return_value={"username": username},
+    ) as mock_otp_auth:
+        await browser.get_by_role("button", name="Sign in").click()
+        expected_url = ujoin(public_url(app), "/hub/home/")
+        await expect(browser).to_have_url(expected_url)
+
+    # check that OTP was passed
+    assert mock_otp_auth.called
+    assert mock_otp_auth.call_args.args[0] == username
+    assert mock_otp_auth.call_args.args[1] == (username, "otp!")
+
+
 # SPAWNING
 
 
@@ -985,7 +1033,7 @@ async def test_start_stop_server_on_admin_page(
 
     async def click_spawn_page(browser, username):
         """spawn the server for one user via the Spawn page button, index = 0 or 1"""
-        spawn_btn_xpath = f'//a[contains(@href, "spawn/{username}")]/button[contains(@class, "secondary")]'
+        spawn_btn_xpath = f'//a[contains(@href, "spawn/{username}")]/button[contains(@class, "btn-light")]'
         spawn_btn = browser.locator(spawn_btn_xpath)
         await expect(spawn_btn).to_be_enabled()
         async with browser.expect_navigation(url=f"**/user/{username}/"):
@@ -993,7 +1041,7 @@ async def test_start_stop_server_on_admin_page(
 
     async def click_access_server(browser, username):
         """access to the server for users via the Access Server button"""
-        access_btn_xpath = f'//a[contains(@href, "user/{username}")]/button[contains(@class, "primary")]'
+        access_btn_xpath = f'//a[contains(@href, "user/{username}")]/button[contains(@class, "btn-primary")]'
         access_btn = browser.locator(access_btn_xpath)
         await expect(access_btn).to_be_enabled()
         await access_btn.click()

jupyterhub/tests/conftest.py

@@ -167,6 +167,8 @@ async def cleanup_after(request, io_loop):
         app = MockHub.instance()
         if app.db_file.closed:
             return
+
+        # cleanup users
         for orm_user in app.db.query(orm.User):
             user = app.users[orm_user]
             for name, spawner in list(user.spawners.items()):
@@ -182,6 +184,16 @@ async def cleanup_after(request, io_loop):
         # delete groups
         for group in app.db.query(orm.Group):
             app.db.delete(group)
+
+        # clear services
+        for name, service in app._service_map.items():
+            if service.managed:
+                service.stop()
+        for orm_service in app.db.query(orm.Service):
+            if orm_service.oauth_client:
+                app.oauth_provider.remove_client(orm_service.oauth_client_id)
+            app.db.delete(orm_service)
+        app._service_map.clear()
         app.db.commit()
 
 
@@ -263,10 +275,7 @@ class MockServiceSpawner(jupyterhub.services.service._ServiceSpawner):
     poll_interval = 1
 
 
-_mock_service_counter = 0
-
-
-async def _mockservice(request, app, external=False, url=False):
+async def _mockservice(request, app, name, external=False, url=False):
     """
     Add a service to the application
 
@@ -282,9 +291,6 @@ async def _mockservice(request, app, external=False, url=False):
           If True, register the service at a URL
           (as opposed to headless, API-only).
     """
-    global _mock_service_counter
-    _mock_service_counter += 1
-    name = 'mock-service-%i' % _mock_service_counter
     spec = {'name': name, 'command': mockservice_cmd, 'admin': True}
     if url:
         if app.internal_ssl:
@@ -331,22 +337,33 @@ async def _mockservice(request, app, external=False, url=False):
     return service
 
 
+_service_name_counter = 0
+
+
 @fixture
-async def mockservice(request, app):
+def service_name():
+    global _service_name_counter
+    _service_name_counter += 1
+    name = f'test-service-{_service_name_counter}'
+    return name
+
+
+@fixture
+async def mockservice(request, app, service_name):
     """Mock a service with no external service url"""
-    yield await _mockservice(request, app, url=False)
+    yield await _mockservice(request, app, name=service_name, url=False)
 
 
 @fixture
-async def mockservice_external(request, app):
+async def mockservice_external(request, app, service_name):
     """Mock an externally managed service (don't start anything)"""
-    yield await _mockservice(request, app, external=True, url=False)
+    yield await _mockservice(request, app, name=service_name, external=True, url=False)
 
 
 @fixture
-async def mockservice_url(request, app):
+async def mockservice_url(request, app, service_name):
     """Mock a service with its own url to test external services"""
-    yield await _mockservice(request, app, url=True)
+    yield await _mockservice(request, app, name=service_name, url=True)
 
 
 @fixture
@@ -535,3 +552,17 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):
         queries = db_counts[nodeid]
         if queries:
             terminalreporter.line(f"{queries:<6} {nodeid}")
+
+
+@fixture
+def service_data(service_name):
+    """Data used to create service at runtime"""
+    return {
+        "name": service_name,
+        "oauth_client_id": f"service-{service_name}",
+        "api_token": f"api_token-{service_name}",
+        "oauth_redirect_uri": "http://127.0.0.1:5555/oauth_callback-from-api",
+        "oauth_no_confirm": True,
+        "oauth_client_allowed_scopes": ["inherit"],
+        "info": {'foo': 'bar'},
+    }

jupyterhub/tests/extension/__init__.py

@@ -30,6 +30,7 @@ class JupyterHubTestHandler(JupyterHandler):
         info = {
             "current_user": self.current_user,
             "config": self.app.config,
+            "root_dir": self.contents_manager.root_dir,
             "disable_user_config": getattr(self.app, "disable_user_config", None),
             "settings": self.settings,
             "config_file_paths": self.app.config_file_paths,

jupyterhub/tests/test_api.py

@@ -4,10 +4,12 @@ import json
 import re
 import sys
 import uuid
+from copy import deepcopy
 from datetime import datetime, timedelta
 from unittest import mock
 from urllib.parse import quote, urlparse
 
+import pytest
 from pytest import fixture, mark
 from tornado.httputil import url_concat
 
@@ -2062,7 +2064,7 @@ async def test_get_services(app, mockservice_url):
 async def test_get_service(app, mockservice_url):
     mockservice = mockservice_url
     db = app.db
-    r = await api_request(app, 'services/%s' % mockservice.name)
+    r = await api_request(app, f"services/{mockservice.name}")
     r.raise_for_status()
     assert r.status_code == 200
 
@@ -2081,19 +2083,271 @@ async def test_get_service(app, mockservice_url):
     }
     r = await api_request(
         app,
-        'services/%s' % mockservice.name,
+        f"services/{mockservice.name}",
         headers={'Authorization': 'token %s' % mockservice.api_token},
     )
     r.raise_for_status()
+
     r = await api_request(
-        app, 'services/%s' % mockservice.name, headers=auth_header(db, 'user')
+        app, f"services/{mockservice.name}", headers=auth_header(db, 'user')
     )
     assert r.status_code == 403
 
+    r = await api_request(app, "services/nosuchservice")
+    assert r.status_code == 404
+
+
+@pytest.fixture
+def service_admin_user(create_user_with_scopes):
+    return create_user_with_scopes('admin:services')
+
+
+@mark.services
+async def test_create_service(app, service_admin_user, service_name, service_data):
+    db = app.db
+    r = await api_request(
+        app,
+        f'services/{service_name}',
+        headers=auth_header(db, service_admin_user.name),
+        data=json.dumps(service_data),
+        method='post',
+    )
+
+    r.raise_for_status()
+    assert r.status_code == 201
+    assert r.json()['name'] == service_name
+    orm_service = orm.Service.find(db, service_name)
+    assert orm_service is not None
+
+    oath_client = (
+        db.query(orm.OAuthClient)
+        .filter_by(identifier=service_data['oauth_client_id'])
+        .first()
+    )
+    assert oath_client.redirect_uri == service_data['oauth_redirect_uri']
+
+    assert service_name in app._service_map
+    assert (
+        app._service_map[service_name].oauth_no_confirm
+        == service_data['oauth_no_confirm']
+    )
+
+
+@mark.services
+async def test_create_service_no_role(app, service_name, service_data):
+    db = app.db
+    r = await api_request(
+        app,
+        f'services/{service_name}',
+        headers=auth_header(db, 'user'),
+        data=json.dumps(service_data),
+        method='post',
+    )
+
+    assert r.status_code == 403
+
+
+@mark.services
+async def test_create_service_conflict(
+    app, service_admin_user, mockservice, service_data, service_name
+):
+    db = app.db
+    app.services = [{'name': service_name}]
+    app.init_services()
+
+    r = await api_request(
+        app,
+        f'services/{service_name}',
+        headers=auth_header(db, service_admin_user.name),
+        data=json.dumps(service_data),
+        method='post',
+    )
+
+    assert r.status_code == 409
+
+
+@mark.services
+async def test_create_service_duplication(
+    app, service_admin_user, service_name, service_data
+):
+    db = app.db
+
+    r = await api_request(
+        app,
+        f'services/{service_name}',
+        headers=auth_header(db, service_admin_user.name),
+        data=json.dumps(service_data),
+        method='post',
+    )
+    assert r.status_code == 201
+
+    r = await api_request(
+        app,
+        f'services/{service_name}',
+        headers=auth_header(db, service_admin_user.name),
+        data=json.dumps(service_data),
+        method='post',
+    )
+    assert r.status_code == 409
+
+
+@mark.services
+async def test_create_managed_service(
+    app, service_admin_user, service_name, service_data
+):
+    db = app.db
+    managed_service_data = deepcopy(service_data)
+    managed_service_data['command'] = ['foo']
+    r = await api_request(
+        app,
+        f'services/{service_name}',
+        headers=auth_header(db, service_admin_user.name),
+        data=json.dumps(managed_service_data),
+        method='post',
+    )
+
+    assert r.status_code == 400
+    assert 'Can not create managed service' in r.json()['message']
+    orm_service = orm.Service.find(db, service_name)
+    assert orm_service is None
+
+
+@mark.services
+async def test_create_admin_service(app, admin_user, service_name, service_data):
+    db = app.db
+    managed_service_data = deepcopy(service_data)
+    managed_service_data['admin'] = True
+    r = await api_request(
+        app,
+        f'services/{service_name}',
+        headers=auth_header(db, admin_user.name),
+        data=json.dumps(managed_service_data),
+        method='post',
+    )
+
+    assert r.status_code == 201
+    orm_service = orm.Service.find(db, service_name)
+    assert orm_service is not None
+
+
+@mark.services
+async def test_create_admin_service_without_admin_right(
+    app, service_admin_user, service_data, service_name
+):
+    db = app.db
+    managed_service_data = deepcopy(service_data)
+    managed_service_data['admin'] = True
+    r = await api_request(
+        app,
+        f'services/{service_name}',
+        headers=auth_header(db, service_admin_user.name),
+        data=json.dumps(managed_service_data),
+        method='post',
+    )
+
+    assert r.status_code == 400
+    assert 'Not assigning requested scopes' in r.json()['message']
+    orm_service = orm.Service.find(db, service_name)
+    assert orm_service is None
+
+
+@mark.services
+async def test_create_service_with_scope(
+    app, create_user_with_scopes, service_name, service_data
+):
+    db = app.db
+    managed_service_data = deepcopy(service_data)
+    managed_service_data['oauth_client_allowed_scopes'] = ["admin:users"]
+    managed_service_data['oauth_client_id'] = "service-client-with-scope"
+    user_with_scope = create_user_with_scopes('admin:services', 'admin:users')
+    r = await api_request(
+        app,
+        f'services/{service_name}',
+        headers=auth_header(db, user_with_scope.name),
+        data=json.dumps(managed_service_data),
+        method='post',
+    )
+
+    assert r.status_code == 201
+    orm_service = orm.Service.find(db, service_name)
+    assert orm_service is not None
+
+
+@mark.services
+async def test_create_service_without_requested_scope(
+    app,
+    service_admin_user,
+    service_data,
+    service_name,
+):
+    db = app.db
+    managed_service_data = deepcopy(service_data)
+    managed_service_data['oauth_client_allowed_scopes'] = ["admin:users"]
+    r = await api_request(
+        app,
+        f'services/{service_name}',
+        headers=auth_header(db, service_admin_user.name),
+        data=json.dumps(managed_service_data),
+        method='post',
+    )
+
+    assert r.status_code == 400
+    assert 'Not assigning requested scopes' in r.json()['message']
+    orm_service = orm.Service.find(db, service_name)
+    assert orm_service is None
+
+
+@mark.services
+async def test_delete_service(app, service_admin_user, service_name, service_data):
+    db = app.db
+    r = await api_request(
+        app,
+        f'services/{service_name}',
+        headers=auth_header(db, service_admin_user.name),
+        data=json.dumps(service_data),
+        method='post',
+    )
+    assert r.status_code == 201
+
+    r = await api_request(
+        app,
+        f'services/{service_name}',
+        headers=auth_header(db, service_admin_user.name),
+        method='delete',
+    )
+    assert r.status_code == 200
+
+    orm_service = orm.Service.find(db, service_name)
+    assert orm_service is None
+
+    oath_client = (
+        db.query(orm.OAuthClient)
+        .filter_by(identifier=service_data['oauth_client_id'])
+        .first()
+    )
+    assert oath_client is None
+
+    assert service_name not in app._service_map
+
+    r = await api_request(app, f"services/{service_name}", method="delete")
+    assert r.status_code == 404
+
+
+@mark.services
+async def test_delete_service_from_config(app, service_admin_user, mockservice):
+    db = app.db
+    service_name = mockservice.name
+    r = await api_request(
+        app,
+        f'services/{service_name}',
+        headers=auth_header(db, service_admin_user.name),
+        method='delete',
+    )
+    assert r.status_code == 405
+    assert r.json()['message'] == f'Service {service_name} is not modifiable at runtime'
+
 
 async def test_root_api(app):
-    base_url = app.hub.url
-    url = ujoin(base_url, 'api')
     kwargs = {}
     if app.internal_ssl:
         kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key)

jupyterhub/tests/test_app.py

@@ -220,7 +220,7 @@ def test_cookie_secret_env(tmpdir, request):
     assert not os.path.exists(hub.cookie_secret_file)
 
 
-def test_cookie_secret_string_():
+def test_cookie_secret_string():
     cfg = Config()
 
     cfg.JupyterHub.cookie_secret = "not hex"
@@ -270,18 +270,41 @@ async def test_load_groups(tmpdir, request):
     )
 
 
-async def test_resume_spawners(tmpdir, request):
-    if not os.getenv('JUPYTERHUB_TEST_DB_URL'):
-        p = patch.dict(
-            os.environ,
-            {
-                'JUPYTERHUB_TEST_DB_URL': 'sqlite:///%s'
-                % tmpdir.join('jupyterhub.sqlite')
-            },
-        )
-        p.start()
-        request.addfinalizer(p.stop)
+@pytest.fixture
+def persist_db(tmpdir):
+    """ensure db will persist (overrides default sqlite://:memory:)"""
+    if os.getenv('JUPYTERHUB_TEST_DB_URL'):
+        # already using a db, no need
+        yield
+        return
+    with patch.dict(
+        os.environ,
+        {'JUPYTERHUB_TEST_DB_URL': f"sqlite:///{tmpdir.join('jupyterhub.sqlite')}"},
+    ):
+        yield
 
+
+@pytest.fixture
+def new_hub(request, tmpdir, persist_db):
+    """Fixture to launch a new hub for testing"""
+
+    async def new_hub():
+        kwargs = {}
+        ssl_enabled = getattr(request.module, "ssl_enabled", False)
+        if ssl_enabled:
+            kwargs['internal_certs_location'] = str(tmpdir)
+        app = MockHub(test_clean_db=False, **kwargs)
+        app.config.ConfigurableHTTPProxy.should_start = False
+        app.config.ConfigurableHTTPProxy.auth_token = 'unused'
+        request.addfinalizer(app.stop)
+        await app.initialize([])
+
+        return app
+
+    return new_hub
+
+
+async def test_resume_spawners(tmpdir, request, new_hub):
     async def new_hub():
         kwargs = {}
         ssl_enabled = getattr(request.module, "ssl_enabled", False)
@@ -473,3 +496,42 @@ async def test_user_creation(tmpdir, request):
         "in-group",
         "in-role",
     }
+
+
+async def test_recreate_service_from_database(
+    request, new_hub, service_name, service_data
+):
+    # create a hub and add a service (not from config)
+    app = await new_hub()
+    app.service_from_spec(service_data, from_config=False)
+    app.stop()
+
+    # new hub, should load service from db
+    app = await new_hub()
+    assert service_name in app._service_map
+
+    # verify keys
+    service = app._service_map[service_name]
+    for key, value in service_data.items():
+        if key in {'api_token'}:
+            # skip some keys
+            continue
+        assert getattr(service, key) == value
+
+    assert (
+        service_data['oauth_client_id'] in app.tornado_settings['oauth_no_confirm_list']
+    )
+    oauth_client = (
+        app.db.query(orm.OAuthClient)
+        .filter_by(identifier=service_data['oauth_client_id'])
+        .first()
+    )
+    assert oauth_client.redirect_uri == service_data['oauth_redirect_uri']
+
+    # delete service from db, start one more
+    app.db.delete(service.orm)
+    app.db.commit()
+
+    # start one more, service should be gone
+    app = await new_hub()
+    assert service_name not in app._service_map

jupyterhub/tests/test_auth.py

@@ -105,7 +105,8 @@ async def test_pam_auth_admin_groups():
         _getgrouplist=getgrouplist,
     ):
         authorized = await authenticator.get_authenticated_user(
-            None, {'username': 'also_group_admin', 'password': 'also_group_admin'}
+            None,
+            {'username': 'also_group_admin', 'password': 'also_group_admin'},
         )
     assert authorized['name'] == 'also_group_admin'
     assert authorized['admin'] is True
@@ -118,7 +119,8 @@ async def test_pam_auth_admin_groups():
         _getgrouplist=getgrouplist,
     ):
         authorized = await authenticator.get_authenticated_user(
-            None, {'username': 'override_admin', 'password': 'override_admin'}
+            None,
+            {'username': 'override_admin', 'password': 'override_admin'},
         )
     assert authorized['name'] == 'override_admin'
     assert authorized['admin'] is True

jupyterhub/tests/test_db.py

@@ -44,7 +44,9 @@ def generate_old_db(env_dir, hub_version, db_url):
 
 # changes to this version list must also be reflected
 # in ci/init-db.sh
-@pytest.mark.parametrize('hub_version', ["1.1.0", "1.2.2", "1.3.0", "1.5.0", "2.1.1"])
+@pytest.mark.parametrize(
+    'hub_version', ['1.1.0', '1.2.2', '1.3.0', '1.5.0', '2.1.1', '3.1.1']
+)
 async def test_upgrade(tmpdir, hub_version):
     db_url = os.getenv('JUPYTERHUB_TEST_DB_URL')
     if db_url:

jupyterhub/tests/test_roles.py

@@ -236,6 +236,16 @@ def test_orm_roles_delete_cascade(db):
             ['tokens!group=hobbits'],
             {'tokens!group=hobbits', 'read:tokens!group=hobbits'},
         ),
+        (
+            ['admin:services'],
+            {
+                'read:roles:services',
+                'read:services:name',
+                'admin:services',
+                'list:services',
+                'read:services',
+            },
+        ),
     ],
 )
 def test_get_expanded_scopes(db, scopes, expected_scopes):

jupyterhub/tests/test_singleuser.py

@@ -2,6 +2,7 @@
 import os
 import sys
 from contextlib import nullcontext
+from pprint import pprint
 from subprocess import CalledProcessError, check_output
 from unittest import mock
 from urllib.parse import urlencode, urlparse
@@ -171,9 +172,7 @@ async def test_disable_user_config(request, app, tmpdir, full_spawn):
     )
     r.raise_for_status()
     info = r.json()
-    import pprint
-
-    pprint.pprint(info)
+    pprint(info)
     assert info['disable_user_config']
     server_config = info['config']
     settings = info['settings']
@@ -198,6 +197,79 @@ async def test_disable_user_config(request, app, tmpdir, full_spawn):
                 assert_not_in_home(path, key)
 
 
+@pytest.mark.parametrize("extension", [True, False])
+@pytest.mark.parametrize("notebook_dir", ["", "~", "~/sub", "ABS"])
+async def test_notebook_dir(
+    request, app, tmpdir, user, full_spawn, extension, notebook_dir
+):
+    if extension:
+        try:
+            import jupyter_server  # noqa
+        except ImportError:
+            pytest.skip("needs jupyter-server 2")
+        else:
+            if jupyter_server.version_info < (2,):
+                pytest.skip("needs jupyter-server 2")
+
+    token = user.new_api_token(scopes=["access:servers!user"])
+    headers = {"Authorization": f"Bearer {token}"}
+
+    spawner = user.spawner
+    if extension:
+        user.spawner.environment["JUPYTERHUB_SINGLEUSER_EXTENSION"] = "1"
+    else:
+        user.spawner.environment["JUPYTERHUB_SINGLEUSER_EXTENSION"] = "0"
+
+    home_dir = tmpdir.join("home").mkdir()
+    sub_dir = home_dir.join("sub").mkdir()
+    with sub_dir.join("subfile.txt").open("w") as f:
+        f.write("txt\n")
+    abs_dir = tmpdir.join("abs").mkdir()
+    with abs_dir.join("absfile.txt").open("w") as f:
+        f.write("absfile\n")
+
+    if notebook_dir:
+        expected_root_dir = notebook_dir.replace("ABS", str(abs_dir)).replace(
+            "~", str(home_dir)
+        )
+    else:
+        expected_root_dir = str(home_dir)
+
+    spawner.notebook_dir = notebook_dir.replace("ABS", str(abs_dir))
+
+    # home_dir is defined on SimpleSpawner
+    user.spawner.home_dir = home = str(home_dir)
+    spawner.environment["HOME"] = home
+    await user.spawn()
+    await app.proxy.add_user(user)
+    url = public_url(app, user)
+    r = await async_requests.get(
+        url_path_join(public_url(app, user), 'jupyterhub-test-info'), headers=headers
+    )
+    r.raise_for_status()
+    info = r.json()
+    pprint(info)
+
+    assert info["root_dir"] == expected_root_dir
+    # secondary check: make sure it has the intended effect on root_dir
+    r = await async_requests.get(
+        url_path_join(public_url(app, user), 'api/contents/'), headers=headers
+    )
+    r.raise_for_status()
+    root_contents = sorted(item['name'] for item in r.json()['content'])
+
+    # check contents
+    if not notebook_dir or notebook_dir == "~":
+        # use any to avoid counting possible automatically created files in $HOME
+        assert 'sub' in root_contents
+    elif notebook_dir == "ABS":
+        assert 'absfile.txt' in root_contents
+    elif notebook_dir == "~/sub":
+        assert 'subfile.txt' in root_contents
+    else:
+        raise ValueError(f"No contents check for {notebook_dir=}")
+
+
 def test_help_output():
     out = check_output(
         [sys.executable, '-m', 'jupyterhub.singleuser', '--help-all']

requirements.txt

@@ -6,7 +6,7 @@ jinja2>=2.11.0
 jupyter_telemetry>=0.1.0
 oauthlib>=3.0
 packaging
-pamela; sys_platform != 'win32'
+pamela>=1.1.0; sys_platform != 'win32'
 prometheus_client>=0.4.0
 psutil>=5.6.5; sys_platform == 'win32'
 python-dateutil

share/jupyterhub/templates/login.html

@@ -52,7 +52,6 @@
       class="form-control"
       name="username"
       val="{{username}}"
-      tabindex="1"
       autofocus="autofocus"
     />
     <label for='password_input'>Password:</label>
@@ -62,8 +61,16 @@
       autocomplete="current-password"
       name="password"
       id="password_input"
-      tabindex="2"
     />
+    {% if authenticator.request_otp %}
+    <label for='otp_input'>{{ authenticator.otp_prompt }}</label>
+    <input
+      class="form-control"
+      autocomplete="one-time-password"
+      name="otp"
+      id="otp_input"
+    />
+    {% endif %}
 
     <div class="feedback-container">
       <input