Bump to 3.1.1

Backport PR #4316 on branch 3.x (changelog for 3.1.1)

.github/workflows/test.yml

@@ -29,6 +29,7 @@ env:
   # UTF-8 content may be interpreted as ascii and causes errors without this.
   LANG: C.UTF-8
   PYTEST_ADDOPTS: "--verbose --color=yes"
+  SQLALCHEMY_WARN_20: "1"
 
 permissions:
   contents: read
@@ -140,7 +141,7 @@ jobs:
       - name: Install Python dependencies
         run: |
           pip install --upgrade pip
-          pip install ".[test]"
+          pip install -e ".[test]"
 
           if [ "${{ matrix.oldest_dependencies }}" != "" ]; then
             # take any dependencies in requirements.txt such as tornado>=5.0
@@ -152,6 +153,7 @@ jobs:
 
           if [ "${{ matrix.main_dependencies }}" != "" ]; then
               pip install git+https://github.com/ipython/traitlets#egg=traitlets --force
+              pip install --upgrade --pre sqlalchemy
           fi
           if [ "${{ matrix.legacy_notebook }}" != "" ]; then
               pip uninstall jupyter_server --yes

ci/check_installed_data.py

@@ -2,19 +2,34 @@
 # Check that installed package contains everything we expect
 
 
-import os
+from pathlib import Path
 
+import jupyterhub
 from jupyterhub._data import DATA_FILES_PATH
 
-print("Checking jupyterhub._data")
+print("Checking jupyterhub._data", end=" ")
-print(f"DATA_FILES_PATH={DATA_FILES_PATH}")
+print(f"DATA_FILES_PATH={DATA_FILES_PATH}", end=" ")
-assert os.path.exists(DATA_FILES_PATH), DATA_FILES_PATH
+DATA_FILES_PATH = Path(DATA_FILES_PATH)
+assert DATA_FILES_PATH.is_dir(), DATA_FILES_PATH
 for subpath in (
     "templates/page.html",
     "static/css/style.min.css",
     "static/components/jquery/dist/jquery.js",
     "static/js/admin-react.js",
 ):
-    path = os.path.join(DATA_FILES_PATH, subpath)
+    path = DATA_FILES_PATH / subpath
-    assert os.path.exists(path), path
+    assert path.is_file(), path
+
+print("OK")
+
+print("Checking package_data", end=" ")
+jupyterhub_path = Path(jupyterhub.__file__).parent.resolve()
+for subpath in (
+    "alembic.ini",
+    "alembic/versions/833da8570507_rbac.py",
+    "event-schemas/server-actions/v1.yaml",
+):
+    path = jupyterhub_path / subpath
+    assert path.is_file(), path
+
 print("OK")

docs/source/_static/rest-api.yml

@@ -6,7 +6,7 @@ info:
   description: The REST API for JupyterHub
   license:
     name: BSD-3-Clause
-  version: 3.1.0
+  version: 3.1.1
 servers:
   - url: /hub/api
 security:

docs/source/changelog.md

@@ -8,6 +8,17 @@ command line for details.
 
 ## 3.1
 
+### 3.1.1 - 2023-01-27
+
+3.1.1 has only tiny bugfixes, enabling compatibility with the latest sqlalchemy 2.0 release, and fixing some metadata files that were not being included in wheel installs.
+
+([full changelog](https://github.com/jupyterhub/jupyterhub/compare/3.1.0...3.1.1))
+
+#### Bugs fixed
+
+- Backport PR #4303 on branch 3.x: make sure event-schemas are installed [#4317](https://github.com/jupyterhub/jupyterhub/pull/4317) ([@minrk](https://github.com/minrk))
+- Backport PR #4302 on branch 3.x: sqlalchemy 2 compatibility [#4315](https://github.com/jupyterhub/jupyterhub/pull/4315) ([@minrk](https://github.com/minrk))
+
 ### 3.1.0 - 2022-12-05
 
 3.1.0 is a small release, fixing various bugs and introducing some small new features and metrics.

jupyterhub/_version.py

@@ -2,7 +2,7 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 # version_info updated by running `tbump`
-version_info = (3, 1, 0, "", "")
+version_info = (3, 1, 1, "", "")
 
 # pep 440 version: no dot before beta/rc, but before .dev
 # 0.1.0rc1

jupyterhub/apihandlers/groups.py

@@ -94,8 +94,9 @@ class GroupListAPIHandler(_GroupAPIHandler):
             # create the group
             self.log.info("Creating new group %s with %i users", name, len(users))
             self.log.debug("Users: %s", usernames)
-            group = orm.Group(name=name, users=users)
+            group = orm.Group(name=name)
             self.db.add(group)
+            group.users = users
             self.db.commit()
             created.append(group)
         self.write(json.dumps([self.group_model(group) for group in created]))
@@ -131,8 +132,9 @@ class GroupAPIHandler(_GroupAPIHandler):
         # create the group
         self.log.info("Creating new group %s with %i users", group_name, len(users))
         self.log.debug("Users: %s", usernames)
-        group = orm.Group(name=group_name, users=users)
+        group = orm.Group(name=group_name)
         self.db.add(group)
+        group.users = users
         self.db.commit()
         self.write(json.dumps(self.group_model(group)))
         self.set_status(201)

jupyterhub/app.py

@@ -1948,9 +1948,9 @@ class JupyterHub(Application):
             user = orm.User.find(db, name)
             if user is None:
                 user = orm.User(name=name, admin=True)
+                db.add(user)
                 roles.assign_default_roles(self.db, entity=user)
                 new_users.append(user)
-                db.add(user)
             else:
                 user.admin = True
         # the admin_users config variable will never be used after this point.
@@ -2343,6 +2343,7 @@ class JupyterHub(Application):
             if orm_service is None:
                 # not found, create a new one
                 orm_service = orm.Service(name=name)
+                self.db.add(orm_service)
                 if spec.get('admin', False):
                     self.log.warning(
                         f"Service {name} sets `admin: True`, which is deprecated in JupyterHub 2.0."
@@ -2351,7 +2352,6 @@ class JupyterHub(Application):
                         "the Service admin flag will be ignored."
                     )
                     roles.update_roles(self.db, entity=orm_service, roles=['admin'])
-                self.db.add(orm_service)
             orm_service.admin = spec.get('admin', False)
             self.db.commit()
             service = Service(

jupyterhub/oauth/provider.py

@@ -257,16 +257,16 @@ class JupyterHubRequestValidator(RequestValidator):
             raise ValueError("No such client: %s" % client_id)
 
         orm_code = orm.OAuthCode(
-            client=orm_client,
             code=code['code'],
             # oauth has 5 minutes to complete
             expires_at=int(orm.OAuthCode.now() + 300),
             scopes=list(request.scopes),
-            user=request.user.orm_user,
             redirect_uri=orm_client.redirect_uri,
             session_id=request.session_id,
         )
         self.db.add(orm_code)
+        orm_code.client = orm_client
+        orm_code.user = request.user.orm_user
         self.db.commit()
 
     def get_authorization_code_scopes(self, client_id, code, redirect_uri, request):

jupyterhub/orm.py

@@ -8,7 +8,9 @@ from datetime import datetime, timedelta
 
 import alembic.command
 import alembic.config
+import sqlalchemy
 from alembic.script import ScriptDirectory
+from packaging.version import parse as parse_version
 from sqlalchemy import (
     Boolean,
     Column,
@@ -24,8 +26,8 @@ from sqlalchemy import (
     inspect,
     or_,
     select,
+    text,
 )
-from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import (
     Session,
     backref,
@@ -34,6 +36,13 @@ from sqlalchemy.orm import (
     relationship,
     sessionmaker,
 )
+
+try:
+    from sqlalchemy.orm import declarative_base
+except ImportError:
+    # sqlalchemy < 1.4
+    from sqlalchemy.ext.declarative import declarative_base
+
 from sqlalchemy.pool import StaticPool
 from sqlalchemy.types import LargeBinary, Text, TypeDecorator
 from tornado.log import app_log
@@ -749,6 +758,7 @@ class APIToken(Hashed, Base):
             session_id=session_id,
             scopes=list(scopes),
         )
+        db.add(orm_token)
         orm_token.token = token
         if user:
             assert user.id is not None
@@ -759,7 +769,6 @@ class APIToken(Hashed, Base):
         if expires_in is not None:
             orm_token.expires_at = cls.now() + timedelta(seconds=expires_in)
 
-        db.add(orm_token)
         db.commit()
         return token
 
@@ -901,7 +910,7 @@ def register_ping_connection(engine):
     """
 
     @event.listens_for(engine, "engine_connect")
-    def ping_connection(connection, branch):
+    def ping_connection(connection, branch=None):
         if branch:
             # "branch" refers to a sub-connection of a connection,
             # we don't want to bother pinging on these.
@@ -912,11 +921,17 @@ def register_ping_connection(engine):
         save_should_close_with_result = connection.should_close_with_result
         connection.should_close_with_result = False
 
+        if parse_version(sqlalchemy.__version__) < parse_version("1.4"):
+            one = [1]
+        else:
+            one = 1
+
         try:
-            # run a SELECT 1.   use a core select() so that
+            # run a SELECT 1. use a core select() so that
             # the SELECT of a scalar value without a table is
             # appropriately formatted for the backend
-            connection.scalar(select([1]))
+            with connection.begin() as transaction:
+                connection.scalar(select(one))
         except exc.DBAPIError as err:
             # catch SQLAlchemy's DBAPIError, which is a wrapper
             # for the DBAPI's exception.  It includes a .connection_invalidated
@@ -931,7 +946,8 @@ def register_ping_connection(engine):
                 # itself and establish a new connection.  The disconnect detection
                 # here also causes the whole connection pool to be invalidated
                 # so that all stale connections are discarded.
-                connection.scalar(select([1]))
+                with connection.begin() as transaction:
+                    connection.scalar(select(one))
             else:
                 raise
         finally:
@@ -955,7 +971,13 @@ def check_db_revision(engine):
 
     from .dbutil import _temp_alembic_ini
 
-    with _temp_alembic_ini(engine.url) as ini:
+    if hasattr(engine.url, "render_as_string"):
+        # sqlalchemy >= 1.4
+        engine_url = engine.url.render_as_string(hide_password=False)
+    else:
+        engine_url = str(engine.url)
+
+    with _temp_alembic_ini(engine_url) as ini:
         cfg = alembic.config.Config(ini)
         scripts = ScriptDirectory.from_config(cfg)
         head = scripts.get_heads()[0]
@@ -990,9 +1012,10 @@ def check_db_revision(engine):
 
     # check database schema version
     # it should always be defined at this point
-    alembic_revision = engine.execute(
+    with engine.begin() as connection:
-        'SELECT version_num FROM alembic_version'
+        alembic_revision = connection.execute(
-    ).first()[0]
+            text('SELECT version_num FROM alembic_version')
+        ).first()[0]
     if alembic_revision == head:
         app_log.debug("database schema version found: %s", alembic_revision)
     else:
@@ -1009,13 +1032,16 @@ def mysql_large_prefix_check(engine):
     """Check mysql has innodb_large_prefix set"""
     if not str(engine.url).startswith('mysql'):
         return False
-    variables = dict(
+    with engine.begin() as connection:
-        engine.execute(
+        variables = dict(
-            'show variables where variable_name like '
+            connection.execute(
-            '"innodb_large_prefix" or '
+                text(
-            'variable_name like "innodb_file_format";'
+                    'show variables where variable_name like '
-        ).fetchall()
+                    '"innodb_large_prefix" or '
-    )
+                    'variable_name like "innodb_file_format";'
+                )
+            ).fetchall()
+        )
     if (
         variables.get('innodb_file_format', 'Barracuda') == 'Barracuda'
         and variables.get('innodb_large_prefix', 'ON') == 'ON'

jupyterhub/tests/test_api.py

@@ -521,11 +521,12 @@ async def test_get_self(app):
     db.add(oauth_client)
     db.commit()
     oauth_token = orm.APIToken(
-        user=u.orm_user,
-        oauth_client=oauth_client,
         token=token,
     )
     db.add(oauth_token)
+    oauth_token.user = u.orm_user
+    oauth_token.oauth_client = oauth_client
+
     db.commit()
     r = await api_request(
         app,
@@ -2117,13 +2118,13 @@ def test_shutdown(app):
 
     def stop():
         stop.called = True
-        loop.call_later(1, real_stop)
+        loop.call_later(2, real_stop)
 
     real_cleanup = app.cleanup
 
     def cleanup():
         cleanup.called = True
-        return real_cleanup()
+        loop.call_later(1, real_cleanup)
 
     app.cleanup = cleanup
 

jupyterhub/tests/test_orm.py

@@ -323,7 +323,9 @@ def test_spawner_delete_cascade(db):
     db.add(user)
     db.commit()
 
-    spawner = orm.Spawner(user=user)
+    spawner = orm.Spawner()
+    db.add(spawner)
+    spawner.user = user
     db.commit()
     spawner.server = server = orm.Server()
     db.commit()
@@ -350,16 +352,19 @@ def test_user_delete_cascade(db):
     # these should all be deleted automatically when the user goes away
     user.new_api_token()
     api_token = user.api_tokens[0]
-    spawner = orm.Spawner(user=user)
+    spawner = orm.Spawner()
+    db.add(spawner)
+    spawner.user = user
     db.commit()
     spawner.server = server = orm.Server()
-    oauth_code = orm.OAuthCode(client=oauth_client, user=user)
+    oauth_code = orm.OAuthCode()
     db.add(oauth_code)
-    oauth_token = orm.APIToken(
+    oauth_code.client = oauth_client
-        oauth_client=oauth_client,
+    oauth_code.user = user
-        user=user,
+    oauth_token = orm.APIToken()
-    )
     db.add(oauth_token)
+    oauth_token.oauth_client = oauth_client
+    oauth_token.user = user
     db.commit()
 
     # record all of the ids
@@ -390,13 +395,14 @@ def test_oauth_client_delete_cascade(db):
 
     # create a bunch of objects that reference the User
     # these should all be deleted automatically when the user goes away
-    oauth_code = orm.OAuthCode(client=oauth_client, user=user)
+    oauth_code = orm.OAuthCode()
     db.add(oauth_code)
-    oauth_token = orm.APIToken(
+    oauth_code.client = oauth_client
-        oauth_client=oauth_client,
+    oauth_code.user = user
-        user=user,
+    oauth_token = orm.APIToken()
-    )
     db.add(oauth_token)
+    oauth_token.oauth_client = oauth_client
+    oauth_token.user = user
     db.commit()
     assert user.api_tokens == [oauth_token]
 
@@ -517,11 +523,11 @@ def test_expiring_oauth_token(app, user):
     db.add(client)
     orm_token = orm.APIToken(
         token=token,
-        oauth_client=client,
-        user=user,
         expires_at=now() + timedelta(seconds=30),
     )
     db.add(orm_token)
+    orm_token.oauth_client = client
+    orm_token.user = user
     db.commit()
 
     found = orm.APIToken.find(db, token)

jupyterhub/tests/test_pages.py

@@ -1045,11 +1045,10 @@ async def test_oauth_token_page(app):
     user = app.users[orm.User.find(app.db, name)]
     client = orm.OAuthClient(identifier='token')
     app.db.add(client)
-    oauth_token = orm.APIToken(
+    oauth_token = orm.APIToken()
-        oauth_client=client,
-        user=user,
-    )
     app.db.add(oauth_token)
+    oauth_token.oauth_client = client
+    oauth_token.user = user
     app.db.commit()
     r = await get_page('token', app, cookies=cookies)
     r.raise_for_status()

jupyterhub/tests/test_roles.py

@@ -3,9 +3,11 @@
 # Distributed under the terms of the Modified BSD License.
 import json
 import os
+import warnings
 
 import pytest
 from pytest import mark
+from sqlalchemy.exc import SADeprecationWarning
 from tornado.log import app_log
 
 from .. import orm, roles
@@ -343,7 +345,13 @@ async def test_creating_roles(app, role, role_def, response_type, response):
     # make sure no warnings/info logged when the role exists and its definition hasn't been changed
     elif response_type == 'no-log':
         with pytest.warns(response) as record:
+            # don't catch already-suppressed sqlalchemy warnings
+            warnings.simplefilter("ignore", SADeprecationWarning)
             roles.create_role(db, role_def)
+
+        for warning in record.list:
+            # show warnings for debugging
+            print("Unexpected warning", warning)
         assert not record.list
         role = orm.Role.find(db, role_def['name'])
         assert role is not None

jupyterhub/tests/utils.py

@@ -6,12 +6,27 @@ from concurrent.futures import ThreadPoolExecutor
 import pytest
 import requests
 from certipy import Certipy
+from sqlalchemy import text
 
 from jupyterhub import metrics, orm
 from jupyterhub.objects import Server
 from jupyterhub.roles import assign_default_roles, update_roles
 from jupyterhub.utils import url_path_join as ujoin
 
+try:
+    from sqlalchemy.exc import RemovedIn20Warning
+except ImportError:
+
+    class RemovedIn20Warning(DeprecationWarning):
+        """
+        I only exist so I can be used in warnings filters in pytest.ini
+
+        I will never be displayed.
+
+        sqlalchemy 1.4 introduces RemovedIn20Warning,
+        but we still test against older sqlalchemy.
+        """
+
 
 class _AsyncRequests:
     """Wrapper around requests to return a Future from request methods
@@ -84,8 +99,8 @@ def check_db_locks(func):
         def _check(_=None):
             temp_session = app.session_factory()
             try:
-                temp_session.execute('CREATE TABLE dummy (foo INT)')
+                temp_session.execute(text('CREATE TABLE dummy (foo INT)'))
-                temp_session.execute('DROP TABLE dummy')
+                temp_session.execute(text('DROP TABLE dummy'))
             finally:
                 temp_session.close()
 

jupyterhub/user.py

@@ -416,9 +416,10 @@ class User:
                 yield orm_spawner
 
     def _new_orm_spawner(self, server_name):
-        """Creat the low-level orm Spawner object"""
+        """Create the low-level orm Spawner object"""
-        orm_spawner = orm.Spawner(user=self.orm_user, name=server_name)
+        orm_spawner = orm.Spawner(name=server_name)
         self.db.add(orm_spawner)
+        orm_spawner.user = self.orm_user
         self.db.commit()
         assert server_name in self.orm_spawners
         return orm_spawner

pyproject.toml

@@ -43,7 +43,7 @@ target_version = [
 github_url = "https://github.com/jupyterhub/jupyterhub"
 
 [tool.tbump.version]
-current = "3.1.0"
+current = "3.1.1"
 
 # Example of a semver regexp.
 # Make sure this matches current_version before

pytest.ini

@@ -18,3 +18,7 @@ markers =
     slow: mark a test as slow
     role: mark as a test for roles
     selenium: web tests that run with selenium
+
+filterwarnings =
+    error:.*:jupyterhub.tests.utils.RemovedIn20Warning
+    ignore:.*event listener has changed as of version 2.0.*:sqlalchemy.exc.SADeprecationWarning

setup.py

@@ -50,7 +50,12 @@ def get_package_data():
     (mostly alembic config)
     """
     package_data = {}
-    package_data['jupyterhub'] = ['alembic.ini', 'alembic/*', 'alembic/versions/*']
+    package_data['jupyterhub'] = [
+        'alembic.ini',
+        'alembic/*',
+        'alembic/versions/*',
+        'event-schemas/*/*.yaml',
+    ]
     return package_data