From eb19a7304469cf93873c99320a98aed3ea778891 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 9 Nov 2018 13:22:08 +0100 Subject: [PATCH 01/14] add Spawner.get_url for retrieving the spawner url. Occurs on startup. Default is a no-op and no Spawner needs to implement this. --- jupyterhub/spawner.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/jupyterhub/spawner.py b/jupyterhub/spawner.py index 6d0d2b9b..3dd141d1 100644 --- a/jupyterhub/spawner.py +++ b/jupyterhub/spawner.py @@ -661,6 +661,22 @@ class Spawner(LoggingConfigurable): return env + async def get_url(self): + """Get the URL to connect to the server + + Sometimes JupyterHub may ask the Spawner for its url. + This can occur e.g. when JupyterHub has restarted while a server was not finished starting, + giving Spawners a chance to recover the URL where their server is running. + + The default is to trust that JupyterHub has the right information. + Only override this method in Spawners that know how to + check the correct URL for the servers they start. + + This will only be asked of Spawners that claim to be running + (`poll()` returns `None`). + """ + return self.server.url + def template_namespace(self): """Return the template namespace for format-string formatting. From 69933e240febb4bdc60afbe06d82dac8815d60c4 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 23 Nov 2018 16:06:38 +0100 Subject: [PATCH 02/14] document specifying options via rest api --- docs/rest-api.yml | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/rest-api.yml b/docs/rest-api.yml index 1eff3a53..8ea61094 100644 --- a/docs/rest-api.yml +++ b/docs/rest-api.yml @@ -185,6 +185,15 @@ paths: in: path required: true type: string + - options: + description: | + Spawn options can be passed as a JSON body + when spawning via the API instead of spawn form. + The structure of the options + will depend on the Spawner's configuration. + in: body + required: false + type: object responses: '201': description: The user's notebook server has started @@ -217,13 +226,15 @@ paths: in: path required: true type: string - - name: remove + - options: description: | - Whether to fully remove the server, rather than just stop it. - Removing a server deletes things like the state of the stopped server. + Spawn options can be passed as a JSON body + when spawning via the API instead of spawn form. + The structure of the options + will depend on the Spawner's configuration. in: body required: false - type: boolean + type: object responses: '201': description: The user's notebook named-server has started @@ -242,6 +253,13 @@ paths: in: path required: true type: string + - name: remove + description: | + Whether to fully remove the server, rather than just stop it. + Removing a server deletes things like the state of the stopped server. + in: body + required: false + type: boolean responses: '204': description: The user's notebook named-server has stopped From 9aa2110409ba486c5a538bef55a802c547ea62a8 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Wed, 28 Nov 2018 14:41:48 +1300 Subject: [PATCH 03/14] Remove unused dependency --- jupyterhub/auth.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jupyterhub/auth.py b/jupyterhub/auth.py index 16550826..d3959e04 100644 --- a/jupyterhub/auth.py +++ b/jupyterhub/auth.py @@ -17,7 +17,6 @@ except Exception as e: _pamela_error = e from tornado.concurrent import run_on_executor -from tornado import gen from traitlets.config import LoggingConfigurable from traitlets import Bool, Set, Unicode, Dict, Any, default, observe From 2b265b252975445ddab3a7b7718c0d06590831c0 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 9 Nov 2018 13:23:18 +0100 Subject: [PATCH 04/14] Check if a Spawner is running at the given URL on startup avoids assuming partial spawns that may have resulted in running processes but not actually available servers are treated as fully working servers. --- jupyterhub/app.py | 34 ++++++++++++++++++++++++++++++++++ jupyterhub/user.py | 15 ++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 036fb6de..c533ec85 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -1687,6 +1687,40 @@ class JupyterHub(Application): spawner._log_name) status = -1 + if status is None: + # poll claims it's running. + # Check if it's really there + url_in_db = spawner.server.url + url = await spawner.get_url() + if url != url_in_db: + self.log.warning( + "%s had invalid url %s. Updating to %s", + spawner._log_name, url_in_db, url, + ) + urlinfo = urlparse(url) + spawner.server.protocol = urlinfo.scheme + spawner.server.ip = urlinfo.hostname + if urlinfo.port: + spawner.server.port = urlinfo.port + elif urlinfo.scheme == 'http': + spawner.server.port = 80 + elif urlinfo.scheme == 'https': + spawner.server.port = 443 + self.db.commit() + + self.log.debug( + "Verifying that %s is running at %s", + spawner._log_name, url, + ) + try: + await user._wait_up(spawner) + except TimeoutError: + self.log.error( + "%s does not appear to be running at %s, shutting it down.", + spawner._log_name, url, + ) + status = -1 + if status is None: self.log.info("%s still running", user.name) spawner.add_poll_callback(user_stopped, user, name) diff --git a/jupyterhub/user.py b/jupyterhub/user.py index 74003ed2..99e4c941 100644 --- a/jupyterhub/user.py +++ b/jupyterhub/user.py @@ -531,7 +531,7 @@ class User: self.settings['statsd'].incr('spawner.failure.error') e.reason = 'error' try: - await self.stop() + await self.stop(spawner.server_name) except Exception: self.log.error("Failed to cleanup {user}'s server that failed to start".format( user=self.name, @@ -550,6 +550,15 @@ class User: spawner.orm_spawner.state = spawner.get_state() db.commit() spawner._waiting_for_response = True + await self._wait_up(spawner) + + async def _wait_up(self, spawner): + """Wait for a server to finish starting. + + Shuts the server down if it doesn't respond within + spawner.http_timeout. + """ + server = spawner.server key = self.settings.get('internal_ssl_key') cert = self.settings.get('internal_ssl_cert') ca = self.settings.get('internal_ssl_ca') @@ -578,7 +587,7 @@ class User: )) self.settings['statsd'].incr('spawner.failure.http_error') try: - await self.stop() + await self.stop(spawner.server_name) except Exception: self.log.error("Failed to cleanup {user}'s server that failed to start".format( user=self.name, @@ -594,7 +603,7 @@ class User: finally: spawner._waiting_for_response = False spawner._start_pending = False - return self + return spawner async def stop(self, server_name=''): """Stop the user's spawner From 887f2a2c242b09badca5735518b84dfce0596a04 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 29 Nov 2018 17:14:29 +0100 Subject: [PATCH 05/14] it's spawner.name --- jupyterhub/user.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyterhub/user.py b/jupyterhub/user.py index 99e4c941..b5b80e16 100644 --- a/jupyterhub/user.py +++ b/jupyterhub/user.py @@ -531,7 +531,7 @@ class User: self.settings['statsd'].incr('spawner.failure.error') e.reason = 'error' try: - await self.stop(spawner.server_name) + await self.stop(spawner.name) except Exception: self.log.error("Failed to cleanup {user}'s server that failed to start".format( user=self.name, @@ -587,7 +587,7 @@ class User: )) self.settings['statsd'].incr('spawner.failure.http_error') try: - await self.stop(spawner.server_name) + await self.stop(spawner.name) except Exception: self.log.error("Failed to cleanup {user}'s server that failed to start".format( user=self.name, From 4b36dce29fac3ae3e90f4e8fd2ca41eb4bf56853 Mon Sep 17 00:00:00 2001 From: Clemens Tolboom Date: Mon, 3 Dec 2018 19:05:50 +0100 Subject: [PATCH 06/14] Fix markdown link into rst --- docs/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 41071bad..54d221e5 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -30,7 +30,7 @@ For convenient administration of the Hub, its users, and services, JupyterHub also provides a `REST API`_. The JupyterHub team and Project Jupyter value our community, and JupyterHub -follows the Jupyter [Community Guides](https://jupyter.readthedocs.io/en/latest/community/content-community.html). +follows the Jupyter `Community Guides `_. Contents ======== From 05874e9f819c6642f27b03a03008c6c57fdc648b Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Mon, 3 Dec 2018 20:33:22 -0800 Subject: [PATCH 07/14] Update incorrect anchor Closes #2338 --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 07e6ecb0..a19b07e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,5 @@ # Contributing -See the [contribution guide](https://jupyterhub.readthedocs.io/en/latest/index.html#contributor) section -at the JupyterHub documentation. \ No newline at end of file +See the [contribution guide](https://jupyterhub.readthedocs.io/en/latest/index.html#contributing) section +at the JupyterHub documentation. From 47e66580dbbceaffa30190fe48d965fe5d76b5a8 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 4 Dec 2018 15:02:53 +0100 Subject: [PATCH 08/14] =?UTF-8?q?Don=E2=80=99t=20mix=20no=5Fpatience=20and?= =?UTF-8?q?=20bad=5Fspawn?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this introduces a race between the early RuntimeError being tested and the no_patience causing handlers to return early if async start isn’t complete. With tornado coroutines, an early RuntimeError could be guaranteed to resolve promptly, but asyncio isn’t as consistent, possibly causing some of the recent flaky tests. --- jupyterhub/tests/test_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyterhub/tests/test_api.py b/jupyterhub/tests/test_api.py index d4e91cd0..80d7d777 100644 --- a/jupyterhub/tests/test_api.py +++ b/jupyterhub/tests/test_api.py @@ -718,7 +718,7 @@ def test_never_spawn(app, no_patience, never_spawn): @mark.gen_test -def test_bad_spawn(app, no_patience, bad_spawn): +def test_bad_spawn(app, bad_spawn): db = app.db name = 'prim' user = add_user(db, app=app, name=name) @@ -835,7 +835,7 @@ def test_progress_ready(request, app): @mark.gen_test -def test_progress_bad(request, app, no_patience, bad_spawn): +def test_progress_bad(request, app, bad_spawn): """Test progress API when spawner has already failed""" db = app.db name = 'simon' From 4229d68d236cd8b16adccc7e671f591b38270986 Mon Sep 17 00:00:00 2001 From: Will Starms Date: Thu, 6 Dec 2018 15:32:49 -0600 Subject: [PATCH 09/14] Fix bad tester/code --- jupyterhub/auth.py | 2 +- jupyterhub/tests/test_auth.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jupyterhub/auth.py b/jupyterhub/auth.py index d3959e04..f2f6e9d4 100644 --- a/jupyterhub/auth.py +++ b/jupyterhub/auth.py @@ -703,7 +703,7 @@ class PAMAuthenticator(LocalAuthenticator): # (returning None instead of just the username) as this indicates some sort of system failure admin_group_gids = {self._getgrnam(x).gr_gid for x in self.admin_groups} - user_group_gids = {x.gr_gid for x in self._getgrouplist(username, self._getpwnam(username).pw_gid)} + user_group_gids = set(self._getgrouplist(username, self._getpwnam(username).pw_gid)) admin_status = len(admin_group_gids & user_group_gids) != 0 except Exception as e: diff --git a/jupyterhub/tests/test_auth.py b/jupyterhub/tests/test_auth.py index e872d6bc..1cb1d484 100644 --- a/jupyterhub/tests/test_auth.py +++ b/jupyterhub/tests/test_auth.py @@ -67,10 +67,10 @@ def test_pam_auth_admin_groups(): system_users = [group_admin, also_group_admin, override_admin, non_admin] user_group_map = { - 'group_admin': [jh_users, jh_admins], - 'also_group_admin': [jh_users, wheel], - 'override_admin': [jh_users], - 'non_admin': [jh_users] + 'group_admin': [jh_users.gr_gid, jh_admins.gr_gid], + 'also_group_admin': [jh_users.gr_gid, wheel.gr_gid], + 'override_admin': [jh_users.gr_gid], + 'non_admin': [jh_users.gr_gid] } def getgrnam(name): From 53c3201c17836116addeb4fb4035813d10708c23 Mon Sep 17 00:00:00 2001 From: chang-zhijie <609212560@qq.com> Date: Tue, 11 Dec 2018 17:00:16 +0800 Subject: [PATCH 10/14] store tokens passed via url or header, not only url. --- jupyterhub/services/auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyterhub/services/auth.py b/jupyterhub/services/auth.py index 1020ae31..90627bc8 100644 --- a/jupyterhub/services/auth.py +++ b/jupyterhub/services/auth.py @@ -852,8 +852,8 @@ class HubAuthenticated(object): self._hub_auth_user_cache = None raise - # store ?token=... tokens passed via url in a cookie for future requests - url_token = self.get_argument('token', '') + # store tokens passed via url or header in a cookie for future requests + url_token = self.hub_auth.get_token(self) if ( user_model and url_token From d5f87fe09f3a72a46c0371a4f519c7e8ae6f377e Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 12 Dec 2018 12:44:02 +0100 Subject: [PATCH 11/14] update fixtures for pytest-asyncio instead of pytest-tornado verify that we don't make any async generator tests which would pass silently without running any tests --- dev-requirements.txt | 2 +- jupyterhub/tests/conftest.py | 29 +++++++++++++++++++++++------ jupyterhub/tests/utils.py | 8 ++++++-- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 4c90579a..0fc2169d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,7 +6,7 @@ coverage cryptography html5lib # needed for beautifulsoup pytest-cov -pytest-tornado +pytest-asyncio pytest>=3.3 notebook requests-mock diff --git a/jupyterhub/tests/conftest.py b/jupyterhub/tests/conftest.py index e44dd60c..0c490079 100644 --- a/jupyterhub/tests/conftest.py +++ b/jupyterhub/tests/conftest.py @@ -29,6 +29,7 @@ Fixtures to add functionality or spawning behavior import asyncio from getpass import getuser +import inspect import logging import os import sys @@ -55,6 +56,16 @@ import jupyterhub.services.service _db = None +def pytest_collection_modifyitems(items): + """add asyncio marker to all async tests""" + for item in items: + if inspect.iscoroutinefunction(item.obj): + item.add_marker('asyncio') + if hasattr(inspect, 'isasyncgenfunction'): + # double-check that we aren't mixing yield and async def + assert not inspect.isasyncgenfunction(item.obj) + + @fixture(scope='module') def ssl_tmpdir(tmpdir_factory): return tmpdir_factory.mktemp('ssl') @@ -126,15 +137,21 @@ def db(): @fixture(scope='module') -def io_loop(request): +def event_loop(request): + """Same as pytest-asyncio.event_loop, but re-scoped to module-level""" + event_loop = asyncio.new_event_loop() + asyncio.set_event_loop(event_loop) + return event_loop + + +@fixture(scope='module') +def io_loop(event_loop, request): """Same as pytest-tornado.io_loop, but re-scoped to module-level""" ioloop.IOLoop.configure(AsyncIOMainLoop) - aio_loop = asyncio.new_event_loop() - asyncio.set_event_loop(aio_loop) - io_loop = ioloop.IOLoop() + io_loop = AsyncIOMainLoop() io_loop.make_current() - assert asyncio.get_event_loop() is aio_loop - assert io_loop.asyncio_loop is aio_loop + assert asyncio.get_event_loop() is event_loop + assert io_loop.asyncio_loop is event_loop def _close(): io_loop.clear_current() diff --git a/jupyterhub/tests/utils.py b/jupyterhub/tests/utils.py index e2b4c1b1..a59bf87e 100644 --- a/jupyterhub/tests/utils.py +++ b/jupyterhub/tests/utils.py @@ -13,11 +13,15 @@ class _AsyncRequests: def __init__(self): self.executor = ThreadPoolExecutor(1) + real_submit = self.executor.submit + self.executor.submit = lambda *args, **kwargs: asyncio.wrap_future( + real_submit(*args, **kwargs) + ) def __getattr__(self, name): requests_method = getattr(requests, name) - return lambda *args, **kwargs: asyncio.wrap_future( - self.executor.submit(requests_method, *args, **kwargs) + return lambda *args, **kwargs: self.executor.submit( + requests_method, *args, **kwargs ) From 37cdba370f531c480a63078f10bdd78b7de5dfe1 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 12 Dec 2018 13:05:45 +0100 Subject: [PATCH 12/14] update tests for pytest-asyncio - remove gen_test marker - use async def - find/replace yield->await approximately one million times --- jupyterhub/tests/test_api.py | 514 ++++++++---------- jupyterhub/tests/test_app.py | 38 +- jupyterhub/tests/test_auth.py | 132 ++--- jupyterhub/tests/test_crypto.py | 22 +- jupyterhub/tests/test_db.py | 5 +- jupyterhub/tests/test_dummyauth.py | 21 +- .../tests/test_internal_ssl_connections.py | 21 +- jupyterhub/tests/test_named_servers.py | 40 +- jupyterhub/tests/test_orm.py | 32 +- jupyterhub/tests/test_pages.py | 305 +++++------ jupyterhub/tests/test_proxy.py | 71 ++- jupyterhub/tests/test_services.py | 39 +- jupyterhub/tests/test_services_auth.py | 74 ++- jupyterhub/tests/test_singleuser.py | 37 +- jupyterhub/tests/test_spawner.py | 141 +++-- jupyterhub/tests/test_utils.py | 2 - 16 files changed, 666 insertions(+), 828 deletions(-) diff --git a/jupyterhub/tests/test_api.py b/jupyterhub/tests/test_api.py index 80d7d777..3fdf0f3d 100644 --- a/jupyterhub/tests/test_api.py +++ b/jupyterhub/tests/test_api.py @@ -1,5 +1,6 @@ """Tests for the REST API.""" +import asyncio from datetime import datetime from concurrent.futures import Future import json @@ -84,8 +85,7 @@ def auth_header(db, name): @check_db_locks -@gen.coroutine -def api_request(app, *api_path, **kwargs): +async def api_request(app, *api_path, **kwargs): """Make an API request""" base_url = app.hub.url headers = kwargs.setdefault('headers', {}) @@ -102,7 +102,7 @@ def api_request(app, *api_path, **kwargs): if app.internal_ssl: kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key) kwargs["verify"] = app.internal_ssl_ca - resp = yield f(url, **kwargs) + resp = await f(url, **kwargs) assert "frame-ancestors 'self'" in resp.headers['Content-Security-Policy'] assert ujoin(app.hub.base_url, "security/csp-report") in resp.headers['Content-Security-Policy'] assert 'http' not in resp.headers['Content-Security-Policy'] @@ -116,10 +116,9 @@ def api_request(app, *api_path, **kwargs): # -------------------- -@mark.gen_test -def test_auth_api(app): +async def test_auth_api(app): db = app.db - r = yield api_request(app, 'authorizations', 'gobbledygook') + r = await api_request(app, 'authorizations', 'gobbledygook') assert r.status_code == 404 # make a new cookie token @@ -127,33 +126,32 @@ def test_auth_api(app): api_token = user.new_api_token() # check success: - r = yield api_request(app, 'authorizations/token', api_token) + r = await api_request(app, 'authorizations/token', api_token) assert r.status_code == 200 reply = r.json() assert reply['name'] == user.name # check fail - r = yield api_request(app, 'authorizations/token', api_token, + r = await api_request(app, 'authorizations/token', api_token, headers={'Authorization': 'no sir'}, ) assert r.status_code == 403 - r = yield api_request(app, 'authorizations/token', api_token, + r = await api_request(app, 'authorizations/token', api_token, headers={'Authorization': 'token: %s' % user.cookie_id}, ) assert r.status_code == 403 -@mark.gen_test -def test_referer_check(app): +async def test_referer_check(app): url = ujoin(public_host(app), app.hub.base_url) host = urlparse(url).netloc user = find_user(app.db, 'admin') if user is None: user = add_user(app.db, name='admin', admin=True) - cookies = yield app.login_user('admin') + cookies = await app.login_user('admin') - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers={ 'Authorization': '', 'Referer': 'null', @@ -161,7 +159,7 @@ def test_referer_check(app): ) assert r.status_code == 403 - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers={ 'Authorization': '', 'Referer': 'http://attack.com/csrf/vulnerability', @@ -169,7 +167,7 @@ def test_referer_check(app): ) assert r.status_code == 403 - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers={ 'Authorization': '', 'Referer': url, @@ -178,7 +176,7 @@ def test_referer_check(app): ) assert r.status_code == 200 - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers={ 'Authorization': '', 'Referer': ujoin(url, 'foo/bar/baz/bat'), @@ -240,10 +238,9 @@ def fill_user(model): TIMESTAMP = normalize_timestamp(datetime.now().isoformat() + 'Z') @mark.user -@mark.gen_test -def test_get_users(app): +async def test_get_users(app): db = app.db - r = yield api_request(app, 'users') + r = await api_request(app, 'users') assert r.status_code == 200 users = sorted(r.json(), key=lambda d: d['name']) @@ -260,19 +257,18 @@ def test_get_users(app): }), ] - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers=auth_header(db, 'user'), ) assert r.status_code == 403 @mark.user -@mark.gen_test -def test_get_self(app): +async def test_get_self(app): db = app.db # basic get self - r = yield api_request(app, 'user') + r = await api_request(app, 'user') r.raise_for_status() assert r.json()['kind'] == 'user' @@ -290,7 +286,7 @@ def test_get_self(app): ) db.add(oauth_token) db.commit() - r = yield api_request(app, 'user', headers={ + r = await api_request(app, 'user', headers={ 'Authorization': 'token ' + token, }) r.raise_for_status() @@ -298,18 +294,17 @@ def test_get_self(app): assert model['name'] == u.name # invalid auth gets 403 - r = yield api_request(app, 'user', headers={ + r = await api_request(app, 'user', headers={ 'Authorization': 'token notvalid', }) assert r.status_code == 403 @mark.user -@mark.gen_test -def test_add_user(app): +async def test_add_user(app): db = app.db name = 'newuser' - r = yield api_request(app, 'users', name, method='post') + r = await api_request(app, 'users', name, method='post') assert r.status_code == 201 user = find_user(db, name) assert user is not None @@ -318,10 +313,9 @@ def test_add_user(app): @mark.user -@mark.gen_test -def test_get_user(app): +async def test_get_user(app): name = 'user' - r = yield api_request(app, 'users', name) + r = await api_request(app, 'users', name) assert r.status_code == 200 user = normalize_user(r.json()) @@ -329,21 +323,19 @@ def test_get_user(app): @mark.user -@mark.gen_test -def test_add_multi_user_bad(app): - r = yield api_request(app, 'users', method='post') +async def test_add_multi_user_bad(app): + r = await api_request(app, 'users', method='post') assert r.status_code == 400 - r = yield api_request(app, 'users', method='post', data='{}') + r = await api_request(app, 'users', method='post', data='{}') assert r.status_code == 400 - r = yield api_request(app, 'users', method='post', data='[]') + r = await api_request(app, 'users', method='post', data='[]') assert r.status_code == 400 @mark.user -@mark.gen_test -def test_add_multi_user_invalid(app): +async def test_add_multi_user_invalid(app): app.authenticator.username_pattern = r'w.*' - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': ['Willow', 'Andrew', 'Tara']}) ) app.authenticator.username_pattern = '' @@ -352,11 +344,10 @@ def test_add_multi_user_invalid(app): @mark.user -@mark.gen_test -def test_add_multi_user(app): +async def test_add_multi_user(app): db = app.db names = ['a', 'b'] - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': names}), ) assert r.status_code == 201 @@ -371,7 +362,7 @@ def test_add_multi_user(app): assert not user.admin # try to create the same users again - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': names}), ) assert r.status_code == 409 @@ -379,7 +370,7 @@ def test_add_multi_user(app): names = ['a', 'b', 'ab'] # try to create the same users again - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': names}), ) assert r.status_code == 201 @@ -389,11 +380,10 @@ def test_add_multi_user(app): @mark.user -@mark.gen_test -def test_add_multi_user_admin(app): +async def test_add_multi_user_admin(app): db = app.db names = ['c', 'd'] - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': names, 'admin': True}), ) assert r.status_code == 201 @@ -409,35 +399,32 @@ def test_add_multi_user_admin(app): @mark.user -@mark.gen_test -def test_add_user_bad(app): +async def test_add_user_bad(app): db = app.db name = 'dne_newuser' - r = yield api_request(app, 'users', name, method='post') + r = await api_request(app, 'users', name, method='post') assert r.status_code == 400 user = find_user(db, name) assert user is None @mark.user -@mark.gen_test -def test_add_user_duplicate(app): +async def test_add_user_duplicate(app): db = app.db name = 'user' user = find_user(db, name) # double-check that it exists assert user is not None - r = yield api_request(app, 'users', name, method='post') + r = await api_request(app, 'users', name, method='post') # special 409 conflict for creating a user that already exists assert r.status_code == 409 @mark.user -@mark.gen_test -def test_add_admin(app): +async def test_add_admin(app): db = app.db name = 'newadmin' - r = yield api_request(app, 'users', name, method='post', + r = await api_request(app, 'users', name, method='post', data=json.dumps({'admin': True}), ) assert r.status_code == 201 @@ -448,27 +435,25 @@ def test_add_admin(app): @mark.user -@mark.gen_test -def test_delete_user(app): +async def test_delete_user(app): db = app.db mal = add_user(db, name='mal') - r = yield api_request(app, 'users', 'mal', method='delete') + r = await api_request(app, 'users', 'mal', method='delete') assert r.status_code == 204 @mark.user -@mark.gen_test -def test_make_admin(app): +async def test_make_admin(app): db = app.db name = 'admin2' - r = yield api_request(app, 'users', name, method='post') + r = await api_request(app, 'users', name, method='post') assert r.status_code == 201 user = find_user(db, name) assert user is not None assert user.name == name assert not user.admin - r = yield api_request(app, 'users', name, method='patch', + r = await api_request(app, 'users', name, method='patch', data=json.dumps({'admin': True}) ) assert r.status_code == 200 @@ -479,8 +464,7 @@ def test_make_admin(app): @mark.user -@mark.gen_test -def test_set_auth_state(app, auth_state_enabled): +async def test_set_auth_state(app, auth_state_enabled): auth_state = {'secret': 'hello'} db = app.db name = 'admin' @@ -488,58 +472,55 @@ def test_set_auth_state(app, auth_state_enabled): assert user is not None assert user.name == name - r = yield api_request(app, 'users', name, method='patch', + r = await api_request(app, 'users', name, method='patch', data=json.dumps({'auth_state': auth_state}) ) assert r.status_code == 200 - users_auth_state = yield user.get_auth_state() + users_auth_state = await user.get_auth_state() assert users_auth_state == auth_state @mark.user -@mark.gen_test -def test_user_set_auth_state(app, auth_state_enabled): +async def test_user_set_auth_state(app, auth_state_enabled): auth_state = {'secret': 'hello'} db = app.db name = 'user' user = find_user(db, name, app=app) assert user is not None assert user.name == name - user_auth_state = yield user.get_auth_state() + user_auth_state = await user.get_auth_state() assert user_auth_state is None - r = yield api_request( + r = await api_request( app, 'users', name, method='patch', data=json.dumps({'auth_state': auth_state}), headers=auth_header(app.db, name), ) assert r.status_code == 403 - user_auth_state = yield user.get_auth_state() + user_auth_state = await user.get_auth_state() assert user_auth_state is None @mark.user -@mark.gen_test -def test_admin_get_auth_state(app, auth_state_enabled): +async def test_admin_get_auth_state(app, auth_state_enabled): auth_state = {'secret': 'hello'} db = app.db name = 'admin' user = find_user(db, name, app=app) assert user is not None assert user.name == name - yield user.save_auth_state(auth_state) + await user.save_auth_state(auth_state) - r = yield api_request(app, 'users', name) + r = await api_request(app, 'users', name) assert r.status_code == 200 assert r.json()['auth_state'] == auth_state @mark.user -@mark.gen_test -def test_user_get_auth_state(app, auth_state_enabled): +async def test_user_get_auth_state(app, auth_state_enabled): # explicitly check that a user will not get their own auth state via the API auth_state = {'secret': 'hello'} db = app.db @@ -547,17 +528,16 @@ def test_user_get_auth_state(app, auth_state_enabled): user = find_user(db, name, app=app) assert user is not None assert user.name == name - yield user.save_auth_state(auth_state) + await user.save_auth_state(auth_state) - r = yield api_request(app, 'users', name, + r = await api_request(app, 'users', name, headers=auth_header(app.db, name)) assert r.status_code == 200 assert 'auth_state' not in r.json() -@mark.gen_test -def test_spawn(app): +async def test_spawn(app): db = app.db name = 'wash' user = add_user(db, app=app, name=name) @@ -566,7 +546,7 @@ def test_spawn(app): 'i': 5, } before_servers = sorted(db.query(orm.Server), key=lambda s: s.url) - r = yield api_request(app, 'users', name, 'server', method='post', + r = await api_request(app, 'users', name, 'server', method='post', data=json.dumps(options), ) assert r.status_code == 201 @@ -576,7 +556,7 @@ def test_spawn(app): spawner = app_user.spawner assert app_user.spawner.user_options == options assert not app_user.spawner._spawn_pending - status = yield app_user.spawner.poll() + status = await app_user.spawner.poll() assert status is None assert spawner.server.base_url == ujoin(app.base_url, 'user/%s' % name) + '/' @@ -585,26 +565,26 @@ def test_spawn(app): if app.internal_ssl: kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key) kwargs["verify"] = app.internal_ssl_ca - r = yield async_requests.get(url, **kwargs) + r = await async_requests.get(url, **kwargs) assert r.status_code == 200 assert r.text == spawner.server.base_url - r = yield async_requests.get(ujoin(url, 'args'), **kwargs) + r = await async_requests.get(ujoin(url, 'args'), **kwargs) assert r.status_code == 200 argv = r.json() assert '--port' in ' '.join(argv) - r = yield async_requests.get(ujoin(url, 'env'), **kwargs) + r = await async_requests.get(ujoin(url, 'env'), **kwargs) env = r.json() for expected in ['JUPYTERHUB_USER', 'JUPYTERHUB_BASE_URL', 'JUPYTERHUB_API_TOKEN']: assert expected in env if app.subdomain_host: assert env['JUPYTERHUB_HOST'] == app.subdomain_host - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') assert r.status_code == 204 assert 'pid' not in user.orm_spawners[''].state - status = yield app_user.spawner.poll() + status = await app_user.spawner.poll() assert status == 0 # check that we cleaned up after ourselves @@ -616,8 +596,7 @@ def test_spawn(app): assert app.users.count_active_users()['pending'] == 0 -@mark.gen_test -def test_spawn_handler(app): +async def test_spawn_handler(app): """Test that the requesting Handler is passed to Spawner.handler""" db = app.db name = 'salmon' @@ -625,7 +604,7 @@ def test_spawn_handler(app): app_user = app.users[name] # spawn via API with ?foo=bar - r = yield api_request(app, 'users', name, 'server', method='post', params={'foo': 'bar'}) + r = await api_request(app, 'users', name, 'server', method='post', params={'foo': 'bar'}) r.raise_for_status() # verify that request params got passed down @@ -634,24 +613,23 @@ def test_spawn_handler(app): if app.external_certs: kwargs['verify'] = app.external_certs['files']['ca'] url = public_url(app, user) - r = yield async_requests.get(ujoin(url, 'env'), **kwargs) + r = await async_requests.get(ujoin(url, 'env'), **kwargs) env = r.json() assert 'HANDLER_ARGS' in env assert env['HANDLER_ARGS'] == 'foo=bar' # make user spawner.handler doesn't persist after spawn finishes assert app_user.spawner.handler is None - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() @mark.slow -@mark.gen_test -def test_slow_spawn(app, no_patience, slow_spawn): +async def test_slow_spawn(app, no_patience, slow_spawn): db = app.db name = 'zoe' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() assert r.status_code == 202 assert app_user.spawner is not None @@ -659,83 +637,78 @@ def test_slow_spawn(app, no_patience, slow_spawn): assert not app_user.spawner._stop_pending assert app.users.count_active_users()['pending'] == 1 - @gen.coroutine - def wait_spawn(): + async def wait_spawn(): while not app_user.running: - yield gen.sleep(0.1) + await gen.sleep(0.1) - yield wait_spawn() + await wait_spawn() assert not app_user.spawner._spawn_pending - status = yield app_user.spawner.poll() + status = await app_user.spawner.poll() assert status is None - @gen.coroutine - def wait_stop(): + async def wait_stop(): while app_user.spawner._stop_pending: - yield gen.sleep(0.1) + await gen.sleep(0.1) - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() assert r.status_code == 202 assert app_user.spawner is not None assert app_user.spawner._stop_pending - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() assert r.status_code == 202 assert app_user.spawner is not None assert app_user.spawner._stop_pending - yield wait_stop() + await wait_stop() assert not app_user.spawner._stop_pending assert app_user.spawner is not None - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') # 204 deleted if there's no such server assert r.status_code == 204 assert app.users.count_active_users()['pending'] == 0 assert app.users.count_active_users()['active'] == 0 -@mark.gen_test -def test_never_spawn(app, no_patience, never_spawn): +async def test_never_spawn(app, no_patience, never_spawn): db = app.db name = 'badger' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert app_user.spawner is not None assert app_user.spawner._spawn_pending assert app.users.count_active_users()['pending'] == 1 while app_user.spawner.pending: - yield gen.sleep(0.1) + await gen.sleep(0.1) print(app_user.spawner.pending) assert not app_user.spawner._spawn_pending - status = yield app_user.spawner.poll() + status = await app_user.spawner.poll() assert status is not None # failed spawn should decrements pending count assert app.users.count_active_users()['pending'] == 0 -@mark.gen_test -def test_bad_spawn(app, bad_spawn): +async def test_bad_spawn(app, bad_spawn): db = app.db name = 'prim' user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 500 assert app.users.count_active_users()['pending'] == 0 -@mark.gen_test -def test_slow_bad_spawn(app, no_patience, slow_bad_spawn): +async def test_slow_bad_spawn(app, no_patience, slow_bad_spawn): db = app.db name = 'zaphod' user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() while user.spawner.pending: - yield gen.sleep(0.1) + await gen.sleep(0.1) # spawn failed assert not user.running assert app.users.count_active_users()['pending'] == 0 @@ -752,31 +725,30 @@ def next_event(it): return json.loads(line.split(':', 1)[1]) @mark.slow -@mark.gen_test -def test_progress(request, app, no_patience, slow_spawn): +async def test_progress(request, app, no_patience, slow_spawn): db = app.db name = 'martin' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) assert r.headers['content-type'] == 'text/event-stream' ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 0, 'message': 'Server requested', } - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 50, 'message': 'Spawning server...', } - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) url = app_user.url assert evt == { 'progress': 100, @@ -787,32 +759,29 @@ def test_progress(request, app, no_patience, slow_spawn): } -@mark.gen_test -def test_progress_not_started(request, app): +async def test_progress_not_started(request, app): db = app.db name = 'nope' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server/progress') + r = await api_request(app, 'users', name, 'server/progress') assert r.status_code == 404 -@mark.gen_test -def test_progress_not_found(request, app): +async def test_progress_not_found(request, app): db = app.db name = 'noserver' - r = yield api_request(app, 'users', 'nosuchuser', 'server/progress') + r = await api_request(app, 'users', 'nosuchuser', 'server/progress') assert r.status_code == 404 app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server/progress') + r = await api_request(app, 'users', name, 'server/progress') assert r.status_code == 404 -@mark.gen_test -def test_progress_ready(request, app): +async def test_progress_ready(request, app): """Test progress API when spawner is already started e.g. a race between requesting progress and progress already being complete @@ -820,35 +789,34 @@ def test_progress_ready(request, app): db = app.db name = 'saga' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) assert r.headers['content-type'] == 'text/event-stream' ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 100 assert evt['ready'] assert evt['url'] == app_user.url -@mark.gen_test -def test_progress_bad(request, app, bad_spawn): +async def test_progress_bad(request, app, bad_spawn): """Test progress API when spawner has already failed""" db = app.db name = 'simon' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 500 - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) assert r.headers['content-type'] == 'text/event-stream' ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 100, 'failed': True, @@ -856,25 +824,24 @@ def test_progress_bad(request, app, bad_spawn): } -@mark.gen_test -def test_progress_bad_slow(request, app, no_patience, slow_bad_spawn): +async def test_progress_bad_slow(request, app, no_patience, slow_bad_spawn): """Test progress API when spawner fails while watching""" db = app.db name = 'eugene' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 202 - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) assert r.headers['content-type'] == 'text/event-stream' ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 0 - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 50 - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 100, 'failed': True, @@ -910,8 +877,7 @@ async def progress_forever_native(): """, globals()) -@mark.gen_test -def test_spawn_progress_cutoff(request, app, no_patience, slow_spawn): +async def test_spawn_progress_cutoff(request, app, no_patience, slow_spawn): """Progress events stop when Spawner finishes even if progress iterator is still going. @@ -926,26 +892,25 @@ def test_spawn_progress_cutoff(request, app, no_patience, slow_spawn): app_user.spawner.progress = progress_forever app_user.spawner.delay = 1 - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 0 - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 1, 'message': 'Stage 1', } - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 100 -@mark.gen_test -def test_spawn_limit(app, no_patience, slow_spawn, request): +async def test_spawn_limit(app, no_patience, slow_spawn, request): db = app.db p = mock.patch.dict(app.tornado_settings, {'concurrent_spawn_limit': 2}) @@ -958,24 +923,24 @@ def test_spawn_limit(app, no_patience, slow_spawn, request): users[0].spawner._start_future = Future() users[1].spawner._start_future = Future() for name in names: - yield api_request(app, 'users', name, 'server', method='post') + await api_request(app, 'users', name, 'server', method='post') assert app.users.count_active_users()['pending'] == 2 # ykka and hjarka's spawns are both pending. Essun should fail with 429 name = 'essun' user = add_user(db, app=app, name=name) user.spawner._start_future = Future() - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 429 # allow ykka to start users[0].spawner._start_future.set_result(None) # wait for ykka to finish while not users[0].running: - yield gen.sleep(0.1) + await gen.sleep(0.1) assert app.users.count_active_users()['pending'] == 1 - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() assert app.users.count_active_users()['pending'] == 2 users.append(user) @@ -983,20 +948,19 @@ def test_spawn_limit(app, no_patience, slow_spawn, request): for user in users[1:]: user.spawner._start_future.set_result(None) while not all(u.running for u in users): - yield gen.sleep(0.1) + await gen.sleep(0.1) # everybody's running, pending count should be back to 0 assert app.users.count_active_users()['pending'] == 0 for u in users: u.spawner.delay = 0 - r = yield api_request(app, 'users', u.name, 'server', method='delete') + r = await api_request(app, 'users', u.name, 'server', method='delete') r.raise_for_status() while any(u.spawner.active for u in users): - yield gen.sleep(0.1) + await gen.sleep(0.1) @mark.slow -@mark.gen_test -def test_active_server_limit(app, request): +async def test_active_server_limit(app, request): db = app.db p = mock.patch.dict(app.tornado_settings, {'active_server_limit': 2}) @@ -1007,7 +971,7 @@ def test_active_server_limit(app, request): names = ['ykka', 'hjarka'] users = [ add_user(db, app=app, name=name) for name in names ] for name in names: - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() counts = app.users.count_active_users() assert counts['active'] == 2 @@ -1017,7 +981,7 @@ def test_active_server_limit(app, request): # ykka and hjarka's servers are running. Essun should fail with 429 name = 'essun' user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 429 counts = app.users.count_active_users() assert counts['active'] == 2 @@ -1025,13 +989,13 @@ def test_active_server_limit(app, request): assert counts['pending'] == 0 # stop one server - yield api_request(app, 'users', names[0], 'server', method='delete') + await api_request(app, 'users', names[0], 'server', method='delete') counts = app.users.count_active_users() assert counts['active'] == 1 assert counts['ready'] == 1 assert counts['pending'] == 0 - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() counts = app.users.count_active_users() assert counts['active'] == 2 @@ -1044,7 +1008,7 @@ def test_active_server_limit(app, request): for u in users: if not u.spawner.active: continue - r = yield api_request(app, 'users', u.name, 'server', method='delete') + r = await api_request(app, 'users', u.name, 'server', method='delete') r.raise_for_status() counts = app.users.count_active_users() @@ -1053,78 +1017,75 @@ def test_active_server_limit(app, request): assert counts['pending'] == 0 @mark.slow -@mark.gen_test -def test_start_stop_race(app, no_patience, slow_spawn): +async def test_start_stop_race(app, no_patience, slow_spawn): user = add_user(app.db, app, name='panda') spawner = user.spawner # start the server - r = yield api_request(app, 'users', user.name, 'server', method='post') + r = await api_request(app, 'users', user.name, 'server', method='post') assert r.status_code == 202 assert spawner.pending == 'spawn' # additional spawns while spawning shouldn't trigger a new spawn with mock.patch.object(spawner, 'start') as m: - r = yield api_request(app, 'users', user.name, 'server', method='post') + r = await api_request(app, 'users', user.name, 'server', method='post') assert r.status_code == 202 assert m.call_count == 0 # stop while spawning is not okay - r = yield api_request(app, 'users', user.name, 'server', method='delete') + r = await api_request(app, 'users', user.name, 'server', method='delete') assert r.status_code == 400 while not spawner.ready: - yield gen.sleep(0.1) + await gen.sleep(0.1) spawner.delay = 3 # stop the spawner - r = yield api_request(app, 'users', user.name, 'server', method='delete') + r = await api_request(app, 'users', user.name, 'server', method='delete') assert r.status_code == 202 assert spawner.pending == 'stop' # make sure we get past deleting from the proxy - yield gen.sleep(1) + await gen.sleep(1) # additional stops while stopping shouldn't trigger a new stop with mock.patch.object(spawner, 'stop') as m: - r = yield api_request(app, 'users', user.name, 'server', method='delete') + r = await api_request(app, 'users', user.name, 'server', method='delete') assert r.status_code == 202 assert m.call_count == 0 # start while stopping is not allowed with mock.patch.object(spawner, 'start') as m: - r = yield api_request(app, 'users', user.name, 'server', method='post') + r = await api_request(app, 'users', user.name, 'server', method='post') assert r.status_code == 400 while spawner.active: - yield gen.sleep(0.1) + await gen.sleep(0.1) # start after stop is okay - r = yield api_request(app, 'users', user.name, 'server', method='post') + r = await api_request(app, 'users', user.name, 'server', method='post') assert r.status_code == 202 -@mark.gen_test -def test_get_proxy(app): - r = yield api_request(app, 'proxy') +async def test_get_proxy(app): + r = await api_request(app, 'proxy') r.raise_for_status() reply = r.json() assert list(reply.keys()) == [app.hub.routespec] -@mark.gen_test -def test_cookie(app): +async def test_cookie(app): db = app.db name = 'patience' user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 201 assert 'pid' in user.orm_spawners[''].state app_user = app.users[name] - cookies = yield app.login_user(name) + cookies = await app.login_user(name) cookie_name = app.hub.cookie_name # cookie jar gives '"cookie-value"', we want 'cookie-value' cookie = cookies[cookie_name][1:-1] - r = yield api_request(app, 'authorizations/cookie', + r = await api_request(app, 'authorizations/cookie', cookie_name, "nothintoseehere", ) assert r.status_code == 404 - r = yield api_request(app, 'authorizations/cookie', + r = await api_request(app, 'authorizations/cookie', cookie_name, quote(cookie, safe=''), ) r.raise_for_status() @@ -1132,7 +1093,7 @@ def test_cookie(app): assert reply['name'] == name # deprecated cookie in body: - r = yield api_request(app, 'authorizations/cookie', + r = await api_request(app, 'authorizations/cookie', cookie_name, data=cookie, ) r.raise_for_status() @@ -1146,27 +1107,25 @@ def normalize_token(token): return token -@mark.gen_test -def test_check_token(app): +async def test_check_token(app): name = 'book' user = add_user(app.db, app=app, name=name) token = user.new_api_token() - r = yield api_request(app, 'authorizations/token', token) + r = await api_request(app, 'authorizations/token', token) r.raise_for_status() user_model = r.json() assert user_model['name'] == name - r = yield api_request(app, 'authorizations/token', 'notauthorized') + r = await api_request(app, 'authorizations/token', 'notauthorized') assert r.status_code == 404 -@mark.gen_test @mark.parametrize("headers, status", [ ({}, 200), ({'Authorization': 'token bad'}, 403), ]) -def test_get_new_token_deprecated(app, headers, status): +async def test_get_new_token_deprecated(app, headers, status): # request a new token - r = yield api_request(app, 'authorizations', 'token', + r = await api_request(app, 'authorizations', 'token', method='post', headers=headers, ) @@ -1175,20 +1134,19 @@ def test_get_new_token_deprecated(app, headers, status): return reply = r.json() assert 'token' in reply - r = yield api_request(app, 'authorizations', 'token', reply['token']) + r = await api_request(app, 'authorizations', 'token', reply['token']) r.raise_for_status() reply = r.json() assert reply['name'] == 'admin' -@mark.gen_test -def test_token_formdata_deprecated(app): +async def test_token_formdata_deprecated(app): """Create a token for a user with formdata and no auth header""" data = { 'username': 'fake', 'password': 'fake', } - r = yield api_request(app, 'authorizations', 'token', + r = await api_request(app, 'authorizations', 'token', method='post', data=json.dumps(data) if data else None, noauth=True, @@ -1196,20 +1154,19 @@ def test_token_formdata_deprecated(app): assert r.status_code == 200 reply = r.json() assert 'token' in reply - r = yield api_request(app, 'authorizations', 'token', reply['token']) + r = await api_request(app, 'authorizations', 'token', reply['token']) r.raise_for_status() reply = r.json() assert reply['name'] == data['username'] -@mark.gen_test @mark.parametrize("as_user, for_user, status", [ ('admin', 'other', 200), ('admin', 'missing', 400), ('user', 'other', 403), ('user', 'user', 200), ]) -def test_token_as_user_deprecated(app, as_user, for_user, status): +async def test_token_as_user_deprecated(app, as_user, for_user, status): # ensure both users exist u = add_user(app.db, app, name=as_user) if for_user != 'missing': @@ -1218,7 +1175,7 @@ def test_token_as_user_deprecated(app, as_user, for_user, status): headers = { 'Authorization': 'token %s' % u.new_api_token(), } - r = yield api_request(app, 'authorizations', 'token', + r = await api_request(app, 'authorizations', 'token', method='post', data=json.dumps(data), headers=headers, @@ -1228,19 +1185,18 @@ def test_token_as_user_deprecated(app, as_user, for_user, status): if status != 200: return assert 'token' in reply - r = yield api_request(app, 'authorizations', 'token', reply['token']) + r = await api_request(app, 'authorizations', 'token', reply['token']) r.raise_for_status() reply = r.json() assert reply['name'] == data['username'] -@mark.gen_test @mark.parametrize("headers, status, note, expires_in", [ ({}, 200, 'test note', None), ({}, 200, '', 100), ({'Authorization': 'token bad'}, 403, '', None), ]) -def test_get_new_token(app, headers, status, note, expires_in): +async def test_get_new_token(app, headers, status, note, expires_in): options = {} if note: options['note'] = note @@ -1251,7 +1207,7 @@ def test_get_new_token(app, headers, status, note, expires_in): else: body = '' # request a new token - r = yield api_request(app, 'users/admin/tokens', + r = await api_request(app, 'users/admin/tokens', method='post', headers=headers, data=body, @@ -1279,28 +1235,27 @@ def test_get_new_token(app, headers, status, note, expires_in): initial.pop('token') # check the validity of the new token - r = yield api_request(app, 'users/admin/tokens', token_id) + r = await api_request(app, 'users/admin/tokens', token_id) r.raise_for_status() reply = r.json() assert normalize_token(reply) == initial # delete the token - r = yield api_request(app, 'users/admin/tokens', token_id, + r = await api_request(app, 'users/admin/tokens', token_id, method='delete') assert r.status_code == 204 # verify deletion - r = yield api_request(app, 'users/admin/tokens', token_id) + r = await api_request(app, 'users/admin/tokens', token_id) assert r.status_code == 404 -@mark.gen_test @mark.parametrize("as_user, for_user, status", [ ('admin', 'other', 200), ('admin', 'missing', 404), ('user', 'other', 403), ('user', 'user', 200), ]) -def test_token_for_user(app, as_user, for_user, status): +async def test_token_for_user(app, as_user, for_user, status): # ensure both users exist u = add_user(app.db, app, name=as_user) if for_user != 'missing': @@ -1309,7 +1264,7 @@ def test_token_for_user(app, as_user, for_user, status): headers = { 'Authorization': 'token %s' % u.new_api_token(), } - r = yield api_request(app, 'users', for_user, 'tokens', + r = await api_request(app, 'users', for_user, 'tokens', method='post', data=json.dumps(data), headers=headers, @@ -1320,7 +1275,7 @@ def test_token_for_user(app, as_user, for_user, status): return assert 'token' in reply token_id = reply['id'] - r = yield api_request(app, 'users', for_user, 'tokens', token_id, + r = await api_request(app, 'users', for_user, 'tokens', token_id, headers=headers, ) r.raise_for_status() @@ -1334,20 +1289,19 @@ def test_token_for_user(app, as_user, for_user, status): # delete the token - r = yield api_request(app, 'users', for_user, 'tokens', token_id, + r = await api_request(app, 'users', for_user, 'tokens', token_id, method='delete', headers=headers, ) assert r.status_code == 204 - r = yield api_request(app, 'users', for_user, 'tokens', token_id, + r = await api_request(app, 'users', for_user, 'tokens', token_id, headers=headers, ) assert r.status_code == 404 -@mark.gen_test -def test_token_authenticator_noauth(app): +async def test_token_authenticator_noauth(app): """Create a token for a user relying on Authenticator.authenticate and no auth header""" name = 'user' data = { @@ -1356,7 +1310,7 @@ def test_token_authenticator_noauth(app): 'password': name, }, } - r = yield api_request(app, 'users', name, 'tokens', + r = await api_request(app, 'users', name, 'tokens', method='post', data=json.dumps(data) if data else None, noauth=True, @@ -1364,27 +1318,26 @@ def test_token_authenticator_noauth(app): assert r.status_code == 200 reply = r.json() assert 'token' in reply - r = yield api_request(app, 'authorizations', 'token', reply['token']) + r = await api_request(app, 'authorizations', 'token', reply['token']) r.raise_for_status() reply = r.json() assert reply['name'] == name -@mark.gen_test @mark.parametrize("as_user, for_user, status", [ ('admin', 'other', 200), ('admin', 'missing', 404), ('user', 'other', 403), ('user', 'user', 200), ]) -def test_token_list(app, as_user, for_user, status): +async def test_token_list(app, as_user, for_user, status): u = add_user(app.db, app, name=as_user) if for_user != 'missing': for_user_obj = add_user(app.db, app, name=for_user) headers = { 'Authorization': 'token %s' % u.new_api_token(), } - r = yield api_request(app, 'users', for_user, 'tokens', + r = await api_request(app, 'users', for_user, 'tokens', headers=headers, ) assert r.status_code == status @@ -1397,7 +1350,7 @@ def test_token_list(app, as_user, for_user, status): assert all(token['user'] == for_user for token in reply['oauth_tokens']) # validate individual token ids for token in reply['api_tokens'] + reply['oauth_tokens']: - r = yield api_request(app, 'users', for_user, 'tokens', token['id'], + r = await api_request(app, 'users', for_user, 'tokens', token['id'], headers=headers, ) r.raise_for_status() @@ -1411,9 +1364,8 @@ def test_token_list(app, as_user, for_user, status): @mark.group -@mark.gen_test -def test_groups_list(app): - r = yield api_request(app, 'groups') +async def test_groups_list(app): + r = await api_request(app, 'groups') r.raise_for_status() reply = r.json() assert reply == [] @@ -1423,7 +1375,7 @@ def test_groups_list(app): app.db.add(group) app.db.commit() - r = yield api_request(app, 'groups') + r = await api_request(app, 'groups') r.raise_for_status() reply = r.json() assert reply == [{ @@ -1434,11 +1386,10 @@ def test_groups_list(app): @mark.group -@mark.gen_test -def test_add_multi_group(app): +async def test_add_multi_group(app): db = app.db names = ['group1', 'group2'] - r = yield api_request(app, 'groups', method='post', + r = await api_request(app, 'groups', method='post', data=json.dumps({'groups': names}), ) assert r.status_code == 201 @@ -1447,24 +1398,23 @@ def test_add_multi_group(app): assert names == r_names # try to create the same groups again - r = yield api_request(app, 'groups', method='post', + r = await api_request(app, 'groups', method='post', data=json.dumps({'groups': names}), ) assert r.status_code == 409 @mark.group -@mark.gen_test -def test_group_get(app): +async def test_group_get(app): group = orm.Group.find(app.db, name='alphaflight') user = add_user(app.db, app=app, name='sasquatch') group.users.append(user) app.db.commit() - r = yield api_request(app, 'groups/runaways') + r = await api_request(app, 'groups/runaways') assert r.status_code == 404 - r = yield api_request(app, 'groups/alphaflight') + r = await api_request(app, 'groups/alphaflight') r.raise_for_status() reply = r.json() assert reply == { @@ -1475,19 +1425,18 @@ def test_group_get(app): @mark.group -@mark.gen_test -def test_group_create_delete(app): +async def test_group_create_delete(app): db = app.db - r = yield api_request(app, 'groups/runaways', method='delete') + r = await api_request(app, 'groups/runaways', method='delete') assert r.status_code == 404 - r = yield api_request(app, 'groups/new', method='post', + r = await api_request(app, 'groups/new', method='post', data=json.dumps({'users': ['doesntexist']}), ) assert r.status_code == 400 assert orm.Group.find(db, name='new') is None - r = yield api_request(app, 'groups/omegaflight', method='post', + r = await api_request(app, 'groups/omegaflight', method='post', data=json.dumps({'users': ['sasquatch']}), ) r.raise_for_status() @@ -1498,30 +1447,29 @@ def test_group_create_delete(app): assert sasquatch in omegaflight.users # create duplicate raises 400 - r = yield api_request(app, 'groups/omegaflight', method='post') + r = await api_request(app, 'groups/omegaflight', method='post') assert r.status_code == 409 - r = yield api_request(app, 'groups/omegaflight', method='delete') + r = await api_request(app, 'groups/omegaflight', method='delete') assert r.status_code == 204 assert omegaflight not in sasquatch.groups assert orm.Group.find(db, name='omegaflight') is None # delete nonexistent gives 404 - r = yield api_request(app, 'groups/omegaflight', method='delete') + r = await api_request(app, 'groups/omegaflight', method='delete') assert r.status_code == 404 @mark.group -@mark.gen_test -def test_group_add_users(app): +async def test_group_add_users(app): db = app.db # must specify users - r = yield api_request(app, 'groups/alphaflight/users', method='post', data='{}') + r = await api_request(app, 'groups/alphaflight/users', method='post', data='{}') assert r.status_code == 400 names = ['aurora', 'guardian', 'northstar', 'sasquatch', 'shaman', 'snowbird'] users = [ find_user(db, name=name) or add_user(db, app=app, name=name) for name in names ] - r = yield api_request(app, 'groups/alphaflight/users', method='post', data=json.dumps({ + r = await api_request(app, 'groups/alphaflight/users', method='post', data=json.dumps({ 'users': names, })) r.raise_for_status() @@ -1535,16 +1483,15 @@ def test_group_add_users(app): @mark.group -@mark.gen_test -def test_group_delete_users(app): +async def test_group_delete_users(app): db = app.db # must specify users - r = yield api_request(app, 'groups/alphaflight/users', method='delete', data='{}') + r = await api_request(app, 'groups/alphaflight/users', method='delete', data='{}') assert r.status_code == 400 names = ['aurora', 'guardian', 'northstar', 'sasquatch', 'shaman', 'snowbird'] users = [ find_user(db, name=name) for name in names ] - r = yield api_request(app, 'groups/alphaflight/users', method='delete', data=json.dumps({ + r = await api_request(app, 'groups/alphaflight/users', method='delete', data=json.dumps({ 'users': names[:2], })) r.raise_for_status() @@ -1564,11 +1511,10 @@ def test_group_delete_users(app): @mark.services -@mark.gen_test -def test_get_services(app, mockservice_url): +async def test_get_services(app, mockservice_url): mockservice = mockservice_url db = app.db - r = yield api_request(app, 'services') + r = await api_request(app, 'services') r.raise_for_status() assert r.status_code == 200 @@ -1585,18 +1531,17 @@ def test_get_services(app, mockservice_url): } } - r = yield api_request(app, 'services', + r = await api_request(app, 'services', headers=auth_header(db, 'user'), ) assert r.status_code == 403 @mark.services -@mark.gen_test -def test_get_service(app, mockservice_url): +async def test_get_service(app, mockservice_url): mockservice = mockservice_url db = app.db - r = yield api_request(app, 'services/%s' % mockservice.name) + r = await api_request(app, 'services/%s' % mockservice.name) r.raise_for_status() assert r.status_code == 200 @@ -1611,27 +1556,26 @@ def test_get_service(app, mockservice_url): 'info': {}, } - r = yield api_request(app, 'services/%s' % mockservice.name, + r = await api_request(app, 'services/%s' % mockservice.name, headers={ 'Authorization': 'token %s' % mockservice.api_token } ) r.raise_for_status() - r = yield api_request(app, 'services/%s' % mockservice.name, + r = await api_request(app, 'services/%s' % mockservice.name, headers=auth_header(db, 'user'), ) assert r.status_code == 403 -@mark.gen_test -def test_root_api(app): +async def test_root_api(app): base_url = app.hub.url url = ujoin(base_url, 'api') kwargs = {} if app.internal_ssl: kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key) kwargs["verify"] = app.internal_ssl_ca - r = yield async_requests.get(url, **kwargs) + r = await async_requests.get(url, **kwargs) r.raise_for_status() expected = { 'version': jupyterhub.__version__ @@ -1639,9 +1583,8 @@ def test_root_api(app): assert r.json() == expected -@mark.gen_test -def test_info(app): - r = yield api_request(app, 'info') +async def test_info(app): + r = await api_request(app, 'info') r.raise_for_status() data = r.json() assert data['version'] == jupyterhub.__version__ @@ -1669,16 +1612,14 @@ def test_info(app): # ----------------- -@mark.gen_test -def test_options(app): - r = yield api_request(app, 'users', method='options') +async def test_options(app): + r = await api_request(app, 'users', method='options') r.raise_for_status() assert 'Access-Control-Allow-Headers' in r.headers -@mark.gen_test -def test_bad_json_body(app): - r = yield api_request(app, 'users', method='post', data='notjson') +async def test_bad_json_body(app): + r = await api_request(app, 'users', method='post', data='notjson') assert r.status_code == 400 @@ -1693,9 +1634,8 @@ def test_shutdown(app): # have to do things a little funky since we are going to stop the loop, # which makes gen_test unhappy. So we run the loop ourselves. - @gen.coroutine - def shutdown(): - r = yield api_request(app, 'shutdown', method='post', + async def shutdown(): + r = await api_request(app, 'shutdown', method='post', data=json.dumps({'servers': True, 'proxy': True,}), ) return r diff --git a/jupyterhub/tests/test_app.py b/jupyterhub/tests/test_app.py index c758489d..f34d3c5a 100644 --- a/jupyterhub/tests/test_app.py +++ b/jupyterhub/tests/test_app.py @@ -63,8 +63,7 @@ def test_generate_config(): assert 'Authenticator.whitelist' in cfg_text -@pytest.mark.gen_test -def test_init_tokens(request): +async def test_init_tokens(request): with TemporaryDirectory() as td: db_file = os.path.join(td, 'jupyterhub.sqlite') tokens = { @@ -77,7 +76,7 @@ def test_init_tokens(request): if ssl_enabled: kwargs['internal_certs_location'] = td app = MockHub(**kwargs) - yield app.initialize([]) + await app.initialize([]) db = app.db for token, username in tokens.items(): api_token = orm.APIToken.find(db, token) @@ -87,7 +86,7 @@ def test_init_tokens(request): # simulate second startup, reloading same tokens: app = MockHub(**kwargs) - yield app.initialize([]) + await app.initialize([]) db = app.db for token, username in tokens.items(): api_token = orm.APIToken.find(db, token) @@ -99,7 +98,7 @@ def test_init_tokens(request): tokens['short'] = 'gman' app = MockHub(**kwargs) with pytest.raises(ValueError): - yield app.initialize([]) + await app.initialize([]) assert orm.User.find(app.db, 'gman') is None @@ -169,8 +168,7 @@ def test_cookie_secret_env(tmpdir, request): assert not os.path.exists(hub.cookie_secret_file) -@pytest.mark.gen_test -def test_load_groups(tmpdir, request): +async def test_load_groups(tmpdir, request): to_load = { 'blue': ['cyclops', 'rogue', 'wolverine'], 'gold': ['storm', 'jean-grey', 'colossus'], @@ -181,8 +179,8 @@ def test_load_groups(tmpdir, request): kwargs['internal_certs_location'] = str(tmpdir) hub = MockHub(**kwargs) hub.init_db() - yield hub.init_users() - yield hub.init_groups() + await hub.init_users() + await hub.init_groups() db = hub.db blue = orm.Group.find(db, name='blue') assert blue is not None @@ -192,16 +190,15 @@ def test_load_groups(tmpdir, request): assert sorted([ u.name for u in gold.users ]) == sorted(to_load['gold']) -@pytest.mark.gen_test -def test_resume_spawners(tmpdir, request): +async def test_resume_spawners(tmpdir, request): if not os.getenv('JUPYTERHUB_TEST_DB_URL'): p = patch.dict(os.environ, { 'JUPYTERHUB_TEST_DB_URL': 'sqlite:///%s' % tmpdir.join('jupyterhub.sqlite'), }) p.start() request.addfinalizer(p.stop) - @gen.coroutine - def new_hub(): + + async def new_hub(): kwargs = {} ssl_enabled = getattr(request.module, "ssl_enabled", False) if ssl_enabled: @@ -209,26 +206,27 @@ def test_resume_spawners(tmpdir, request): app = MockHub(test_clean_db=False, **kwargs) app.config.ConfigurableHTTPProxy.should_start = False app.config.ConfigurableHTTPProxy.auth_token = 'unused' - yield app.initialize([]) + await app.initialize([]) return app - app = yield new_hub() + + app = await new_hub() db = app.db # spawn a user's server name = 'kurt' user = add_user(db, app, name=name) - yield user.spawn() + await user.spawn() proc = user.spawner.proc assert proc is not None # stop the Hub without cleaning up servers app.cleanup_servers = False - yield app.stop() + app.stop() # proc is still running assert proc.poll() is None # resume Hub, should still be running - app = yield new_hub() + app = await new_hub() db = app.db user = app.users[name] assert user.running @@ -236,7 +234,7 @@ def test_resume_spawners(tmpdir, request): # stop the Hub without cleaning up servers app.cleanup_servers = False - yield app.stop() + app.stop() # stop the server while the Hub is down. BAMF! proc.terminate() @@ -244,7 +242,7 @@ def test_resume_spawners(tmpdir, request): assert proc.poll() is not None # resume Hub, should be stopped - app = yield new_hub() + app = await new_hub() db = app.db user = app.users[name] assert not user.running diff --git a/jupyterhub/tests/test_auth.py b/jupyterhub/tests/test_auth.py index e872d6bc..13f4e610 100644 --- a/jupyterhub/tests/test_auth.py +++ b/jupyterhub/tests/test_auth.py @@ -14,47 +14,44 @@ from jupyterhub import auth, crypto, orm from .mocking import MockPAMAuthenticator, MockStructGroup, MockStructPasswd from .test_api import add_user -@pytest.mark.gen_test -def test_pam_auth(): +async def test_pam_auth(): authenticator = MockPAMAuthenticator() - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'match', 'password': 'match', }) assert authorized['name'] == 'match' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'match', 'password': 'nomatch', }) assert authorized is None # Account check is on by default for increased security - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'notallowedmatch', 'password': 'notallowedmatch', }) assert authorized is None -@pytest.mark.gen_test -def test_pam_auth_account_check_disabled(): +async def test_pam_auth_account_check_disabled(): authenticator = MockPAMAuthenticator(check_account=False) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'allowedmatch', 'password': 'allowedmatch', }) assert authorized['name'] == 'allowedmatch' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'notallowedmatch', 'password': 'notallowedmatch', }) assert authorized['name'] == 'notallowedmatch' -@pytest.mark.gen_test -def test_pam_auth_admin_groups(): +async def test_pam_auth_admin_groups(): jh_users = MockStructGroup('jh_users', ['group_admin', 'also_group_admin', 'override_admin', 'non_admin'], 1234) jh_admins = MockStructGroup('jh_admins', ['group_admin'], 5678) wheel = MockStructGroup('wheel', ['also_group_admin'], 9999) @@ -90,7 +87,7 @@ def test_pam_auth_admin_groups(): _getgrnam=getgrnam, _getpwnam=getpwnam, _getgrouplist=getgrouplist): - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'group_admin', 'password': 'group_admin' }) @@ -102,7 +99,7 @@ def test_pam_auth_admin_groups(): _getgrnam=getgrnam, _getpwnam=getpwnam, _getgrouplist=getgrouplist): - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'also_group_admin', 'password': 'also_group_admin' }) @@ -114,7 +111,7 @@ def test_pam_auth_admin_groups(): _getgrnam=getgrnam, _getpwnam=getpwnam, _getgrouplist=getgrouplist): - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'override_admin', 'password': 'override_admin' }) @@ -126,7 +123,7 @@ def test_pam_auth_admin_groups(): _getgrnam=getgrnam, _getpwnam=getpwnam, _getgrouplist=getgrouplist): - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'non_admin', 'password': 'non_admin' }) @@ -134,55 +131,52 @@ def test_pam_auth_admin_groups(): assert authorized['admin'] is False -@pytest.mark.gen_test -def test_pam_auth_whitelist(): +async def test_pam_auth_whitelist(): authenticator = MockPAMAuthenticator(whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) assert authorized['name'] == 'kaylee' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'nomatch', }) assert authorized is None - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'mal', 'password': 'mal', }) assert authorized is None -@pytest.mark.gen_test -def test_pam_auth_group_whitelist(): +async def test_pam_auth_group_whitelist(): def getgrnam(name): return MockStructGroup('grp', ['kaylee']) authenticator = MockPAMAuthenticator(group_whitelist={'group'}) with mock.patch.object(authenticator, '_getgrnam', getgrnam): - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) assert authorized['name'] == 'kaylee' with mock.patch.object(authenticator, '_getgrnam', getgrnam): - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'mal', 'password': 'mal', }) assert authorized is None -@pytest.mark.gen_test -def test_pam_auth_blacklist(): +async def test_pam_auth_blacklist(): # Null case compared to next case authenticator = MockPAMAuthenticator() - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'wash', }) @@ -190,7 +184,7 @@ def test_pam_auth_blacklist(): # Blacklist basics authenticator = MockPAMAuthenticator(blacklist={'wash'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'wash', }) @@ -198,7 +192,7 @@ def test_pam_auth_blacklist(): # User in both white and blacklists: default deny. Make error someday? authenticator = MockPAMAuthenticator(blacklist={'wash'}, whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'wash', }) @@ -206,7 +200,7 @@ def test_pam_auth_blacklist(): # User not in blacklist can log in authenticator = MockPAMAuthenticator(blacklist={'wash'}, whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) @@ -214,7 +208,7 @@ def test_pam_auth_blacklist(): # User in whitelist, blacklist irrelevent authenticator = MockPAMAuthenticator(blacklist={'mal'}, whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'wash', }) @@ -222,7 +216,7 @@ def test_pam_auth_blacklist(): # User in neither list authenticator = MockPAMAuthenticator(blacklist={'mal'}, whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'simon', 'password': 'simon', }) @@ -230,34 +224,31 @@ def test_pam_auth_blacklist(): # blacklist == {} authenticator = MockPAMAuthenticator(blacklist=set(), whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) assert authorized['name'] == 'kaylee' -@pytest.mark.gen_test -def test_pam_auth_no_such_group(): +async def test_pam_auth_no_such_group(): authenticator = MockPAMAuthenticator(group_whitelist={'nosuchcrazygroup'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) assert authorized is None -@pytest.mark.gen_test -def test_wont_add_system_user(): +async def test_wont_add_system_user(): user = orm.User(name='lioness4321') authenticator = auth.PAMAuthenticator(whitelist={'mal'}) authenticator.create_system_users = False with pytest.raises(KeyError): - yield authenticator.add_user(user) + await authenticator.add_user(user) -@pytest.mark.gen_test -def test_cant_add_system_user(): +async def test_cant_add_system_user(): user = orm.User(name='lioness4321') authenticator = auth.PAMAuthenticator(whitelist={'mal'}) authenticator.add_user_cmd = ['jupyterhub-fake-command'] @@ -279,12 +270,11 @@ def test_cant_add_system_user(): with mock.patch.object(auth, 'Popen', DummyPopen): with pytest.raises(RuntimeError) as exc: - yield authenticator.add_user(user) + await authenticator.add_user(user) assert str(exc.value) == 'Failed to create system user lioness4321: dummy error' -@pytest.mark.gen_test -def test_add_system_user(): +async def test_add_system_user(): user = orm.User(name='lioness4321') authenticator = auth.PAMAuthenticator(whitelist={'mal'}) authenticator.create_system_users = True @@ -300,17 +290,16 @@ def test_add_system_user(): return with mock.patch.object(auth, 'Popen', DummyPopen): - yield authenticator.add_user(user) + await authenticator.add_user(user) assert record['cmd'] == ['echo', '/home/lioness4321', 'lioness4321'] -@pytest.mark.gen_test -def test_delete_user(): +async def test_delete_user(): user = orm.User(name='zoe') a = MockPAMAuthenticator(whitelist={'mal'}) assert 'zoe' not in a.whitelist - yield a.add_user(user) + await a.add_user(user) assert 'zoe' in a.whitelist a.delete_user(user) assert 'zoe' not in a.whitelist @@ -330,47 +319,43 @@ def test_handlers(app): assert handlers[0][0] == '/login' -@pytest.mark.gen_test -def test_auth_state(app, auth_state_enabled): +async def test_auth_state(app, auth_state_enabled): """auth_state enabled and available""" name = 'kiwi' user = add_user(app.db, app, name=name) assert user.encrypted_auth_state is None - cookies = yield app.login_user(name) - auth_state = yield user.get_auth_state() + cookies = await app.login_user(name) + auth_state = await user.get_auth_state() assert auth_state == app.authenticator.auth_state -@pytest.mark.gen_test -def test_auth_admin_non_admin(app): +async def test_auth_admin_non_admin(app): """admin should be passed through for non-admin users""" name = 'kiwi' user = add_user(app.db, app, name=name, admin=False) assert user.admin is False - cookies = yield app.login_user(name) + cookies = await app.login_user(name) assert user.admin is False -@pytest.mark.gen_test -def test_auth_admin_is_admin(app): +async def test_auth_admin_is_admin(app): """admin should be passed through for admin users""" # Admin user defined in MockPAMAuthenticator. name = 'admin' user = add_user(app.db, app, name=name, admin=False) assert user.admin is False - cookies = yield app.login_user(name) + cookies = await app.login_user(name) assert user.admin is True -@pytest.mark.gen_test -def test_auth_admin_retained_if_unset(app): +async def test_auth_admin_retained_if_unset(app): """admin should be unchanged if authenticator doesn't return admin value""" name = 'kiwi' # Add user as admin. user = add_user(app.db, app, name=name, admin=True) assert user.admin is True # User should remain unchanged. - cookies = yield app.login_user(name) + cookies = await app.login_user(name) assert user.admin is True @@ -384,56 +369,53 @@ def auth_state_unavailable(auth_state_enabled): yield -@pytest.mark.gen_test -def test_auth_state_disabled(app, auth_state_unavailable): +async def test_auth_state_disabled(app, auth_state_unavailable): name = 'driebus' user = add_user(app.db, app, name=name) assert user.encrypted_auth_state is None with pytest.raises(HTTPError): - cookies = yield app.login_user(name) - auth_state = yield user.get_auth_state() + cookies = await app.login_user(name) + auth_state = await user.get_auth_state() assert auth_state is None -@pytest.mark.gen_test -def test_normalize_names(): +async def test_normalize_names(): a = MockPAMAuthenticator() - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'ZOE', 'password': 'ZOE', }) assert authorized['name'] == 'zoe' - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'Glenn', 'password': 'Glenn', }) assert authorized['name'] == 'glenn' - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'hExi', 'password': 'hExi', }) assert authorized['name'] == 'hexi' - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'Test', 'password': 'Test', }) assert authorized['name'] == 'test' -@pytest.mark.gen_test -def test_username_map(): +async def test_username_map(): a = MockPAMAuthenticator(username_map={'wash': 'alpha'}) - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'WASH', 'password': 'WASH', }) assert authorized['name'] == 'alpha' - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'Inara', 'password': 'Inara', }) diff --git a/jupyterhub/tests/test_crypto.py b/jupyterhub/tests/test_crypto.py index e133745f..db7799a0 100644 --- a/jupyterhub/tests/test_crypto.py +++ b/jupyterhub/tests/test_crypto.py @@ -11,6 +11,7 @@ keys = [('%i' % i).encode('ascii') * 32 for i in range(3)] hex_keys = [ b2a_hex(key).decode('ascii') for key in keys ] b64_keys = [ b2a_base64(key).decode('ascii').strip() for key in keys ] + @pytest.mark.parametrize("key_env, keys", [ (hex_keys[0], [keys[0]]), (';'.join([b64_keys[0], hex_keys[1]]), keys[:2]), @@ -52,30 +53,27 @@ def crypt_keeper(): ck.keys = save_keys -@pytest.mark.gen_test -def test_roundtrip(crypt_keeper): +async def test_roundtrip(crypt_keeper): data = {'key': 'value'} - encrypted = yield encrypt(data) - decrypted = yield decrypt(encrypted) + encrypted = await encrypt(data) + decrypted = await decrypt(encrypted) assert decrypted == data -@pytest.mark.gen_test -def test_missing_crypto(crypt_keeper): +async def test_missing_crypto(crypt_keeper): with patch.object(crypto, 'cryptography', None): with pytest.raises(crypto.CryptographyUnavailable): - yield encrypt({}) + await encrypt({}) with pytest.raises(crypto.CryptographyUnavailable): - yield decrypt(b'whatever') + await decrypt(b'whatever') -@pytest.mark.gen_test -def test_missing_keys(crypt_keeper): +async def test_missing_keys(crypt_keeper): crypt_keeper.keys = [] with pytest.raises(crypto.NoEncryptionKeys): - yield encrypt({}) + await encrypt({}) with pytest.raises(crypto.NoEncryptionKeys): - yield decrypt(b'whatever') + await decrypt(b'whatever') diff --git a/jupyterhub/tests/test_db.py b/jupyterhub/tests/test_db.py index cc166aad..18a8e952 100644 --- a/jupyterhub/tests/test_db.py +++ b/jupyterhub/tests/test_db.py @@ -40,8 +40,7 @@ def generate_old_db(env_dir, hub_version, db_url): '0.8.1', ], ) -@pytest.mark.gen_test -def test_upgrade(tmpdir, hub_version): +async def test_upgrade(tmpdir, hub_version): db_url = os.getenv('JUPYTERHUB_TEST_DB_URL') if db_url: db_url += '_upgrade_' + hub_version.replace('.', '') @@ -69,7 +68,7 @@ def test_upgrade(tmpdir, hub_version): assert len(sqlite_files) == 1 upgradeapp = UpgradeDB(config=cfg) - yield upgradeapp.initialize([]) + upgradeapp.initialize([]) upgradeapp.start() # check that backup was created: diff --git a/jupyterhub/tests/test_dummyauth.py b/jupyterhub/tests/test_dummyauth.py index 8e419ef2..84fb480e 100644 --- a/jupyterhub/tests/test_dummyauth.py +++ b/jupyterhub/tests/test_dummyauth.py @@ -7,48 +7,45 @@ import pytest from jupyterhub.auth import DummyAuthenticator -@pytest.mark.gen_test -def test_dummy_auth_without_global_password(): +async def test_dummy_auth_without_global_password(): authenticator = DummyAuthenticator() - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'test_user', 'password': 'test_pass', }) assert authorized['name'] == 'test_user' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'test_user', 'password': '', }) assert authorized['name'] == 'test_user' -@pytest.mark.gen_test -def test_dummy_auth_without_username(): +async def test_dummy_auth_without_username(): authenticator = DummyAuthenticator() - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': '', 'password': 'test_pass', }) assert authorized is None -@pytest.mark.gen_test -def test_dummy_auth_with_global_password(): +async def test_dummy_auth_with_global_password(): authenticator = DummyAuthenticator() authenticator.password = "test_password" - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'test_user', 'password': 'test_password', }) assert authorized['name'] == 'test_user' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'test_user', 'password': 'qwerty', }) assert authorized is None - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'some_other_user', 'password': 'test_password', }) diff --git a/jupyterhub/tests/test_internal_ssl_connections.py b/jupyterhub/tests/test_internal_ssl_connections.py index 42b306aa..be5ddcca 100644 --- a/jupyterhub/tests/test_internal_ssl_connections.py +++ b/jupyterhub/tests/test_internal_ssl_connections.py @@ -39,39 +39,36 @@ def wait_for_spawner(spawner, timeout=10): yield wait() -@pytest.mark.gen_test -def test_connection_hub_wrong_certs(app): +async def test_connection_hub_wrong_certs(app): """Connecting to the internal hub url fails without correct certs""" with pytest.raises(SSLError): kwargs = {'verify': False} - r = yield async_requests.get(app.hub.url, **kwargs) + r = await async_requests.get(app.hub.url, **kwargs) r.raise_for_status() -@pytest.mark.gen_test -def test_connection_proxy_api_wrong_certs(app): +async def test_connection_proxy_api_wrong_certs(app): """Connecting to the proxy api fails without correct certs""" with pytest.raises(SSLError): kwargs = {'verify': False} - r = yield async_requests.get(app.proxy.api_url, **kwargs) + r = await async_requests.get(app.proxy.api_url, **kwargs) r.raise_for_status() -@pytest.mark.gen_test -def test_connection_notebook_wrong_certs(app): +async def test_connection_notebook_wrong_certs(app): """Connecting to a notebook fails without correct certs""" with mock.patch.dict( app.config.LocalProcessSpawner, {'cmd': [sys.executable, '-m', 'jupyterhub.tests.mocksu']} ): user = add_user(app.db, app, name='foo') - yield user.spawn() - yield wait_for_spawner(user.spawner) + await user.spawn() + await wait_for_spawner(user.spawner) spawner = user.spawner - status = yield spawner.poll() + status = await spawner.poll() assert status is None with pytest.raises(SSLError): kwargs = {'verify': False} - r = yield async_requests.get(spawner.server.url, **kwargs) + r = await async_requests.get(spawner.server.url, **kwargs) r.raise_for_status() diff --git a/jupyterhub/tests/test_named_servers.py b/jupyterhub/tests/test_named_servers.py index 8f808d33..40bca0cf 100644 --- a/jupyterhub/tests/test_named_servers.py +++ b/jupyterhub/tests/test_named_servers.py @@ -17,16 +17,15 @@ def named_servers(app): yield -@pytest.mark.gen_test -def test_default_server(app, named_servers): +async def test_default_server(app, named_servers): """Test the default /users/:user/server handler when named servers are enabled""" username = 'rosie' user = add_user(app.db, app, name=username) - r = yield api_request(app, 'users', username, 'server', method='post') + r = await api_request(app, 'users', username, 'server', method='post') assert r.status_code == 201 assert r.text == '' - r = yield api_request(app, 'users', username) + r = await api_request(app, 'users', username) r.raise_for_status() user_model = normalize_user(r.json()) @@ -51,11 +50,11 @@ def test_default_server(app, named_servers): # now stop the server - r = yield api_request(app, 'users', username, 'server', method='delete') + r = await api_request(app, 'users', username, 'server', method='delete') assert r.status_code == 204 assert r.text == '' - r = yield api_request(app, 'users', username) + r = await api_request(app, 'users', username) r.raise_for_status() user_model = normalize_user(r.json()) @@ -66,20 +65,19 @@ def test_default_server(app, named_servers): }) -@pytest.mark.gen_test -def test_create_named_server(app, named_servers): +async def test_create_named_server(app, named_servers): username = 'walnut' user = add_user(app.db, app, name=username) # assert user.allow_named_servers == True - cookies = yield app.login_user(username) + cookies = await app.login_user(username) servername = 'trevor' - r = yield api_request(app, 'users', username, 'servers', servername, method='post') + r = await api_request(app, 'users', username, 'servers', servername, method='post') r.raise_for_status() assert r.status_code == 201 assert r.text == '' url = url_path_join(public_url(app, user), servername, 'env') - r = yield async_requests.get(url, cookies=cookies) + r = await async_requests.get(url, cookies=cookies) r.raise_for_status() assert r.url == url env = r.json() @@ -87,7 +85,7 @@ def test_create_named_server(app, named_servers): assert prefix == user.spawners[servername].server.base_url assert prefix.endswith('/user/%s/%s/' % (username, servername)) - r = yield api_request(app, 'users', username) + r = await api_request(app, 'users', username) r.raise_for_status() user_model = normalize_user(r.json()) @@ -111,22 +109,21 @@ def test_create_named_server(app, named_servers): }) -@pytest.mark.gen_test -def test_delete_named_server(app, named_servers): +async def test_delete_named_server(app, named_servers): username = 'donaar' user = add_user(app.db, app, name=username) assert user.allow_named_servers cookies = app.login_user(username) servername = 'splugoth' - r = yield api_request(app, 'users', username, 'servers', servername, method='post') + r = await api_request(app, 'users', username, 'servers', servername, method='post') r.raise_for_status() assert r.status_code == 201 - r = yield api_request(app, 'users', username, 'servers', servername, method='delete') + r = await api_request(app, 'users', username, 'servers', servername, method='delete') r.raise_for_status() assert r.status_code == 204 - r = yield api_request(app, 'users', username) + r = await api_request(app, 'users', username) r.raise_for_status() user_model = normalize_user(r.json()) @@ -140,7 +137,7 @@ def test_delete_named_server(app, named_servers): # low-level record still exists assert servername in user.orm_spawners - r = yield api_request( + r = await api_request( app, 'users', username, 'servers', servername, method='delete', data=json.dumps({'remove': True}), @@ -151,11 +148,10 @@ def test_delete_named_server(app, named_servers): assert servername not in user.orm_spawners -@pytest.mark.gen_test -def test_named_server_disabled(app): +async def test_named_server_disabled(app): username = 'user' servername = 'okay' - r = yield api_request(app, 'users', username, 'servers', servername, method='post') + r = await api_request(app, 'users', username, 'servers', servername, method='post') assert r.status_code == 400 - r = yield api_request(app, 'users', username, 'servers', servername, method='delete') + r = await api_request(app, 'users', username, 'servers', servername, method='delete') assert r.status_code == 400 diff --git a/jupyterhub/tests/test_orm.py b/jupyterhub/tests/test_orm.py index cb442f42..5c8eff93 100644 --- a/jupyterhub/tests/test_orm.py +++ b/jupyterhub/tests/test_orm.py @@ -205,8 +205,7 @@ def test_token_find(db): assert found is None -@pytest.mark.gen_test -def test_spawn_fails(db): +async def test_spawn_fails(db): orm_user = orm.User(name='aeofel') db.add(orm_user) db.commit() @@ -223,7 +222,7 @@ def test_spawn_fails(db): }) with pytest.raises(RuntimeError) as exc: - yield user.spawn() + await user.spawn() assert user.spawners[''].server is None assert not user.running @@ -246,8 +245,7 @@ def test_groups(db): assert group.users == [] -@pytest.mark.gen_test -def test_auth_state(db): +async def test_auth_state(db): orm_user = orm.User(name='eve') db.add(orm_user) db.commit() @@ -262,51 +260,51 @@ def test_auth_state(db): state = {'key': 'value'} ck.keys = [] with pytest.raises(crypto.EncryptionUnavailable): - yield user.save_auth_state(state) + await user.save_auth_state(state) assert user.encrypted_auth_state is None # saving/loading None doesn't require keys - yield user.save_auth_state(None) - current = yield user.get_auth_state() + await user.save_auth_state(None) + current = await user.get_auth_state() assert current is None first_key = os.urandom(32) second_key = os.urandom(32) ck.keys = [first_key] - yield user.save_auth_state(state) + await user.save_auth_state(state) assert user.encrypted_auth_state is not None - decrypted_state = yield user.get_auth_state() + decrypted_state = await user.get_auth_state() assert decrypted_state == state # can't read auth_state without keys ck.keys = [] - auth_state = yield user.get_auth_state() + auth_state = await user.get_auth_state() assert auth_state is None # key rotation works db.rollback() ck.keys = [second_key, first_key] - decrypted_state = yield user.get_auth_state() + decrypted_state = await user.get_auth_state() assert decrypted_state == state new_state = {'key': 'newvalue'} - yield user.save_auth_state(new_state) + await user.save_auth_state(new_state) db.commit() ck.keys = [first_key] db.rollback() # can't read anymore with new-key after encrypting with second-key - decrypted_state = yield user.get_auth_state() + decrypted_state = await user.get_auth_state() assert decrypted_state is None - yield user.save_auth_state(new_state) - decrypted_state = yield user.get_auth_state() + await user.save_auth_state(new_state) + decrypted_state = await user.get_auth_state() assert decrypted_state == new_state ck.keys = [] db.rollback() - decrypted_state = yield user.get_auth_state() + decrypted_state = await user.get_auth_state() assert decrypted_state is None diff --git a/jupyterhub/tests/test_pages.py b/jupyterhub/tests/test_pages.py index 6c65a63e..cd810f6c 100644 --- a/jupyterhub/tests/test_pages.py +++ b/jupyterhub/tests/test_pages.py @@ -30,91 +30,81 @@ def get_page(path, app, hub=True, **kw): return async_requests.get(ujoin(base_url, path), **kw) -@pytest.mark.gen_test -def test_root_no_auth(app): +async def test_root_no_auth(app): url = ujoin(public_host(app), app.hub.base_url) - r = yield async_requests.get(url) + r = await async_requests.get(url) r.raise_for_status() assert r.url == ujoin(url, 'login') -@pytest.mark.gen_test -def test_root_auth(app): - cookies = yield app.login_user('river') - r = yield async_requests.get(public_url(app), cookies=cookies) +async def test_root_auth(app): + cookies = await app.login_user('river') + r = await async_requests.get(public_url(app), cookies=cookies) r.raise_for_status() assert r.url.startswith(public_url(app, app.users['river'])) -@pytest.mark.gen_test -def test_root_redirect(app): +async def test_root_redirect(app): name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) next_url = ujoin(app.base_url, 'user/other/test.ipynb') url = '/?' + urlencode({'next': next_url}) - r = yield get_page(url, app, cookies=cookies) + r = await get_page(url, app, cookies=cookies) r.raise_for_status() path = urlparse(r.url).path assert path == ujoin(app.base_url, 'user/%s/test.ipynb' % name) -@pytest.mark.gen_test -def test_root_default_url_noauth(app): +async def test_root_default_url_noauth(app): with mock.patch.dict(app.tornado_settings, {'default_url': '/foo/bar'}): - r = yield get_page('/', app, allow_redirects=False) + r = await get_page('/', app, allow_redirects=False) r.raise_for_status() url = r.headers.get('Location', '') path = urlparse(url).path assert path == '/foo/bar' -@pytest.mark.gen_test -def test_root_default_url_auth(app): +async def test_root_default_url_auth(app): name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) with mock.patch.dict(app.tornado_settings, {'default_url': '/foo/bar'}): - r = yield get_page('/', app, cookies=cookies, allow_redirects=False) + r = await get_page('/', app, cookies=cookies, allow_redirects=False) r.raise_for_status() url = r.headers.get('Location', '') path = urlparse(url).path assert path == '/foo/bar' -@pytest.mark.gen_test -def test_home_no_auth(app): - r = yield get_page('home', app, allow_redirects=False) +async def test_home_no_auth(app): + r = await get_page('home', app, allow_redirects=False) r.raise_for_status() assert r.status_code == 302 assert '/hub/login' in r.headers['Location'] -@pytest.mark.gen_test -def test_home_auth(app): - cookies = yield app.login_user('river') - r = yield get_page('home', app, cookies=cookies) +async def test_home_auth(app): + cookies = await app.login_user('river') + r = await get_page('home', app, cookies=cookies) r.raise_for_status() assert r.url.endswith('home') -@pytest.mark.gen_test -def test_admin_no_auth(app): - r = yield get_page('admin', app) +async def test_admin_no_auth(app): + r = await get_page('admin', app) assert r.status_code == 403 -@pytest.mark.gen_test -def test_admin_not_admin(app): - cookies = yield app.login_user('wash') - r = yield get_page('admin', app, cookies=cookies) +async def test_admin_not_admin(app): + cookies = await app.login_user('wash') + r = await get_page('admin', app, cookies=cookies) assert r.status_code == 403 -@pytest.mark.gen_test -def test_admin(app): - cookies = yield app.login_user('admin') - r = yield get_page('admin', app, cookies=cookies, allow_redirects=False) +async def test_admin(app): + cookies = await app.login_user('admin') + r = await get_page('admin', app, cookies=cookies, allow_redirects=False) r.raise_for_status() assert r.url.endswith('/admin') @@ -125,127 +115,120 @@ def test_admin(app): 'admin', 'name', ]) -@pytest.mark.gen_test -def test_admin_sort(app, sort): - cookies = yield app.login_user('admin') - r = yield get_page('admin?sort=%s' % sort, app, cookies=cookies) +async def test_admin_sort(app, sort): + cookies = await app.login_user('admin') + r = await get_page('admin?sort=%s' % sort, app, cookies=cookies) r.raise_for_status() assert r.status_code == 200 -@pytest.mark.gen_test -def test_spawn_redirect(app): +async def test_spawn_redirect(app): name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) u = app.users[orm.User.find(app.db, name)] - status = yield u.spawner.poll() + status = await u.spawner.poll() assert status is not None # test spawn page when no server is running - r = yield get_page('spawn', app, cookies=cookies) + r = await get_page('spawn', app, cookies=cookies) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, 'user/%s/' % name) # should have started server - status = yield u.spawner.poll() + status = await u.spawner.poll() assert status is None # test spawn page when server is already running (just redirect) - r = yield get_page('spawn', app, cookies=cookies) + r = await get_page('spawn', app, cookies=cookies) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/' % name) # stop server to ensure /user/name is handled by the Hub - r = yield api_request(app, 'users', name, 'server', method='delete', cookies=cookies) + r = await api_request(app, 'users', name, 'server', method='delete', cookies=cookies) r.raise_for_status() # test handing of trailing slash on `/user/name` - r = yield get_page('user/' + name, app, hub=False, cookies=cookies) + r = await get_page('user/' + name, app, hub=False, cookies=cookies) r.raise_for_status() path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/' % name) -@pytest.mark.gen_test -def test_spawn_handler_access(app): +async def test_spawn_handler_access(app): name = 'winston' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) u = app.users[orm.User.find(app.db, name)] - status = yield u.spawner.poll() + status = await u.spawner.poll() assert status is not None # spawn server via browser link with ?arg=value - r = yield get_page('spawn', app, cookies=cookies, params={'arg': 'value'}) + r = await get_page('spawn', app, cookies=cookies, params={'arg': 'value'}) r.raise_for_status() # verify that request params got passed down # implemented in MockSpawner - r = yield async_requests.get(ujoin(public_url(app, u), 'env')) + r = await async_requests.get(ujoin(public_url(app, u), 'env')) env = r.json() assert 'HANDLER_ARGS' in env assert env['HANDLER_ARGS'] == 'arg=value' # stop server - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() -@pytest.mark.gen_test -def test_spawn_admin_access(app, admin_access): +async def test_spawn_admin_access(app, admin_access): """GET /user/:name as admin with admin-access spawns user's server""" - cookies = yield app.login_user('admin') + cookies = await app.login_user('admin') name = 'mariel' user = add_user(app.db, app=app, name=name) app.db.commit() - r = yield get_page('user/' + name, app, cookies=cookies) + r = await get_page('user/' + name, app, cookies=cookies) r.raise_for_status() assert (r.url.split('?')[0] + '/').startswith(public_url(app, user)) - r = yield get_page('user/{}/env'.format(name), app, hub=False, cookies=cookies) + r = await get_page('user/{}/env'.format(name), app, hub=False, cookies=cookies) r.raise_for_status() env = r.json() assert env['JUPYTERHUB_USER'] == name -@pytest.mark.gen_test -def test_spawn_page(app): +async def test_spawn_page(app): with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): - cookies = yield app.login_user('jones') - r = yield get_page('spawn', app, cookies=cookies) + cookies = await app.login_user('jones') + r = await get_page('spawn', app, cookies=cookies) assert r.url.endswith('/spawn') assert FormSpawner.options_form in r.text - r = yield get_page('spawn?next=foo', app, cookies=cookies) + r = await get_page('spawn?next=foo', app, cookies=cookies) assert r.url.endswith('/spawn?next=foo') assert FormSpawner.options_form in r.text -@pytest.mark.gen_test -def test_spawn_page_admin(app, admin_access): +async def test_spawn_page_admin(app, admin_access): with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): - cookies = yield app.login_user('admin') + cookies = await app.login_user('admin') u = add_user(app.db, app=app, name='melanie') - r = yield get_page('spawn/' + u.name, app, cookies=cookies) + r = await get_page('spawn/' + u.name, app, cookies=cookies) assert r.url.endswith('/spawn/' + u.name) assert FormSpawner.options_form in r.text assert "Spawning server for {}".format(u.name) in r.text -@pytest.mark.gen_test -def test_spawn_form(app): +async def test_spawn_form(app): with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): base_url = ujoin(public_host(app), app.hub.base_url) - cookies = yield app.login_user('jones') + cookies = await app.login_user('jones') orm_u = orm.User.find(app.db, 'jones') u = app.users[orm_u] - yield u.stop() + await u.stop() next_url = ujoin(app.base_url, 'user/jones/tree') - r = yield async_requests.post( + r = await async_requests.post( url_concat(ujoin(base_url, 'spawn'), {'next': next_url}), cookies=cookies, data={'bounds': ['-1', '1'], 'energy': '511keV'}, @@ -259,15 +242,14 @@ def test_spawn_form(app): } -@pytest.mark.gen_test -def test_spawn_form_admin_access(app, admin_access): +async def test_spawn_form_admin_access(app, admin_access): with mock.patch.dict(app.tornado_settings, {'spawner_class': FormSpawner}): base_url = ujoin(public_host(app), app.hub.base_url) - cookies = yield app.login_user('admin') + cookies = await app.login_user('admin') u = add_user(app.db, app=app, name='martha') next_url = ujoin(app.base_url, 'user', u.name, 'tree') - r = yield async_requests.post( + r = await async_requests.post( url_concat(ujoin(base_url, 'spawn', u.name), {'next': next_url}), cookies=cookies, data={'bounds': ['-3', '3'], 'energy': '938MeV'}, @@ -282,16 +264,15 @@ def test_spawn_form_admin_access(app, admin_access): } -@pytest.mark.gen_test -def test_spawn_form_with_file(app): +async def test_spawn_form_with_file(app): with mock.patch.dict(app.tornado_settings, {'spawner_class': FormSpawner}): base_url = ujoin(public_host(app), app.hub.base_url) - cookies = yield app.login_user('jones') + cookies = await app.login_user('jones') orm_u = orm.User.find(app.db, 'jones') u = app.users[orm_u] - yield u.stop() + await u.stop() - r = yield async_requests.post(ujoin(base_url, 'spawn'), + r = await async_requests.post(ujoin(base_url, 'spawn'), cookies=cookies, data={ 'bounds': ['-1', '1'], @@ -310,12 +291,11 @@ def test_spawn_form_with_file(app): } -@pytest.mark.gen_test -def test_user_redirect(app): +async def test_user_redirect(app): name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) - r = yield get_page('/user-redirect/tree/top/', app) + r = await get_page('/user-redirect/tree/top/', app) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path @@ -325,32 +305,31 @@ def test_user_redirect(app): 'next': ujoin(app.hub.base_url, '/user-redirect/tree/top/') }) - r = yield get_page('/user-redirect/notebooks/test.ipynb', app, cookies=cookies) + r = await get_page('/user-redirect/notebooks/test.ipynb', app, cookies=cookies) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/notebooks/test.ipynb' % name) -@pytest.mark.gen_test -def test_user_redirect_deprecated(app): +async def test_user_redirect_deprecated(app): """redirecting from /user/someonelse/ URLs (deprecated)""" name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) - r = yield get_page('/user/baduser', app, cookies=cookies, hub=False) + r = await get_page('/user/baduser', app, cookies=cookies, hub=False) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/' % name) - r = yield get_page('/user/baduser/test.ipynb', app, cookies=cookies, hub=False) + r = await get_page('/user/baduser/test.ipynb', app, cookies=cookies, hub=False) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/test.ipynb' % name) - r = yield get_page('/user/baduser/test.ipynb', app, hub=False) + r = await get_page('/user/baduser/test.ipynb', app, hub=False) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path @@ -361,11 +340,10 @@ def test_user_redirect_deprecated(app): }) -@pytest.mark.gen_test -def test_login_fail(app): +async def test_login_fail(app): name = 'wash' base_url = public_url(app) - r = yield async_requests.post(base_url + 'hub/login', + r = await async_requests.post(base_url + 'hub/login', data={ 'username': name, 'password': 'wrong', @@ -375,8 +353,7 @@ def test_login_fail(app): assert not r.cookies -@pytest.mark.gen_test -def test_login_strip(app): +async def test_login_strip(app): """Test that login form doesn't strip whitespace from passwords""" form_data = { 'username': 'spiff', @@ -389,7 +366,7 @@ def test_login_strip(app): called_with.append(data) with mock.patch.object(app.authenticator, 'authenticate', mock_authenticate): - yield async_requests.post(base_url + 'hub/login', + await async_requests.post(base_url + 'hub/login', data=form_data, allow_redirects=False, ) @@ -415,9 +392,8 @@ def test_login_strip(app): (False, '//other.domain', ''), ] ) -@pytest.mark.gen_test -def test_login_redirect(app, running, next_url, location): - cookies = yield app.login_user('river') +async def test_login_redirect(app, running, next_url, location): + cookies = await app.login_user('river') user = app.users['river'] if location: location = ujoin(app.base_url, location) @@ -433,18 +409,17 @@ def test_login_redirect(app, running, next_url, location): if running and not user.active: # ensure running - yield user.spawn() + await user.spawn() elif user.active and not running: # ensure not running - yield user.stop() - r = yield get_page(url, app, cookies=cookies, allow_redirects=False) + await user.stop() + r = await get_page(url, app, cookies=cookies, allow_redirects=False) r.raise_for_status() assert r.status_code == 302 assert location == r.headers['Location'] -@pytest.mark.gen_test -def test_auto_login(app, request): +async def test_auto_login(app, request): class DummyLoginHandler(BaseHandler): def get(self): self.write('ok!') @@ -453,7 +428,7 @@ def test_auto_login(app, request): (ujoin(app.hub.base_url, 'dummy'), DummyLoginHandler), ]) # no auto_login: end up at /hub/login - r = yield async_requests.get(base_url) + r = await async_requests.get(base_url) assert r.url == public_url(app, path='hub/login') # enable auto_login: redirect from /hub/login to /hub/dummy authenticator = Authenticator(auto_login=True) @@ -462,28 +437,28 @@ def test_auto_login(app, request): with mock.patch.dict(app.tornado_settings, { 'authenticator': authenticator, }): - r = yield async_requests.get(base_url) + r = await async_requests.get(base_url) assert r.url == public_url(app, path='hub/dummy') -@pytest.mark.gen_test -def test_auto_login_logout(app): + +async def test_auto_login_logout(app): name = 'burnham' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) with mock.patch.dict(app.tornado_settings, { 'authenticator': Authenticator(auto_login=True), }): - r = yield async_requests.get(public_host(app) + app.tornado_settings['logout_url'], cookies=cookies) + r = await async_requests.get(public_host(app) + app.tornado_settings['logout_url'], cookies=cookies) r.raise_for_status() logout_url = public_host(app) + app.tornado_settings['logout_url'] assert r.url == logout_url assert r.cookies == {} -@pytest.mark.gen_test -def test_logout(app): + +async def test_logout(app): name = 'wash' - cookies = yield app.login_user(name) - r = yield async_requests.get(public_host(app) + app.tornado_settings['logout_url'], cookies=cookies) + cookies = await app.login_user(name) + r = await async_requests.get(public_host(app) + app.tornado_settings['logout_url'], cookies=cookies) r.raise_for_status() login_url = public_host(app) + app.tornado_settings['login_url'] assert r.url == login_url @@ -491,7 +466,6 @@ def test_logout(app): @pytest.mark.parametrize('shutdown_on_logout', [True, False]) -@pytest.mark.gen_test async def test_shutdown_on_logout(app, shutdown_on_logout): name = 'shutitdown' cookies = await app.login_user(name) @@ -535,50 +509,46 @@ async def test_shutdown_on_logout(app, shutdown_on_logout): assert spawner.ready == (not shutdown_on_logout) -@pytest.mark.gen_test -def test_login_no_whitelist_adds_user(app): +async def test_login_no_whitelist_adds_user(app): auth = app.authenticator mock_add_user = mock.Mock() with mock.patch.object(auth, 'add_user', mock_add_user): - cookies = yield app.login_user('jubal') + cookies = await app.login_user('jubal') user = app.users['jubal'] assert mock_add_user.mock_calls == [mock.call(user)] -@pytest.mark.gen_test -def test_static_files(app): +async def test_static_files(app): base_url = ujoin(public_host(app), app.hub.base_url) - r = yield async_requests.get(ujoin(base_url, 'logo')) + r = await async_requests.get(ujoin(base_url, 'logo')) r.raise_for_status() assert r.headers['content-type'] == 'image/png' - r = yield async_requests.get(ujoin(base_url, 'static', 'images', 'jupyter.png')) + r = await async_requests.get(ujoin(base_url, 'static', 'images', 'jupyter.png')) r.raise_for_status() assert r.headers['content-type'] == 'image/png' - r = yield async_requests.get(ujoin(base_url, 'static', 'css', 'style.min.css')) + r = await async_requests.get(ujoin(base_url, 'static', 'css', 'style.min.css')) r.raise_for_status() assert r.headers['content-type'] == 'text/css' -@pytest.mark.gen_test -def test_token_auth(app): - cookies = yield app.login_user('token') - r = yield get_page('token', app, cookies=cookies) +async def test_token_auth(app): + cookies = await app.login_user('token') + r = await get_page('token', app, cookies=cookies) r.raise_for_status() assert r.status_code == 200 -@pytest.mark.gen_test -def test_oauth_token_page(app): +async def test_oauth_token_page(app): name = 'token' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) user = app.users[orm.User.find(app.db, name)] client = orm.OAuthClient(identifier='token') app.db.add(client) oauth_token = orm.OAuthAccessToken(client=client, user=user, grant_type=orm.GrantType.authorization_code) app.db.add(oauth_token) app.db.commit() - r = yield get_page('token', app, cookies=cookies) + r = await get_page('token', app, cookies=cookies) r.raise_for_status() assert r.status_code == 200 @@ -587,14 +557,11 @@ def test_oauth_token_page(app): 503, 404, ]) - -@pytest.mark.gen_test -def test_proxy_error(app, error_status): - r = yield get_page('/error/%i' % error_status, app) +async def test_proxy_error(app, error_status): + r = await get_page('/error/%i' % error_status, app) assert r.status_code == 200 -@pytest.mark.gen_test @pytest.mark.parametrize( "announcements", [ @@ -604,7 +571,7 @@ def test_proxy_error(app, error_status): "login,logout", ] ) -def test_announcements(app, announcements): +async def test_announcements(app, announcements): """Test announcements on various pages""" # Default announcement - same on all pages ann01 = "ANNOUNCE01" @@ -620,26 +587,26 @@ def test_announcements(app, announcements): else: assert ann01 in text - cookies = yield app.login_user("jones") + cookies = await app.login_user("jones") with mock.patch.dict( app.tornado_settings, {"template_vars": template_vars, "spawner_class": FormSpawner}, ): - r = yield get_page("login", app) + r = await get_page("login", app) r.raise_for_status() assert_announcement("login", r.text) - r = yield get_page("spawn", app, cookies=cookies) + r = await get_page("spawn", app, cookies=cookies) r.raise_for_status() assert_announcement("spawn", r.text) - r = yield get_page("home", app, cookies=cookies) # hub/home + r = await get_page("home", app, cookies=cookies) # hub/home r.raise_for_status() assert_announcement("home", r.text) # need auto_login=True to get logout page auto_login = app.authenticator.auto_login app.authenticator.auto_login = True try: - r = yield get_page("logout", app, cookies=cookies) + r = await get_page("logout", app, cookies=cookies) finally: app.authenticator.auto_login = auto_login r.raise_for_status() @@ -654,18 +621,16 @@ def test_announcements(app, announcements): "redirect_uri=ok&client_id=nosuchthing", ] ) -@pytest.mark.gen_test -def test_bad_oauth_get(app, params): - cookies = yield app.login_user("authorizer") - r = yield get_page("hub/api/oauth2/authorize?" + params, app, hub=False, cookies=cookies) +async def test_bad_oauth_get(app, params): + cookies = await app.login_user("authorizer") + r = await get_page("hub/api/oauth2/authorize?" + params, app, hub=False, cookies=cookies) assert r.status_code == 400 -@pytest.mark.gen_test -def test_token_page(app): +async def test_token_page(app): name = "cake" - cookies = yield app.login_user(name) - r = yield get_page("token", app, cookies=cookies) + cookies = await app.login_user(name) + r = await get_page("token", app, cookies=cookies) r.raise_for_status() assert urlparse(r.url).path.endswith('/hub/token') def extract_body(r): @@ -684,7 +649,7 @@ def test_token_page(app): token = user.new_api_token(expires_in=60, note="my-test-token") app.db.commit() - r = yield get_page("token", app, cookies=cookies) + r = await get_page("token", app, cookies=cookies) r.raise_for_status() body = extract_body(r) assert "API Tokens" in body, body @@ -695,10 +660,10 @@ def test_token_page(app): # spawn the user to trigger oauth, etc. # request an oauth token user.spawner.cmd = [sys.executable, '-m', 'jupyterhub.singleuser'] - r = yield get_page("spawn", app, cookies=cookies) + r = await get_page("spawn", app, cookies=cookies) r.raise_for_status() - r = yield get_page("token", app, cookies=cookies) + r = await get_page("token", app, cookies=cookies) r.raise_for_status() body = extract_body(r) assert "API Tokens" in body, body @@ -706,31 +671,27 @@ def test_token_page(app): assert "Authorized Applications" in body, body -@pytest.mark.gen_test -def test_server_not_running_api_request(app): - cookies = yield app.login_user("bees") - r = yield get_page("user/bees/api/status", app, hub=False, cookies=cookies) +async def test_server_not_running_api_request(app): + cookies = await app.login_user("bees") + r = await get_page("user/bees/api/status", app, hub=False, cookies=cookies) assert r.status_code == 404 assert r.headers["content-type"] == "application/json" assert r.json() == {"message": "bees is not running"} -@pytest.mark.gen_test -def test_metrics_no_auth(app): - r = yield get_page("metrics", app) +async def test_metrics_no_auth(app): + r = await get_page("metrics", app) assert r.status_code == 403 -@pytest.mark.gen_test -def test_metrics_auth(app): - cookies = yield app.login_user('river') +async def test_metrics_auth(app): + cookies = await app.login_user('river') metrics_url = ujoin(public_host(app), app.hub.base_url, 'metrics') - r = yield get_page("metrics", app, cookies=cookies) + r = await get_page("metrics", app, cookies=cookies) assert r.status_code == 200 assert r.url == metrics_url -@pytest.mark.gen_test -def test_health_check_request(app): - r = yield get_page('health', app) +async def test_health_check_request(app): + r = await get_page('health', app) assert r.status_code == 200 diff --git a/jupyterhub/tests/test_proxy.py b/jupyterhub/tests/test_proxy.py index 1e4816b7..dd493a86 100644 --- a/jupyterhub/tests/test_proxy.py +++ b/jupyterhub/tests/test_proxy.py @@ -16,6 +16,7 @@ from .mocking import MockHub from .test_api import api_request, add_user from ..utils import wait_for_http_server, url_path_join as ujoin + @pytest.fixture def disable_check_routes(app): # disable periodic check_routes while we are testing @@ -25,9 +26,8 @@ def disable_check_routes(app): finally: app.last_activity_callback.start() -@pytest.mark.gen_test -def test_external_proxy(request): +async def test_external_proxy(request): auth_token = 'secret!' proxy_ip = '127.0.0.1' proxy_port = 54321 @@ -71,23 +71,23 @@ def test_external_proxy(request): def wait_for_proxy(): return wait_for_http_server('http://%s:%i' % (proxy_ip, proxy_port)) - yield wait_for_proxy() + await wait_for_proxy() - yield app.initialize([]) - yield app.start() + await app.initialize([]) + await app.start() assert app.proxy.proxy_process is None # test if api service has a root route '/' - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() assert list(routes.keys()) == [app.hub.routespec] - + # add user to the db and start a single user server name = 'river' add_user(app.db, app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - - routes = yield app.proxy.get_all_routes() + + routes = await app.proxy.get_all_routes() # sets the desired path result user_path = ujoin(app.base_url, 'user/river') + '/' print(app.base_url, user_path) @@ -101,18 +101,18 @@ def test_external_proxy(request): proxy.terminate() proxy.wait(timeout=10) proxy = Popen(cmd, env=env) - yield wait_for_proxy() + await wait_for_proxy() - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() assert list(routes.keys()) == [] # poke the server to update the proxy - r = yield api_request(app, 'proxy', method='post') + r = await api_request(app, 'proxy', method='post') r.raise_for_status() # check that the routes are correct - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() assert sorted(routes.keys()) == [app.hub.routespec, user_spec] # teardown the proxy, and start a new one with different auth and port @@ -131,11 +131,11 @@ def test_external_proxy(request): if app.subdomain_host: cmd.append('--host-routing') proxy = Popen(cmd, env=env) - yield wait_for_proxy() + await wait_for_proxy() # tell the hub where the new proxy is new_api_url = 'http://{}:{}'.format(proxy_ip, proxy_port) - r = yield api_request(app, 'proxy', method='patch', data=json.dumps({ + r = await api_request(app, 'proxy', method='patch', data=json.dumps({ 'api_url': new_api_url, 'auth_token': new_auth_token, })) @@ -145,11 +145,10 @@ def test_external_proxy(request): assert app.proxy.auth_token == new_auth_token # check that the routes are correct - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() assert sorted(routes.keys()) == [app.hub.routespec, user_spec] -@pytest.mark.gen_test @pytest.mark.parametrize("username", [ 'zoe', '50fia', @@ -157,27 +156,27 @@ def test_external_proxy(request): '~TestJH', 'has@', ]) -def test_check_routes(app, username, disable_check_routes): +async def test_check_routes(app, username, disable_check_routes): proxy = app.proxy test_user = add_user(app.db, app, name=username) - r = yield api_request(app, 'users/%s/server' % username, method='post') + r = await api_request(app, 'users/%s/server' % username, method='post') r.raise_for_status() # check a valid route exists for user - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() before = sorted(routes) assert test_user.proxy_spec in before # check if a route is removed when user deleted - yield app.proxy.check_routes(app.users, app._service_map) - yield proxy.delete_user(test_user) - routes = yield app.proxy.get_all_routes() + await app.proxy.check_routes(app.users, app._service_map) + await proxy.delete_user(test_user) + routes = await app.proxy.get_all_routes() during = sorted(routes) assert test_user.proxy_spec not in during # check if a route exists for user - yield app.proxy.check_routes(app.users, app._service_map) - routes = yield app.proxy.get_all_routes() + await app.proxy.check_routes(app.users, app._service_map) + routes = await app.proxy.get_all_routes() after = sorted(routes) assert test_user.proxy_spec in after @@ -185,7 +184,6 @@ def test_check_routes(app, username, disable_check_routes): assert before == after -@pytest.mark.gen_test @pytest.mark.parametrize("routespec", [ '/has%20space/foo/', '/missing-trailing/slash', @@ -194,11 +192,11 @@ def test_check_routes(app, username, disable_check_routes): 'host.name/path/', 'other.host/path/no/slash', ]) -def test_add_get_delete(app, routespec, disable_check_routes): +async def test_add_get_delete(app, routespec, disable_check_routes): arg = routespec if not routespec.endswith('/'): routespec = routespec + '/' - + # host-routes when not host-routing raises an error # and vice versa expect_value_error = bool(app.subdomain_host) ^ (not routespec.startswith('/')) @@ -213,26 +211,25 @@ def test_add_get_delete(app, routespec, disable_check_routes): proxy = app.proxy target = 'https://localhost:1234' with context(): - yield proxy.add_route(arg, target, {}) - routes = yield proxy.get_all_routes() + await proxy.add_route(arg, target, {}) + routes = await proxy.get_all_routes() if not expect_value_error: assert routespec in routes.keys() with context(): - route = yield proxy.get_route(arg) + route = await proxy.get_route(arg) assert route == { 'target': target, 'routespec': routespec, 'data': route.get('data'), } with context(): - yield proxy.delete_route(arg) + await proxy.delete_route(arg) with context(): - route = yield proxy.get_route(arg) + route = await proxy.get_route(arg) assert route is None -@pytest.mark.gen_test @pytest.mark.parametrize("test_data", [None, 'notjson', json.dumps([])]) -def test_proxy_patch_bad_request_data(app, test_data): - r = yield api_request(app, 'proxy', method='patch', data=test_data) +async def test_proxy_patch_bad_request_data(app, test_data): + r = await api_request(app, 'proxy', method='patch', data=test_data) assert r.status_code == 400 diff --git a/jupyterhub/tests/test_services.py b/jupyterhub/tests/test_services.py index 78069078..a318b566 100644 --- a/jupyterhub/tests/test_services.py +++ b/jupyterhub/tests/test_services.py @@ -1,5 +1,6 @@ """Tests for services""" +import asyncio from binascii import hexlify from contextlib import contextmanager import os @@ -8,13 +9,14 @@ import sys from threading import Event import time +from async_generator import asynccontextmanager import pytest import requests from tornado import gen from tornado.ioloop import IOLoop from .mocking import public_url -from ..utils import url_path_join, wait_for_http_server, random_port +from ..utils import url_path_join, wait_for_http_server, random_port, maybe_future from .utils import async_requests mockservice_path = os.path.dirname(os.path.abspath(__file__)) @@ -22,8 +24,8 @@ mockservice_py = os.path.join(mockservice_path, 'mockservice.py') mockservice_cmd = [sys.executable, mockservice_py] -@contextmanager -def external_service(app, name='mockservice'): +@asynccontextmanager +async def external_service(app, name='mockservice'): env = { 'JUPYTERHUB_API_TOKEN': hexlify(os.urandom(5)), 'JUPYTERHUB_SERVICE_NAME': name, @@ -31,17 +33,14 @@ def external_service(app, name='mockservice'): 'JUPYTERHUB_SERVICE_URL': 'http://127.0.0.1:%i' % random_port(), } proc = Popen(mockservice_cmd, env=env) - IOLoop().run_sync( - lambda: wait_for_http_server(env['JUPYTERHUB_SERVICE_URL']) - ) try: + await wait_for_http_server(env['JUPYTERHUB_SERVICE_URL']) yield env finally: proc.terminate() -@pytest.mark.gen_test -def test_managed_service(mockservice): +async def test_managed_service(mockservice): service = mockservice proc = service.proc assert isinstance(proc.pid, object) @@ -58,19 +57,18 @@ def test_managed_service(mockservice): if service.proc is not proc: break else: - yield gen.sleep(0.2) - + await asyncio.sleep(0.2) + assert service.proc.pid != first_pid assert service.proc.poll() is None -@pytest.mark.gen_test -def test_proxy_service(app, mockservice_url): +async def test_proxy_service(app, mockservice_url): service = mockservice_url name = service.name - yield app.proxy.get_all_routes() + await app.proxy.get_all_routes() url = public_url(app, service) + '/foo' - r = yield async_requests.get(url, allow_redirects=False) + r = await async_requests.get(url, allow_redirects=False) path = '/services/{}/foo'.format(name) r.raise_for_status() @@ -78,23 +76,22 @@ def test_proxy_service(app, mockservice_url): assert r.text.endswith(path) -@pytest.mark.gen_test -def test_external_service(app): +async def test_external_service(app): name = 'external' - with external_service(app, name=name) as env: + async with external_service(app, name=name) as env: app.services = [{ 'name': name, 'admin': True, 'url': env['JUPYTERHUB_SERVICE_URL'], 'api_token': env['JUPYTERHUB_API_TOKEN'], }] - yield app.init_services() - yield app.init_api_tokens() - yield app.proxy.add_all_services(app._service_map) + await maybe_future(app.init_services()) + await app.init_api_tokens() + await app.proxy.add_all_services(app._service_map) service = app._service_map[name] url = public_url(app, service) + '/api/users' - r = yield async_requests.get(url, allow_redirects=False) + r = await async_requests.get(url, allow_redirects=False) r.raise_for_status() assert r.status_code == 200 resp = r.json() diff --git a/jupyterhub/tests/test_services_auth.py b/jupyterhub/tests/test_services_auth.py index 64326fa2..30fbec0d 100644 --- a/jupyterhub/tests/test_services_auth.py +++ b/jupyterhub/tests/test_services_auth.py @@ -227,11 +227,10 @@ def test_hub_authenticated(request): assert r.status_code == 403 -@pytest.mark.gen_test -def test_hubauth_cookie(app, mockservice_url): +async def test_hubauth_cookie(app, mockservice_url): """Test HubAuthenticated service with user cookies""" - cookies = yield app.login_user('badger') - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/', cookies=cookies) + cookies = await app.login_user('badger') + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/', cookies=cookies) r.raise_for_status() print(r.text) reply = r.json() @@ -242,15 +241,14 @@ def test_hubauth_cookie(app, mockservice_url): } -@pytest.mark.gen_test -def test_hubauth_token(app, mockservice_url): +async def test_hubauth_token(app, mockservice_url): """Test HubAuthenticated service with user API tokens""" u = add_user(app.db, name='river') token = u.new_api_token() app.db.commit() # token in Authorization header - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/', + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/', headers={ 'Authorization': 'token %s' % token, }) @@ -262,7 +260,7 @@ def test_hubauth_token(app, mockservice_url): } # token in ?token parameter - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=%s' % token) + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=%s' % token) r.raise_for_status() reply = r.json() sub_reply = { key: reply.get(key, 'missing') for key in ['name', 'admin']} @@ -271,7 +269,7 @@ def test_hubauth_token(app, mockservice_url): 'admin': False, } - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=no-such-token', + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=no-such-token', allow_redirects=False, ) assert r.status_code == 302 @@ -281,17 +279,16 @@ def test_hubauth_token(app, mockservice_url): assert path.endswith('/hub/login') -@pytest.mark.gen_test -def test_hubauth_service_token(app, mockservice_url): +async def test_hubauth_service_token(app, mockservice_url): """Test HubAuthenticated service with service API tokens""" - + token = hexlify(os.urandom(5)).decode('utf8') name = 'test-api-service' app.service_tokens[token] = name - yield app.init_api_tokens() + await app.init_api_tokens() # token in Authorization header - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/', + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/', headers={ 'Authorization': 'token %s' % token, }) @@ -305,7 +302,7 @@ def test_hubauth_service_token(app, mockservice_url): assert not r.cookies # token in ?token parameter - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=%s' % token) + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=%s' % token) r.raise_for_status() reply = r.json() assert reply == { @@ -314,7 +311,7 @@ def test_hubauth_service_token(app, mockservice_url): 'admin': False, } - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=no-such-token', + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=no-such-token', allow_redirects=False, ) assert r.status_code == 302 @@ -324,16 +321,15 @@ def test_hubauth_service_token(app, mockservice_url): assert path.endswith('/hub/login') -@pytest.mark.gen_test -def test_oauth_service(app, mockservice_url): +async def test_oauth_service(app, mockservice_url): service = mockservice_url url = url_path_join(public_url(app, mockservice_url) + 'owhoami/?arg=x') # first request is only going to login and get us to the oauth form page s = AsyncSession() name = 'link' - s.cookies = yield app.login_user(name) + s.cookies = await app.login_user(name) - r = yield s.get(url) + r = await s.get(url) r.raise_for_status() # we should be looking at the oauth confirmation page assert urlparse(r.url).path == app.base_url + 'hub/api/oauth2/authorize' @@ -341,7 +337,7 @@ def test_oauth_service(app, mockservice_url): assert set(r.history[0].cookies.keys()) == {'service-%s-oauth-state' % service.name} # submit the oauth form to complete authorization - r = yield s.post(r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}) + r = await s.post(r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}) r.raise_for_status() assert r.url == url # verify oauth cookie is set @@ -350,7 +346,7 @@ def test_oauth_service(app, mockservice_url): assert 'service-%s-oauth-state' % service.name not in set(s.cookies.keys()) # second request should be authenticated, which means no redirects - r = yield s.get(url, allow_redirects=False) + r = await s.get(url, allow_redirects=False) r.raise_for_status() assert r.status_code == 200 reply = r.json() @@ -363,7 +359,7 @@ def test_oauth_service(app, mockservice_url): # token-authenticated request to HubOAuth token = app.users[name].new_api_token() # token in ?token parameter - r = yield async_requests.get(url_concat(url, {'token': token})) + r = await async_requests.get(url_concat(url, {'token': token})) r.raise_for_status() reply = r.json() assert reply['name'] == name @@ -371,7 +367,7 @@ def test_oauth_service(app, mockservice_url): # verify that ?token= requests set a cookie assert len(r.cookies) != 0 # ensure cookie works in future requests - r = yield async_requests.get( + r = await async_requests.get( url, cookies=r.cookies, allow_redirects=False, @@ -382,17 +378,16 @@ def test_oauth_service(app, mockservice_url): assert reply['name'] == name -@pytest.mark.gen_test -def test_oauth_cookie_collision(app, mockservice_url): +async def test_oauth_cookie_collision(app, mockservice_url): service = mockservice_url url = url_path_join(public_url(app, mockservice_url), 'owhoami/') print(url) s = AsyncSession() name = 'mypha' - s.cookies = yield app.login_user(name) + s.cookies = await app.login_user(name) state_cookie_name = 'service-%s-oauth-state' % service.name service_cookie_name = 'service-%s' % service.name - oauth_1 = yield s.get(url) + oauth_1 = await s.get(url) print(oauth_1.headers) print(oauth_1.cookies, oauth_1.url, url) assert state_cookie_name in s.cookies @@ -402,7 +397,7 @@ def test_oauth_cookie_collision(app, mockservice_url): state_1 = s.cookies[state_cookie_name] # start second oauth login before finishing the first - oauth_2 = yield s.get(url) + oauth_2 = await s.get(url) state_cookies = [ c for c in s.cookies.keys() if c.startswith(state_cookie_name) ] assert len(state_cookies) == 2 # get the random-suffix cookie name @@ -412,7 +407,7 @@ def test_oauth_cookie_collision(app, mockservice_url): # finish oauth 2 # submit the oauth form to complete authorization - r = yield s.post( + r = await s.post( oauth_2.url, data={'scopes': ['identify']}, headers={'Referer': oauth_2.url}, @@ -426,7 +421,7 @@ def test_oauth_cookie_collision(app, mockservice_url): service_cookie_2 = s.cookies[service_cookie_name] # finish oauth 1 - r = yield s.post( + r = await s.post( oauth_1.url, data={'scopes': ['identify']}, headers={'Referer': oauth_1.url}, @@ -445,8 +440,7 @@ def test_oauth_cookie_collision(app, mockservice_url): assert state_cookies == [] -@pytest.mark.gen_test -def test_oauth_logout(app, mockservice_url): +async def test_oauth_logout(app, mockservice_url): """Verify that logout via the Hub triggers logout for oauth services 1. clears session id cookie @@ -471,18 +465,18 @@ def test_oauth_logout(app, mockservice_url): # ensure we start empty assert auth_tokens() == [] - s.cookies = yield app.login_user(name) + s.cookies = await app.login_user(name) assert 'jupyterhub-session-id' in s.cookies - r = yield s.get(url) + r = await s.get(url) r.raise_for_status() assert urlparse(r.url).path.endswith('oauth2/authorize') # submit the oauth form to complete authorization - r = yield s.post(r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}) + r = await s.post(r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}) r.raise_for_status() assert r.url == url # second request should be authenticated - r = yield s.get(url, allow_redirects=False) + r = await s.get(url, allow_redirects=False) r.raise_for_status() assert r.status_code == 200 reply = r.json() @@ -501,13 +495,13 @@ def test_oauth_logout(app, mockservice_url): assert len(auth_tokens()) == 1 # hit hub logout URL - r = yield s.get(public_url(app, path='hub/logout')) + r = await s.get(public_url(app, path='hub/logout')) r.raise_for_status() # verify that all cookies other than the service cookie are cleared assert list(s.cookies.keys()) == [service_cookie_name] # verify that clearing session id invalidates service cookie # i.e. redirect back to login page - r = yield s.get(url) + r = await s.get(url) r.raise_for_status() assert r.url.split('?')[0] == public_url(app, path='hub/login') @@ -524,7 +518,7 @@ def test_oauth_logout(app, mockservice_url): # check that we got the old session id back assert session_id == s.cookies['jupyterhub-session-id'] - r = yield s.get(url, allow_redirects=False) + r = await s.get(url, allow_redirects=False) r.raise_for_status() assert r.status_code == 200 reply = r.json() diff --git a/jupyterhub/tests/test_singleuser.py b/jupyterhub/tests/test_singleuser.py index c9ab77d6..f098cbf7 100644 --- a/jupyterhub/tests/test_singleuser.py +++ b/jupyterhub/tests/test_singleuser.py @@ -13,42 +13,41 @@ from ..utils import url_path_join from .utils import async_requests, AsyncSession -@pytest.mark.gen_test -def test_singleuser_auth(app): +async def test_singleuser_auth(app): # use StubSingleUserSpawner to launch a single-user app in a thread app.spawner_class = StubSingleUserSpawner app.tornado_settings['spawner_class'] = StubSingleUserSpawner # login, start the server - cookies = yield app.login_user('nandy') + cookies = await app.login_user('nandy') user = app.users['nandy'] if not user.running: - yield user.spawn() + await user.spawn() url = public_url(app, user) # no cookies, redirects to login page - r = yield async_requests.get(url) + r = await async_requests.get(url) r.raise_for_status() assert '/hub/login' in r.url # with cookies, login successful - r = yield async_requests.get(url, cookies=cookies) + r = await async_requests.get(url, cookies=cookies) r.raise_for_status() assert urlparse(r.url).path.rstrip('/').endswith('/user/nandy/tree') assert r.status_code == 200 # logout - r = yield async_requests.get(url_path_join(url, 'logout'), cookies=cookies) + r = await async_requests.get(url_path_join(url, 'logout'), cookies=cookies) assert len(r.cookies) == 0 # accessing another user's server hits the oauth confirmation page - cookies = yield app.login_user('burgess') + cookies = await app.login_user('burgess') s = AsyncSession() s.cookies = cookies - r = yield s.get(url) + r = await s.get(url) assert urlparse(r.url).path.endswith('/oauth2/authorize') # submit the oauth form to complete authorization - r = yield s.post( + r = await s.post( r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}, @@ -59,28 +58,27 @@ def test_singleuser_auth(app): assert 'burgess' in r.text -@pytest.mark.gen_test -def test_disable_user_config(app): +async def test_disable_user_config(app): # use StubSingleUserSpawner to launch a single-user app in a thread app.spawner_class = StubSingleUserSpawner app.tornado_settings['spawner_class'] = StubSingleUserSpawner # login, start the server - cookies = yield app.login_user('nandy') + cookies = await app.login_user('nandy') user = app.users['nandy'] # stop spawner, if running: if user.running: print("stopping") - yield user.stop() + await user.stop() # start with new config: user.spawner.debug = True user.spawner.disable_user_config = True - yield user.spawn() - yield app.proxy.add_user(user) - + await user.spawn() + await app.proxy.add_user(user) + url = public_url(app, user) - + # with cookies, login successful - r = yield async_requests.get(url, cookies=cookies) + r = await async_requests.get(url, cookies=cookies) r.raise_for_status() assert r.url.rstrip('/').endswith('/user/nandy/tree') assert r.status_code == 200 @@ -90,6 +88,7 @@ def test_help_output(): out = check_output([sys.executable, '-m', 'jupyterhub.singleuser', '--help-all']).decode('utf8', 'replace') assert 'JupyterHub' in out + def test_version(): out = check_output([sys.executable, '-m', 'jupyterhub.singleuser', '--version']).decode('utf8', 'replace') assert jupyterhub.__version__ in out diff --git a/jupyterhub/tests/test_spawner.py b/jupyterhub/tests/test_spawner.py index b6709acf..18b100ce 100644 --- a/jupyterhub/tests/test_spawner.py +++ b/jupyterhub/tests/test_spawner.py @@ -60,10 +60,9 @@ def new_spawner(db, **kwargs): return user._new_spawner('', spawner_class=LocalProcessSpawner, **kwargs) -@pytest.mark.gen_test -def test_spawner(db, request): +async def test_spawner(db, request): spawner = new_spawner(db) - ip, port = yield spawner.start() + ip, port = await spawner.start() assert ip == '127.0.0.1' assert isinstance(port, int) assert port > 0 @@ -72,115 +71,110 @@ def test_spawner(db, request): # wait for the process to get to the while True: loop time.sleep(1) - status = yield spawner.poll() + status = await spawner.poll() assert status is None - yield spawner.stop() - status = yield spawner.poll() + await spawner.stop() + status = await spawner.poll() assert status == 1 -@gen.coroutine -def wait_for_spawner(spawner, timeout=10): +async def wait_for_spawner(spawner, timeout=10): """Wait for an http server to show up - + polling at shorter intervals for early termination """ deadline = time.monotonic() + timeout def wait(): return spawner.server.wait_up(timeout=1, http=True) while time.monotonic() < deadline: - status = yield spawner.poll() + status = await spawner.poll() assert status is None try: - yield wait() + await wait() except TimeoutError: continue else: break - yield wait() + await wait() -@pytest.mark.gen_test -def test_single_user_spawner(app, request): +async def test_single_user_spawner(app, request): user = next(iter(app.users.values()), None) spawner = user.spawner spawner.cmd = ['jupyterhub-singleuser'] - yield user.spawn() + await user.spawn() assert spawner.server.ip == '127.0.0.1' assert spawner.server.port > 0 - yield wait_for_spawner(spawner) - status = yield spawner.poll() + await wait_for_spawner(spawner) + status = await spawner.poll() assert status is None - yield spawner.stop() - status = yield spawner.poll() + await spawner.stop() + status = await spawner.poll() assert status == 0 -@pytest.mark.gen_test -def test_stop_spawner_sigint_fails(db): +async def test_stop_spawner_sigint_fails(db): spawner = new_spawner(db, cmd=[sys.executable, '-c', _uninterruptible]) - yield spawner.start() + await spawner.start() # wait for the process to get to the while True: loop - yield gen.sleep(1) + await gen.sleep(1) - status = yield spawner.poll() + status = await spawner.poll() assert status is None - - yield spawner.stop() - status = yield spawner.poll() + + await spawner.stop() + status = await spawner.poll() assert status == -signal.SIGTERM -@pytest.mark.gen_test -def test_stop_spawner_stop_now(db): +async def test_stop_spawner_stop_now(db): spawner = new_spawner(db) - yield spawner.start() + await spawner.start() # wait for the process to get to the while True: loop - yield gen.sleep(1) + await gen.sleep(1) - status = yield spawner.poll() + status = await spawner.poll() assert status is None - - yield spawner.stop(now=True) - status = yield spawner.poll() + + await spawner.stop(now=True) + status = await spawner.poll() assert status == -signal.SIGTERM -@pytest.mark.gen_test -def test_spawner_poll(db): +async def test_spawner_poll(db): first_spawner = new_spawner(db) user = first_spawner.user - yield first_spawner.start() + await first_spawner.start() proc = first_spawner.proc - status = yield first_spawner.poll() + status = await first_spawner.poll() assert status is None if user.state is None: user.state = {} first_spawner.orm_spawner.state = first_spawner.get_state() assert 'pid' in first_spawner.orm_spawner.state - + # create a new Spawner, loading from state of previous spawner = new_spawner(db, user=first_spawner.user) spawner.start_polling() - + # wait for the process to get to the while True: loop - yield gen.sleep(1) - status = yield spawner.poll() + await gen.sleep(1) + status = await spawner.poll() assert status is None - + # kill the process proc.terminate() for i in range(10): if proc.poll() is None: - yield gen.sleep(1) + await gen.sleep(1) else: break assert proc.poll() is not None - yield gen.sleep(2) - status = yield spawner.poll() + await gen.sleep(2) + status = await spawner.poll() assert status is not None @@ -213,8 +207,7 @@ def test_string_formatting(db): assert s.format_string(s.default_url) == '/base/%s' % name -@pytest.mark.gen_test -def test_popen_kwargs(db): +async def test_popen_kwargs(db): mock_proc = mock.Mock(spec=Popen) def mock_popen(*args, **kwargs): mock_proc.args = args @@ -224,14 +217,13 @@ def test_popen_kwargs(db): s = new_spawner(db, popen_kwargs={'shell': True}, cmd='jupyterhub-singleuser') with mock.patch.object(spawnermod, 'Popen', mock_popen): - yield s.start() + await s.start() assert mock_proc.kwargs['shell'] == True assert mock_proc.args[0][:1] == (['jupyterhub-singleuser']) -@pytest.mark.gen_test -def test_shell_cmd(db, tmpdir, request): +async def test_shell_cmd(db, tmpdir, request): f = tmpdir.join('bashrc') f.write('export TESTVAR=foo\n') s = new_spawner(db, @@ -243,17 +235,17 @@ def test_shell_cmd(db, tmpdir, request): db.commit() s.server = Server.from_orm(server) db.commit() - (ip, port) = yield s.start() + (ip, port) = await s.start() request.addfinalizer(s.stop) s.server.ip = ip s.server.port = port db.commit() - yield wait_for_spawner(s) - r = yield async_requests.get('http://%s:%i/env' % (ip, port)) + await wait_for_spawner(s) + r = await async_requests.get('http://%s:%i/env' % (ip, port)) r.raise_for_status() env = r.json() assert env['TESTVAR'] == 'foo' - yield s.stop() + await s.stop() def test_inherit_overwrite(): @@ -277,8 +269,7 @@ def test_inherit_ok(): pass -@pytest.mark.gen_test -def test_spawner_reuse_api_token(db, app): +async def test_spawner_reuse_api_token(db, app): # setup: user with no tokens, whose spawner has set the .will_resume flag user = add_user(app.db, app, name='snoopy') spawner = user.spawner @@ -286,26 +277,25 @@ def test_spawner_reuse_api_token(db, app): # will_resume triggers reuse of tokens spawner.will_resume = True # first start: gets a new API token - yield user.spawn() + await user.spawn() api_token = spawner.api_token found = orm.APIToken.find(app.db, api_token) assert found assert found.user.name == user.name assert user.api_tokens == [found] - yield user.stop() + await user.stop() # stop now deletes unused spawners. # put back the mock spawner! user.spawners[''] = spawner # second start: should reuse the token - yield user.spawn() + await user.spawn() # verify re-use of API token assert spawner.api_token == api_token # verify that a new token was not created assert user.api_tokens == [found] -@pytest.mark.gen_test -def test_spawner_insert_api_token(app): +async def test_spawner_insert_api_token(app): """Token provided by spawner is not in the db Insert token into db as a user-provided token. @@ -322,17 +312,16 @@ def test_spawner_insert_api_token(app): # The spawner's provided API token would already be in the db # unless there is a bug somewhere else (in the Spawner), # but handle it anyway. - yield user.spawn() + await user.spawn() assert spawner.api_token == api_token found = orm.APIToken.find(app.db, api_token) assert found assert found.user.name == user.name assert user.api_tokens == [found] - yield user.stop() + await user.stop() -@pytest.mark.gen_test -def test_spawner_bad_api_token(app): +async def test_spawner_bad_api_token(app): """Tokens are revoked when a Spawner gets another user's token""" # we need two users for this one user = add_user(app.db, app, name='antimone') @@ -349,13 +338,12 @@ def test_spawner_bad_api_token(app): # starting a user's server with another user's token # should revoke it with pytest.raises(ValueError): - yield user.spawn() + await user.spawn() assert orm.APIToken.find(app.db, other_token) is None assert other_user.api_tokens == [] -@pytest.mark.gen_test -def test_spawner_delete_server(app): +async def test_spawner_delete_server(app): """Test deleting spawner.server This can occur during app startup if their server has been deleted. @@ -393,22 +381,21 @@ def test_spawner_delete_server(app): "has%40x", ] ) -@pytest.mark.gen_test -def test_spawner_routing(app, name): +async def test_spawner_routing(app, name): """Test routing of names with special characters""" db = app.db with mock.patch.dict(app.config.LocalProcessSpawner, {'cmd': [sys.executable, '-m', 'jupyterhub.tests.mocksu']}): user = add_user(app.db, app, name=name) - yield user.spawn() - yield wait_for_spawner(user.spawner) - yield app.proxy.add_user(user) + await user.spawn() + await wait_for_spawner(user.spawner) + await app.proxy.add_user(user) kwargs = {'allow_redirects': False} if app.internal_ssl: kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key) kwargs["verify"] = app.internal_ssl_ca url = url_path_join(public_url(app, user), "test/url") - r = yield async_requests.get(url, **kwargs) + r = await async_requests.get(url, **kwargs) r.raise_for_status() assert r.url == url assert r.text == urlparse(url).path - yield user.stop() + await user.stop() diff --git a/jupyterhub/tests/test_utils.py b/jupyterhub/tests/test_utils.py index 418f4795..7301b420 100644 --- a/jupyterhub/tests/test_utils.py +++ b/jupyterhub/tests/test_utils.py @@ -26,7 +26,6 @@ def schedule_future(io_loop, *, delay, result=None): return f -@pytest.mark.gen_test @pytest.mark.parametrize("deadline, n, delay, expected", [ (0, 3, 1, []), (0, 3, 0, [0, 1, 2]), @@ -43,7 +42,6 @@ async def test_iterate_until(io_loop, deadline, n, delay, expected): assert yielded == expected -@pytest.mark.gen_test async def test_iterate_until_ready_after_deadline(io_loop): f = schedule_future(io_loop, delay=0) From 8ca8225cef1c08960f6135480c27607ad6b39164 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 12 Dec 2018 13:45:36 +0100 Subject: [PATCH 13/14] Python 3.5-friendly async context manager --- jupyterhub/tests/test_services.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jupyterhub/tests/test_services.py b/jupyterhub/tests/test_services.py index a318b566..428e69ea 100644 --- a/jupyterhub/tests/test_services.py +++ b/jupyterhub/tests/test_services.py @@ -9,7 +9,7 @@ import sys from threading import Event import time -from async_generator import asynccontextmanager +from async_generator import asynccontextmanager, async_generator, yield_ import pytest import requests from tornado import gen @@ -25,6 +25,7 @@ mockservice_cmd = [sys.executable, mockservice_py] @asynccontextmanager +@async_generator async def external_service(app, name='mockservice'): env = { 'JUPYTERHUB_API_TOKEN': hexlify(os.urandom(5)), @@ -35,7 +36,7 @@ async def external_service(app, name='mockservice'): proc = Popen(mockservice_cmd, env=env) try: await wait_for_http_server(env['JUPYTERHUB_SERVICE_URL']) - yield env + await yield_(env) finally: proc.terminate() From a46032b549121a1ee6d0a1ac19d1340e266e3966 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 12 Dec 2018 15:41:09 +0100 Subject: [PATCH 14/14] use non-deprecated event to register foreign_keys connection listener --- jupyterhub/orm.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/jupyterhub/orm.py b/jupyterhub/orm.py index f0fe39bc..ab5d4494 100644 --- a/jupyterhub/orm.py +++ b/jupyterhub/orm.py @@ -19,7 +19,6 @@ from sqlalchemy import ( DateTime, Enum, Table, ) from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.interfaces import PoolListener from sqlalchemy.orm import ( Session, interfaces, object_session, relationship, sessionmaker, @@ -559,10 +558,13 @@ class DatabaseSchemaMismatch(Exception): """ -class ForeignKeysListener(PoolListener): - """Enable foreign keys on sqlite""" - def connect(self, dbapi_con, con_record): - dbapi_con.execute('pragma foreign_keys=ON') +def register_foreign_keys(engine): + """register PRAGMA foreign_keys=on on connection""" + @event.listens_for(engine, "connect") + def connect(dbapi_con, con_record): + cursor = dbapi_con.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() def _expire_relationship(target, relationship_prop): @@ -735,8 +737,6 @@ def new_session_factory(url="sqlite:///:memory:", """Create a new session at url""" if url.startswith('sqlite'): kwargs.setdefault('connect_args', {'check_same_thread': False}) - listeners = kwargs.setdefault('listeners', []) - listeners.append(ForeignKeysListener()) elif url.startswith('mysql'): kwargs.setdefault('pool_recycle', 60) @@ -747,6 +747,9 @@ def new_session_factory(url="sqlite:///:memory:", kwargs.setdefault('poolclass', StaticPool) engine = create_engine(url, **kwargs) + if url.startswith('sqlite'): + register_foreign_keys(engine) + # enable pessimistic disconnect handling register_ping_connection(engine)