diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 07e6ecb0..a19b07e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,5 @@ # Contributing -See the [contribution guide](https://jupyterhub.readthedocs.io/en/latest/index.html#contributor) section -at the JupyterHub documentation. \ No newline at end of file +See the [contribution guide](https://jupyterhub.readthedocs.io/en/latest/index.html#contributing) section +at the JupyterHub documentation. diff --git a/dev-requirements.txt b/dev-requirements.txt index 4c90579a..0fc2169d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,7 +6,7 @@ coverage cryptography html5lib # needed for beautifulsoup pytest-cov -pytest-tornado +pytest-asyncio pytest>=3.3 notebook requests-mock diff --git a/docs/rest-api.yml b/docs/rest-api.yml index 1eff3a53..8ea61094 100644 --- a/docs/rest-api.yml +++ b/docs/rest-api.yml @@ -185,6 +185,15 @@ paths: in: path required: true type: string + - options: + description: | + Spawn options can be passed as a JSON body + when spawning via the API instead of spawn form. + The structure of the options + will depend on the Spawner's configuration. + in: body + required: false + type: object responses: '201': description: The user's notebook server has started @@ -217,13 +226,15 @@ paths: in: path required: true type: string - - name: remove + - options: description: | - Whether to fully remove the server, rather than just stop it. - Removing a server deletes things like the state of the stopped server. + Spawn options can be passed as a JSON body + when spawning via the API instead of spawn form. + The structure of the options + will depend on the Spawner's configuration. in: body required: false - type: boolean + type: object responses: '201': description: The user's notebook named-server has started @@ -242,6 +253,13 @@ paths: in: path required: true type: string + - name: remove + description: | + Whether to fully remove the server, rather than just stop it. + Removing a server deletes things like the state of the stopped server. + in: body + required: false + type: boolean responses: '204': description: The user's notebook named-server has stopped diff --git a/docs/source/index.rst b/docs/source/index.rst index 41071bad..54d221e5 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -30,7 +30,7 @@ For convenient administration of the Hub, its users, and services, JupyterHub also provides a `REST API`_. The JupyterHub team and Project Jupyter value our community, and JupyterHub -follows the Jupyter [Community Guides](https://jupyter.readthedocs.io/en/latest/community/content-community.html). +follows the Jupyter `Community Guides `_. Contents ======== diff --git a/jupyterhub/app.py b/jupyterhub/app.py index cd3e1374..c170fd30 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -1692,6 +1692,40 @@ class JupyterHub(Application): spawner._log_name) status = -1 + if status is None: + # poll claims it's running. + # Check if it's really there + url_in_db = spawner.server.url + url = await spawner.get_url() + if url != url_in_db: + self.log.warning( + "%s had invalid url %s. Updating to %s", + spawner._log_name, url_in_db, url, + ) + urlinfo = urlparse(url) + spawner.server.protocol = urlinfo.scheme + spawner.server.ip = urlinfo.hostname + if urlinfo.port: + spawner.server.port = urlinfo.port + elif urlinfo.scheme == 'http': + spawner.server.port = 80 + elif urlinfo.scheme == 'https': + spawner.server.port = 443 + self.db.commit() + + self.log.debug( + "Verifying that %s is running at %s", + spawner._log_name, url, + ) + try: + await user._wait_up(spawner) + except TimeoutError: + self.log.error( + "%s does not appear to be running at %s, shutting it down.", + spawner._log_name, url, + ) + status = -1 + if status is None: self.log.info("%s still running", user.name) spawner.add_poll_callback(user_stopped, user, name) diff --git a/jupyterhub/auth.py b/jupyterhub/auth.py index 16550826..f2f6e9d4 100644 --- a/jupyterhub/auth.py +++ b/jupyterhub/auth.py @@ -17,7 +17,6 @@ except Exception as e: _pamela_error = e from tornado.concurrent import run_on_executor -from tornado import gen from traitlets.config import LoggingConfigurable from traitlets import Bool, Set, Unicode, Dict, Any, default, observe @@ -704,7 +703,7 @@ class PAMAuthenticator(LocalAuthenticator): # (returning None instead of just the username) as this indicates some sort of system failure admin_group_gids = {self._getgrnam(x).gr_gid for x in self.admin_groups} - user_group_gids = {x.gr_gid for x in self._getgrouplist(username, self._getpwnam(username).pw_gid)} + user_group_gids = set(self._getgrouplist(username, self._getpwnam(username).pw_gid)) admin_status = len(admin_group_gids & user_group_gids) != 0 except Exception as e: diff --git a/jupyterhub/orm.py b/jupyterhub/orm.py index f0fe39bc..ab5d4494 100644 --- a/jupyterhub/orm.py +++ b/jupyterhub/orm.py @@ -19,7 +19,6 @@ from sqlalchemy import ( DateTime, Enum, Table, ) from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.interfaces import PoolListener from sqlalchemy.orm import ( Session, interfaces, object_session, relationship, sessionmaker, @@ -559,10 +558,13 @@ class DatabaseSchemaMismatch(Exception): """ -class ForeignKeysListener(PoolListener): - """Enable foreign keys on sqlite""" - def connect(self, dbapi_con, con_record): - dbapi_con.execute('pragma foreign_keys=ON') +def register_foreign_keys(engine): + """register PRAGMA foreign_keys=on on connection""" + @event.listens_for(engine, "connect") + def connect(dbapi_con, con_record): + cursor = dbapi_con.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() def _expire_relationship(target, relationship_prop): @@ -735,8 +737,6 @@ def new_session_factory(url="sqlite:///:memory:", """Create a new session at url""" if url.startswith('sqlite'): kwargs.setdefault('connect_args', {'check_same_thread': False}) - listeners = kwargs.setdefault('listeners', []) - listeners.append(ForeignKeysListener()) elif url.startswith('mysql'): kwargs.setdefault('pool_recycle', 60) @@ -747,6 +747,9 @@ def new_session_factory(url="sqlite:///:memory:", kwargs.setdefault('poolclass', StaticPool) engine = create_engine(url, **kwargs) + if url.startswith('sqlite'): + register_foreign_keys(engine) + # enable pessimistic disconnect handling register_ping_connection(engine) diff --git a/jupyterhub/services/auth.py b/jupyterhub/services/auth.py index 1020ae31..90627bc8 100644 --- a/jupyterhub/services/auth.py +++ b/jupyterhub/services/auth.py @@ -852,8 +852,8 @@ class HubAuthenticated(object): self._hub_auth_user_cache = None raise - # store ?token=... tokens passed via url in a cookie for future requests - url_token = self.get_argument('token', '') + # store tokens passed via url or header in a cookie for future requests + url_token = self.hub_auth.get_token(self) if ( user_model and url_token diff --git a/jupyterhub/spawner.py b/jupyterhub/spawner.py index 21add95f..bfaea6a6 100644 --- a/jupyterhub/spawner.py +++ b/jupyterhub/spawner.py @@ -660,6 +660,22 @@ class Spawner(LoggingConfigurable): return env + async def get_url(self): + """Get the URL to connect to the server + + Sometimes JupyterHub may ask the Spawner for its url. + This can occur e.g. when JupyterHub has restarted while a server was not finished starting, + giving Spawners a chance to recover the URL where their server is running. + + The default is to trust that JupyterHub has the right information. + Only override this method in Spawners that know how to + check the correct URL for the servers they start. + + This will only be asked of Spawners that claim to be running + (`poll()` returns `None`). + """ + return self.server.url + def template_namespace(self): """Return the template namespace for format-string formatting. diff --git a/jupyterhub/tests/conftest.py b/jupyterhub/tests/conftest.py index e44dd60c..0c490079 100644 --- a/jupyterhub/tests/conftest.py +++ b/jupyterhub/tests/conftest.py @@ -29,6 +29,7 @@ Fixtures to add functionality or spawning behavior import asyncio from getpass import getuser +import inspect import logging import os import sys @@ -55,6 +56,16 @@ import jupyterhub.services.service _db = None +def pytest_collection_modifyitems(items): + """add asyncio marker to all async tests""" + for item in items: + if inspect.iscoroutinefunction(item.obj): + item.add_marker('asyncio') + if hasattr(inspect, 'isasyncgenfunction'): + # double-check that we aren't mixing yield and async def + assert not inspect.isasyncgenfunction(item.obj) + + @fixture(scope='module') def ssl_tmpdir(tmpdir_factory): return tmpdir_factory.mktemp('ssl') @@ -126,15 +137,21 @@ def db(): @fixture(scope='module') -def io_loop(request): +def event_loop(request): + """Same as pytest-asyncio.event_loop, but re-scoped to module-level""" + event_loop = asyncio.new_event_loop() + asyncio.set_event_loop(event_loop) + return event_loop + + +@fixture(scope='module') +def io_loop(event_loop, request): """Same as pytest-tornado.io_loop, but re-scoped to module-level""" ioloop.IOLoop.configure(AsyncIOMainLoop) - aio_loop = asyncio.new_event_loop() - asyncio.set_event_loop(aio_loop) - io_loop = ioloop.IOLoop() + io_loop = AsyncIOMainLoop() io_loop.make_current() - assert asyncio.get_event_loop() is aio_loop - assert io_loop.asyncio_loop is aio_loop + assert asyncio.get_event_loop() is event_loop + assert io_loop.asyncio_loop is event_loop def _close(): io_loop.clear_current() diff --git a/jupyterhub/tests/test_api.py b/jupyterhub/tests/test_api.py index fac3e83f..8711a2ad 100644 --- a/jupyterhub/tests/test_api.py +++ b/jupyterhub/tests/test_api.py @@ -1,5 +1,6 @@ """Tests for the REST API.""" +import asyncio from datetime import datetime from concurrent.futures import Future import json @@ -84,8 +85,7 @@ def auth_header(db, name): @check_db_locks -@gen.coroutine -def api_request(app, *api_path, **kwargs): +async def api_request(app, *api_path, **kwargs): """Make an API request""" base_url = app.hub.url headers = kwargs.setdefault('headers', {}) @@ -102,7 +102,7 @@ def api_request(app, *api_path, **kwargs): if app.internal_ssl: kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key) kwargs["verify"] = app.internal_ssl_ca - resp = yield f(url, **kwargs) + resp = await f(url, **kwargs) assert "frame-ancestors 'self'" in resp.headers['Content-Security-Policy'] assert ujoin(app.hub.base_url, "security/csp-report") in resp.headers['Content-Security-Policy'] assert 'http' not in resp.headers['Content-Security-Policy'] @@ -116,10 +116,9 @@ def api_request(app, *api_path, **kwargs): # -------------------- -@mark.gen_test -def test_auth_api(app): +async def test_auth_api(app): db = app.db - r = yield api_request(app, 'authorizations', 'gobbledygook') + r = await api_request(app, 'authorizations', 'gobbledygook') assert r.status_code == 404 # make a new cookie token @@ -127,33 +126,32 @@ def test_auth_api(app): api_token = user.new_api_token() # check success: - r = yield api_request(app, 'authorizations/token', api_token) + r = await api_request(app, 'authorizations/token', api_token) assert r.status_code == 200 reply = r.json() assert reply['name'] == user.name # check fail - r = yield api_request(app, 'authorizations/token', api_token, + r = await api_request(app, 'authorizations/token', api_token, headers={'Authorization': 'no sir'}, ) assert r.status_code == 403 - r = yield api_request(app, 'authorizations/token', api_token, + r = await api_request(app, 'authorizations/token', api_token, headers={'Authorization': 'token: %s' % user.cookie_id}, ) assert r.status_code == 403 -@mark.gen_test -def test_referer_check(app): +async def test_referer_check(app): url = ujoin(public_host(app), app.hub.base_url) host = urlparse(url).netloc user = find_user(app.db, 'admin') if user is None: user = add_user(app.db, name='admin', admin=True) - cookies = yield app.login_user('admin') + cookies = await app.login_user('admin') - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers={ 'Authorization': '', 'Referer': 'null', @@ -161,7 +159,7 @@ def test_referer_check(app): ) assert r.status_code == 403 - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers={ 'Authorization': '', 'Referer': 'http://attack.com/csrf/vulnerability', @@ -169,7 +167,7 @@ def test_referer_check(app): ) assert r.status_code == 403 - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers={ 'Authorization': '', 'Referer': url, @@ -178,7 +176,7 @@ def test_referer_check(app): ) assert r.status_code == 200 - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers={ 'Authorization': '', 'Referer': ujoin(url, 'foo/bar/baz/bat'), @@ -240,10 +238,9 @@ def fill_user(model): TIMESTAMP = normalize_timestamp(datetime.now().isoformat() + 'Z') @mark.user -@mark.gen_test -def test_get_users(app): +async def test_get_users(app): db = app.db - r = yield api_request(app, 'users') + r = await api_request(app, 'users') assert r.status_code == 200 users = sorted(r.json(), key=lambda d: d['name']) @@ -260,19 +257,18 @@ def test_get_users(app): }), ] - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers=auth_header(db, 'user'), ) assert r.status_code == 403 @mark.user -@mark.gen_test -def test_get_self(app): +async def test_get_self(app): db = app.db # basic get self - r = yield api_request(app, 'user') + r = await api_request(app, 'user') r.raise_for_status() assert r.json()['kind'] == 'user' @@ -290,7 +286,7 @@ def test_get_self(app): ) db.add(oauth_token) db.commit() - r = yield api_request(app, 'user', headers={ + r = await api_request(app, 'user', headers={ 'Authorization': 'token ' + token, }) r.raise_for_status() @@ -298,18 +294,17 @@ def test_get_self(app): assert model['name'] == u.name # invalid auth gets 403 - r = yield api_request(app, 'user', headers={ + r = await api_request(app, 'user', headers={ 'Authorization': 'token notvalid', }) assert r.status_code == 403 @mark.user -@mark.gen_test -def test_add_user(app): +async def test_add_user(app): db = app.db name = 'newuser' - r = yield api_request(app, 'users', name, method='post') + r = await api_request(app, 'users', name, method='post') assert r.status_code == 201 user = find_user(db, name) assert user is not None @@ -318,10 +313,9 @@ def test_add_user(app): @mark.user -@mark.gen_test -def test_get_user(app): +async def test_get_user(app): name = 'user' - r = yield api_request(app, 'users', name) + r = await api_request(app, 'users', name) assert r.status_code == 200 user = normalize_user(r.json()) @@ -329,21 +323,19 @@ def test_get_user(app): @mark.user -@mark.gen_test -def test_add_multi_user_bad(app): - r = yield api_request(app, 'users', method='post') +async def test_add_multi_user_bad(app): + r = await api_request(app, 'users', method='post') assert r.status_code == 400 - r = yield api_request(app, 'users', method='post', data='{}') + r = await api_request(app, 'users', method='post', data='{}') assert r.status_code == 400 - r = yield api_request(app, 'users', method='post', data='[]') + r = await api_request(app, 'users', method='post', data='[]') assert r.status_code == 400 @mark.user -@mark.gen_test -def test_add_multi_user_invalid(app): +async def test_add_multi_user_invalid(app): app.authenticator.username_pattern = r'w.*' - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': ['Willow', 'Andrew', 'Tara']}) ) app.authenticator.username_pattern = '' @@ -352,11 +344,10 @@ def test_add_multi_user_invalid(app): @mark.user -@mark.gen_test -def test_add_multi_user(app): +async def test_add_multi_user(app): db = app.db names = ['a', 'b'] - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': names}), ) assert r.status_code == 201 @@ -371,7 +362,7 @@ def test_add_multi_user(app): assert not user.admin # try to create the same users again - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': names}), ) assert r.status_code == 409 @@ -379,7 +370,7 @@ def test_add_multi_user(app): names = ['a', 'b', 'ab'] # try to create the same users again - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': names}), ) assert r.status_code == 201 @@ -389,11 +380,10 @@ def test_add_multi_user(app): @mark.user -@mark.gen_test -def test_add_multi_user_admin(app): +async def test_add_multi_user_admin(app): db = app.db names = ['c', 'd'] - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': names, 'admin': True}), ) assert r.status_code == 201 @@ -409,35 +399,32 @@ def test_add_multi_user_admin(app): @mark.user -@mark.gen_test -def test_add_user_bad(app): +async def test_add_user_bad(app): db = app.db name = 'dne_newuser' - r = yield api_request(app, 'users', name, method='post') + r = await api_request(app, 'users', name, method='post') assert r.status_code == 400 user = find_user(db, name) assert user is None @mark.user -@mark.gen_test -def test_add_user_duplicate(app): +async def test_add_user_duplicate(app): db = app.db name = 'user' user = find_user(db, name) # double-check that it exists assert user is not None - r = yield api_request(app, 'users', name, method='post') + r = await api_request(app, 'users', name, method='post') # special 409 conflict for creating a user that already exists assert r.status_code == 409 @mark.user -@mark.gen_test -def test_add_admin(app): +async def test_add_admin(app): db = app.db name = 'newadmin' - r = yield api_request(app, 'users', name, method='post', + r = await api_request(app, 'users', name, method='post', data=json.dumps({'admin': True}), ) assert r.status_code == 201 @@ -448,27 +435,25 @@ def test_add_admin(app): @mark.user -@mark.gen_test -def test_delete_user(app): +async def test_delete_user(app): db = app.db mal = add_user(db, name='mal') - r = yield api_request(app, 'users', 'mal', method='delete') + r = await api_request(app, 'users', 'mal', method='delete') assert r.status_code == 204 @mark.user -@mark.gen_test -def test_make_admin(app): +async def test_make_admin(app): db = app.db name = 'admin2' - r = yield api_request(app, 'users', name, method='post') + r = await api_request(app, 'users', name, method='post') assert r.status_code == 201 user = find_user(db, name) assert user is not None assert user.name == name assert not user.admin - r = yield api_request(app, 'users', name, method='patch', + r = await api_request(app, 'users', name, method='patch', data=json.dumps({'admin': True}) ) assert r.status_code == 200 @@ -479,8 +464,7 @@ def test_make_admin(app): @mark.user -@mark.gen_test -def test_set_auth_state(app, auth_state_enabled): +async def test_set_auth_state(app, auth_state_enabled): auth_state = {'secret': 'hello'} db = app.db name = 'admin' @@ -488,58 +472,55 @@ def test_set_auth_state(app, auth_state_enabled): assert user is not None assert user.name == name - r = yield api_request(app, 'users', name, method='patch', + r = await api_request(app, 'users', name, method='patch', data=json.dumps({'auth_state': auth_state}) ) assert r.status_code == 200 - users_auth_state = yield user.get_auth_state() + users_auth_state = await user.get_auth_state() assert users_auth_state == auth_state @mark.user -@mark.gen_test -def test_user_set_auth_state(app, auth_state_enabled): +async def test_user_set_auth_state(app, auth_state_enabled): auth_state = {'secret': 'hello'} db = app.db name = 'user' user = find_user(db, name, app=app) assert user is not None assert user.name == name - user_auth_state = yield user.get_auth_state() + user_auth_state = await user.get_auth_state() assert user_auth_state is None - r = yield api_request( + r = await api_request( app, 'users', name, method='patch', data=json.dumps({'auth_state': auth_state}), headers=auth_header(app.db, name), ) assert r.status_code == 403 - user_auth_state = yield user.get_auth_state() + user_auth_state = await user.get_auth_state() assert user_auth_state is None @mark.user -@mark.gen_test -def test_admin_get_auth_state(app, auth_state_enabled): +async def test_admin_get_auth_state(app, auth_state_enabled): auth_state = {'secret': 'hello'} db = app.db name = 'admin' user = find_user(db, name, app=app) assert user is not None assert user.name == name - yield user.save_auth_state(auth_state) + await user.save_auth_state(auth_state) - r = yield api_request(app, 'users', name) + r = await api_request(app, 'users', name) assert r.status_code == 200 assert r.json()['auth_state'] == auth_state @mark.user -@mark.gen_test -def test_user_get_auth_state(app, auth_state_enabled): +async def test_user_get_auth_state(app, auth_state_enabled): # explicitly check that a user will not get their own auth state via the API auth_state = {'secret': 'hello'} db = app.db @@ -547,17 +528,16 @@ def test_user_get_auth_state(app, auth_state_enabled): user = find_user(db, name, app=app) assert user is not None assert user.name == name - yield user.save_auth_state(auth_state) + await user.save_auth_state(auth_state) - r = yield api_request(app, 'users', name, + r = await api_request(app, 'users', name, headers=auth_header(app.db, name)) assert r.status_code == 200 assert 'auth_state' not in r.json() -@mark.gen_test -def test_spawn(app): +async def test_spawn(app): db = app.db name = 'wash' user = add_user(db, app=app, name=name) @@ -566,7 +546,7 @@ def test_spawn(app): 'i': 5, } before_servers = sorted(db.query(orm.Server), key=lambda s: s.url) - r = yield api_request(app, 'users', name, 'server', method='post', + r = await api_request(app, 'users', name, 'server', method='post', data=json.dumps(options), ) assert r.status_code == 201 @@ -576,7 +556,7 @@ def test_spawn(app): spawner = app_user.spawner assert app_user.spawner.user_options == options assert not app_user.spawner._spawn_pending - status = yield app_user.spawner.poll() + status = await app_user.spawner.poll() assert status is None assert spawner.server.base_url == ujoin(app.base_url, 'user/%s' % name) + '/' @@ -585,26 +565,26 @@ def test_spawn(app): if app.internal_ssl: kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key) kwargs["verify"] = app.internal_ssl_ca - r = yield async_requests.get(url, **kwargs) + r = await async_requests.get(url, **kwargs) assert r.status_code == 200 assert r.text == spawner.server.base_url - r = yield async_requests.get(ujoin(url, 'args'), **kwargs) + r = await async_requests.get(ujoin(url, 'args'), **kwargs) assert r.status_code == 200 argv = r.json() assert '--port' in ' '.join(argv) - r = yield async_requests.get(ujoin(url, 'env'), **kwargs) + r = await async_requests.get(ujoin(url, 'env'), **kwargs) env = r.json() for expected in ['JUPYTERHUB_USER', 'JUPYTERHUB_BASE_URL', 'JUPYTERHUB_API_TOKEN']: assert expected in env if app.subdomain_host: assert env['JUPYTERHUB_HOST'] == app.subdomain_host - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') assert r.status_code == 204 assert 'pid' not in user.orm_spawners[''].state - status = yield app_user.spawner.poll() + status = await app_user.spawner.poll() assert status == 0 # check that we cleaned up after ourselves @@ -616,8 +596,7 @@ def test_spawn(app): assert app.users.count_active_users()['pending'] == 0 -@mark.gen_test -def test_spawn_handler(app): +async def test_spawn_handler(app): """Test that the requesting Handler is passed to Spawner.handler""" db = app.db name = 'salmon' @@ -625,7 +604,7 @@ def test_spawn_handler(app): app_user = app.users[name] # spawn via API with ?foo=bar - r = yield api_request(app, 'users', name, 'server', method='post', params={'foo': 'bar'}) + r = await api_request(app, 'users', name, 'server', method='post', params={'foo': 'bar'}) r.raise_for_status() # verify that request params got passed down @@ -634,24 +613,23 @@ def test_spawn_handler(app): if app.external_certs: kwargs['verify'] = app.external_certs['files']['ca'] url = public_url(app, user) - r = yield async_requests.get(ujoin(url, 'env'), **kwargs) + r = await async_requests.get(ujoin(url, 'env'), **kwargs) env = r.json() assert 'HANDLER_ARGS' in env assert env['HANDLER_ARGS'] == 'foo=bar' # make user spawner.handler doesn't persist after spawn finishes assert app_user.spawner.handler is None - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() @mark.slow -@mark.gen_test -def test_slow_spawn(app, no_patience, slow_spawn): +async def test_slow_spawn(app, no_patience, slow_spawn): db = app.db name = 'zoe' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() assert r.status_code == 202 assert app_user.spawner is not None @@ -659,83 +637,78 @@ def test_slow_spawn(app, no_patience, slow_spawn): assert not app_user.spawner._stop_pending assert app.users.count_active_users()['pending'] == 1 - @gen.coroutine - def wait_spawn(): + async def wait_spawn(): while not app_user.running: - yield gen.sleep(0.1) + await gen.sleep(0.1) - yield wait_spawn() + await wait_spawn() assert not app_user.spawner._spawn_pending - status = yield app_user.spawner.poll() + status = await app_user.spawner.poll() assert status is None - @gen.coroutine - def wait_stop(): + async def wait_stop(): while app_user.spawner._stop_pending: - yield gen.sleep(0.1) + await gen.sleep(0.1) - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() assert r.status_code == 202 assert app_user.spawner is not None assert app_user.spawner._stop_pending - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() assert r.status_code == 202 assert app_user.spawner is not None assert app_user.spawner._stop_pending - yield wait_stop() + await wait_stop() assert not app_user.spawner._stop_pending assert app_user.spawner is not None - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') # 204 deleted if there's no such server assert r.status_code == 204 assert app.users.count_active_users()['pending'] == 0 assert app.users.count_active_users()['active'] == 0 -@mark.gen_test -def test_never_spawn(app, no_patience, never_spawn): +async def test_never_spawn(app, no_patience, never_spawn): db = app.db name = 'badger' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert app_user.spawner is not None assert app_user.spawner._spawn_pending assert app.users.count_active_users()['pending'] == 1 while app_user.spawner.pending: - yield gen.sleep(0.1) + await gen.sleep(0.1) print(app_user.spawner.pending) assert not app_user.spawner._spawn_pending - status = yield app_user.spawner.poll() + status = await app_user.spawner.poll() assert status is not None # failed spawn should decrements pending count assert app.users.count_active_users()['pending'] == 0 -@mark.gen_test -def test_bad_spawn(app, no_patience, bad_spawn): +async def test_bad_spawn(app, bad_spawn): db = app.db name = 'prim' user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 500 assert app.users.count_active_users()['pending'] == 0 -@mark.gen_test -def test_slow_bad_spawn(app, no_patience, slow_bad_spawn): +async def test_slow_bad_spawn(app, no_patience, slow_bad_spawn): db = app.db name = 'zaphod' user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() while user.spawner.pending: - yield gen.sleep(0.1) + await gen.sleep(0.1) # spawn failed assert not user.running assert app.users.count_active_users()['pending'] == 0 @@ -752,31 +725,30 @@ def next_event(it): return json.loads(line.split(':', 1)[1]) @mark.slow -@mark.gen_test -def test_progress(request, app, no_patience, slow_spawn): +async def test_progress(request, app, no_patience, slow_spawn): db = app.db name = 'martin' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) assert r.headers['content-type'] == 'text/event-stream' ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 0, 'message': 'Server requested', } - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 50, 'message': 'Spawning server...', } - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) url = app_user.url assert evt == { 'progress': 100, @@ -787,32 +759,29 @@ def test_progress(request, app, no_patience, slow_spawn): } -@mark.gen_test -def test_progress_not_started(request, app): +async def test_progress_not_started(request, app): db = app.db name = 'nope' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server/progress') + r = await api_request(app, 'users', name, 'server/progress') assert r.status_code == 404 -@mark.gen_test -def test_progress_not_found(request, app): +async def test_progress_not_found(request, app): db = app.db name = 'noserver' - r = yield api_request(app, 'users', 'nosuchuser', 'server/progress') + r = await api_request(app, 'users', 'nosuchuser', 'server/progress') assert r.status_code == 404 app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server/progress') + r = await api_request(app, 'users', name, 'server/progress') assert r.status_code == 404 -@mark.gen_test -def test_progress_ready(request, app): +async def test_progress_ready(request, app): """Test progress API when spawner is already started e.g. a race between requesting progress and progress already being complete @@ -820,35 +789,34 @@ def test_progress_ready(request, app): db = app.db name = 'saga' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) assert r.headers['content-type'] == 'text/event-stream' ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 100 assert evt['ready'] assert evt['url'] == app_user.url -@mark.gen_test -def test_progress_bad(request, app, no_patience, bad_spawn): +async def test_progress_bad(request, app, bad_spawn): """Test progress API when spawner has already failed""" db = app.db name = 'simon' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 500 - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) assert r.headers['content-type'] == 'text/event-stream' ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 100, 'failed': True, @@ -856,25 +824,24 @@ def test_progress_bad(request, app, no_patience, bad_spawn): } -@mark.gen_test -def test_progress_bad_slow(request, app, no_patience, slow_bad_spawn): +async def test_progress_bad_slow(request, app, no_patience, slow_bad_spawn): """Test progress API when spawner fails while watching""" db = app.db name = 'eugene' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 202 - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) assert r.headers['content-type'] == 'text/event-stream' ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 0 - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 50 - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 100, 'failed': True, @@ -910,8 +877,7 @@ async def progress_forever_native(): """, globals()) -@mark.gen_test -def test_spawn_progress_cutoff(request, app, no_patience, slow_spawn): +async def test_spawn_progress_cutoff(request, app, no_patience, slow_spawn): """Progress events stop when Spawner finishes even if progress iterator is still going. @@ -926,26 +892,25 @@ def test_spawn_progress_cutoff(request, app, no_patience, slow_spawn): app_user.spawner.progress = progress_forever app_user.spawner.delay = 1 - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 0 - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 1, 'message': 'Stage 1', } - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 100 -@mark.gen_test -def test_spawn_limit(app, no_patience, slow_spawn, request): +async def test_spawn_limit(app, no_patience, slow_spawn, request): db = app.db p = mock.patch.dict(app.tornado_settings, {'concurrent_spawn_limit': 2}) @@ -958,24 +923,24 @@ def test_spawn_limit(app, no_patience, slow_spawn, request): users[0].spawner._start_future = Future() users[1].spawner._start_future = Future() for name in names: - yield api_request(app, 'users', name, 'server', method='post') + await api_request(app, 'users', name, 'server', method='post') assert app.users.count_active_users()['pending'] == 2 # ykka and hjarka's spawns are both pending. Essun should fail with 429 name = 'essun' user = add_user(db, app=app, name=name) user.spawner._start_future = Future() - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 429 # allow ykka to start users[0].spawner._start_future.set_result(None) # wait for ykka to finish while not users[0].running: - yield gen.sleep(0.1) + await gen.sleep(0.1) assert app.users.count_active_users()['pending'] == 1 - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() assert app.users.count_active_users()['pending'] == 2 users.append(user) @@ -983,20 +948,19 @@ def test_spawn_limit(app, no_patience, slow_spawn, request): for user in users[1:]: user.spawner._start_future.set_result(None) while not all(u.running for u in users): - yield gen.sleep(0.1) + await gen.sleep(0.1) # everybody's running, pending count should be back to 0 assert app.users.count_active_users()['pending'] == 0 for u in users: u.spawner.delay = 0 - r = yield api_request(app, 'users', u.name, 'server', method='delete') + r = await api_request(app, 'users', u.name, 'server', method='delete') r.raise_for_status() while any(u.spawner.active for u in users): - yield gen.sleep(0.1) + await gen.sleep(0.1) @mark.slow -@mark.gen_test -def test_active_server_limit(app, request): +async def test_active_server_limit(app, request): db = app.db p = mock.patch.dict(app.tornado_settings, {'active_server_limit': 2}) @@ -1007,7 +971,7 @@ def test_active_server_limit(app, request): names = ['ykka', 'hjarka'] users = [ add_user(db, app=app, name=name) for name in names ] for name in names: - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() counts = app.users.count_active_users() assert counts['active'] == 2 @@ -1017,7 +981,7 @@ def test_active_server_limit(app, request): # ykka and hjarka's servers are running. Essun should fail with 429 name = 'essun' user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 429 counts = app.users.count_active_users() assert counts['active'] == 2 @@ -1025,13 +989,13 @@ def test_active_server_limit(app, request): assert counts['pending'] == 0 # stop one server - yield api_request(app, 'users', names[0], 'server', method='delete') + await api_request(app, 'users', names[0], 'server', method='delete') counts = app.users.count_active_users() assert counts['active'] == 1 assert counts['ready'] == 1 assert counts['pending'] == 0 - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() counts = app.users.count_active_users() assert counts['active'] == 2 @@ -1044,7 +1008,7 @@ def test_active_server_limit(app, request): for u in users: if not u.spawner.active: continue - r = yield api_request(app, 'users', u.name, 'server', method='delete') + r = await api_request(app, 'users', u.name, 'server', method='delete') r.raise_for_status() counts = app.users.count_active_users() @@ -1053,78 +1017,75 @@ def test_active_server_limit(app, request): assert counts['pending'] == 0 @mark.slow -@mark.gen_test -def test_start_stop_race(app, no_patience, slow_spawn): +async def test_start_stop_race(app, no_patience, slow_spawn): user = add_user(app.db, app, name='panda') spawner = user.spawner # start the server - r = yield api_request(app, 'users', user.name, 'server', method='post') + r = await api_request(app, 'users', user.name, 'server', method='post') assert r.status_code == 202 assert spawner.pending == 'spawn' # additional spawns while spawning shouldn't trigger a new spawn with mock.patch.object(spawner, 'start') as m: - r = yield api_request(app, 'users', user.name, 'server', method='post') + r = await api_request(app, 'users', user.name, 'server', method='post') assert r.status_code == 202 assert m.call_count == 0 # stop while spawning is not okay - r = yield api_request(app, 'users', user.name, 'server', method='delete') + r = await api_request(app, 'users', user.name, 'server', method='delete') assert r.status_code == 400 while not spawner.ready: - yield gen.sleep(0.1) + await gen.sleep(0.1) spawner.delay = 3 # stop the spawner - r = yield api_request(app, 'users', user.name, 'server', method='delete') + r = await api_request(app, 'users', user.name, 'server', method='delete') assert r.status_code == 202 assert spawner.pending == 'stop' # make sure we get past deleting from the proxy - yield gen.sleep(1) + await gen.sleep(1) # additional stops while stopping shouldn't trigger a new stop with mock.patch.object(spawner, 'stop') as m: - r = yield api_request(app, 'users', user.name, 'server', method='delete') + r = await api_request(app, 'users', user.name, 'server', method='delete') assert r.status_code == 202 assert m.call_count == 0 # start while stopping is not allowed with mock.patch.object(spawner, 'start') as m: - r = yield api_request(app, 'users', user.name, 'server', method='post') + r = await api_request(app, 'users', user.name, 'server', method='post') assert r.status_code == 400 while spawner.active: - yield gen.sleep(0.1) + await gen.sleep(0.1) # start after stop is okay - r = yield api_request(app, 'users', user.name, 'server', method='post') + r = await api_request(app, 'users', user.name, 'server', method='post') assert r.status_code == 202 -@mark.gen_test -def test_get_proxy(app): - r = yield api_request(app, 'proxy') +async def test_get_proxy(app): + r = await api_request(app, 'proxy') r.raise_for_status() reply = r.json() assert list(reply.keys()) == [app.hub.routespec] -@mark.gen_test -def test_cookie(app): +async def test_cookie(app): db = app.db name = 'patience' user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 201 assert 'pid' in user.orm_spawners[''].state app_user = app.users[name] - cookies = yield app.login_user(name) + cookies = await app.login_user(name) cookie_name = app.hub.cookie_name # cookie jar gives '"cookie-value"', we want 'cookie-value' cookie = cookies[cookie_name][1:-1] - r = yield api_request(app, 'authorizations/cookie', + r = await api_request(app, 'authorizations/cookie', cookie_name, "nothintoseehere", ) assert r.status_code == 404 - r = yield api_request(app, 'authorizations/cookie', + r = await api_request(app, 'authorizations/cookie', cookie_name, quote(cookie, safe=''), ) r.raise_for_status() @@ -1132,7 +1093,7 @@ def test_cookie(app): assert reply['name'] == name # deprecated cookie in body: - r = yield api_request(app, 'authorizations/cookie', + r = await api_request(app, 'authorizations/cookie', cookie_name, data=cookie, ) r.raise_for_status() @@ -1146,27 +1107,25 @@ def normalize_token(token): return token -@mark.gen_test -def test_check_token(app): +async def test_check_token(app): name = 'book' user = add_user(app.db, app=app, name=name) token = user.new_api_token() - r = yield api_request(app, 'authorizations/token', token) + r = await api_request(app, 'authorizations/token', token) r.raise_for_status() user_model = r.json() assert user_model['name'] == name - r = yield api_request(app, 'authorizations/token', 'notauthorized') + r = await api_request(app, 'authorizations/token', 'notauthorized') assert r.status_code == 404 -@mark.gen_test @mark.parametrize("headers, status", [ ({}, 200), ({'Authorization': 'token bad'}, 403), ]) -def test_get_new_token_deprecated(app, headers, status): +async def test_get_new_token_deprecated(app, headers, status): # request a new token - r = yield api_request(app, 'authorizations', 'token', + r = await api_request(app, 'authorizations', 'token', method='post', headers=headers, ) @@ -1175,20 +1134,19 @@ def test_get_new_token_deprecated(app, headers, status): return reply = r.json() assert 'token' in reply - r = yield api_request(app, 'authorizations', 'token', reply['token']) + r = await api_request(app, 'authorizations', 'token', reply['token']) r.raise_for_status() reply = r.json() assert reply['name'] == 'admin' -@mark.gen_test -def test_token_formdata_deprecated(app): +async def test_token_formdata_deprecated(app): """Create a token for a user with formdata and no auth header""" data = { 'username': 'fake', 'password': 'fake', } - r = yield api_request(app, 'authorizations', 'token', + r = await api_request(app, 'authorizations', 'token', method='post', data=json.dumps(data) if data else None, noauth=True, @@ -1196,20 +1154,19 @@ def test_token_formdata_deprecated(app): assert r.status_code == 200 reply = r.json() assert 'token' in reply - r = yield api_request(app, 'authorizations', 'token', reply['token']) + r = await api_request(app, 'authorizations', 'token', reply['token']) r.raise_for_status() reply = r.json() assert reply['name'] == data['username'] -@mark.gen_test @mark.parametrize("as_user, for_user, status", [ ('admin', 'other', 200), ('admin', 'missing', 400), ('user', 'other', 403), ('user', 'user', 200), ]) -def test_token_as_user_deprecated(app, as_user, for_user, status): +async def test_token_as_user_deprecated(app, as_user, for_user, status): # ensure both users exist u = add_user(app.db, app, name=as_user) if for_user != 'missing': @@ -1218,7 +1175,7 @@ def test_token_as_user_deprecated(app, as_user, for_user, status): headers = { 'Authorization': 'token %s' % u.new_api_token(), } - r = yield api_request(app, 'authorizations', 'token', + r = await api_request(app, 'authorizations', 'token', method='post', data=json.dumps(data), headers=headers, @@ -1228,19 +1185,18 @@ def test_token_as_user_deprecated(app, as_user, for_user, status): if status != 200: return assert 'token' in reply - r = yield api_request(app, 'authorizations', 'token', reply['token']) + r = await api_request(app, 'authorizations', 'token', reply['token']) r.raise_for_status() reply = r.json() assert reply['name'] == data['username'] -@mark.gen_test @mark.parametrize("headers, status, note, expires_in", [ ({}, 200, 'test note', None), ({}, 200, '', 100), ({'Authorization': 'token bad'}, 403, '', None), ]) -def test_get_new_token(app, headers, status, note, expires_in): +async def test_get_new_token(app, headers, status, note, expires_in): options = {} if note: options['note'] = note @@ -1251,7 +1207,7 @@ def test_get_new_token(app, headers, status, note, expires_in): else: body = '' # request a new token - r = yield api_request(app, 'users/admin/tokens', + r = await api_request(app, 'users/admin/tokens', method='post', headers=headers, data=body, @@ -1279,28 +1235,27 @@ def test_get_new_token(app, headers, status, note, expires_in): initial.pop('token') # check the validity of the new token - r = yield api_request(app, 'users/admin/tokens', token_id) + r = await api_request(app, 'users/admin/tokens', token_id) r.raise_for_status() reply = r.json() assert normalize_token(reply) == initial # delete the token - r = yield api_request(app, 'users/admin/tokens', token_id, + r = await api_request(app, 'users/admin/tokens', token_id, method='delete') assert r.status_code == 204 # verify deletion - r = yield api_request(app, 'users/admin/tokens', token_id) + r = await api_request(app, 'users/admin/tokens', token_id) assert r.status_code == 404 -@mark.gen_test @mark.parametrize("as_user, for_user, status", [ ('admin', 'other', 200), ('admin', 'missing', 404), ('user', 'other', 403), ('user', 'user', 200), ]) -def test_token_for_user(app, as_user, for_user, status): +async def test_token_for_user(app, as_user, for_user, status): # ensure both users exist u = add_user(app.db, app, name=as_user) if for_user != 'missing': @@ -1309,7 +1264,7 @@ def test_token_for_user(app, as_user, for_user, status): headers = { 'Authorization': 'token %s' % u.new_api_token(), } - r = yield api_request(app, 'users', for_user, 'tokens', + r = await api_request(app, 'users', for_user, 'tokens', method='post', data=json.dumps(data), headers=headers, @@ -1320,7 +1275,7 @@ def test_token_for_user(app, as_user, for_user, status): return assert 'token' in reply token_id = reply['id'] - r = yield api_request(app, 'users', for_user, 'tokens', token_id, + r = await api_request(app, 'users', for_user, 'tokens', token_id, headers=headers, ) r.raise_for_status() @@ -1334,20 +1289,19 @@ def test_token_for_user(app, as_user, for_user, status): # delete the token - r = yield api_request(app, 'users', for_user, 'tokens', token_id, + r = await api_request(app, 'users', for_user, 'tokens', token_id, method='delete', headers=headers, ) assert r.status_code == 204 - r = yield api_request(app, 'users', for_user, 'tokens', token_id, + r = await api_request(app, 'users', for_user, 'tokens', token_id, headers=headers, ) assert r.status_code == 404 -@mark.gen_test -def test_token_authenticator_noauth(app): +async def test_token_authenticator_noauth(app): """Create a token for a user relying on Authenticator.authenticate and no auth header""" name = 'user' data = { @@ -1356,7 +1310,7 @@ def test_token_authenticator_noauth(app): 'password': name, }, } - r = yield api_request(app, 'users', name, 'tokens', + r = await api_request(app, 'users', name, 'tokens', method='post', data=json.dumps(data) if data else None, noauth=True, @@ -1364,14 +1318,13 @@ def test_token_authenticator_noauth(app): assert r.status_code == 200 reply = r.json() assert 'token' in reply - r = yield api_request(app, 'authorizations', 'token', reply['token']) + r = await api_request(app, 'authorizations', 'token', reply['token']) r.raise_for_status() reply = r.json() assert reply['name'] == name -@mark.gen_test -def test_token_authenticator_dict_noauth(app): +async def test_token_authenticator_dict_noauth(app): """Create a token for a user relying on Authenticator.authenticate and no auth header""" app.authenticator.auth_state = { 'who': 'cares', @@ -1383,7 +1336,7 @@ def test_token_authenticator_dict_noauth(app): 'password': name, }, } - r = yield api_request(app, 'users', name, 'tokens', + r = await api_request(app, 'users', name, 'tokens', method='post', data=json.dumps(data) if data else None, noauth=True, @@ -1391,27 +1344,26 @@ def test_token_authenticator_dict_noauth(app): assert r.status_code == 200 reply = r.json() assert 'token' in reply - r = yield api_request(app, 'authorizations', 'token', reply['token']) + r = await api_request(app, 'authorizations', 'token', reply['token']) r.raise_for_status() reply = r.json() assert reply['name'] == name -@mark.gen_test @mark.parametrize("as_user, for_user, status", [ ('admin', 'other', 200), ('admin', 'missing', 404), ('user', 'other', 403), ('user', 'user', 200), ]) -def test_token_list(app, as_user, for_user, status): +async def test_token_list(app, as_user, for_user, status): u = add_user(app.db, app, name=as_user) if for_user != 'missing': for_user_obj = add_user(app.db, app, name=for_user) headers = { 'Authorization': 'token %s' % u.new_api_token(), } - r = yield api_request(app, 'users', for_user, 'tokens', + r = await api_request(app, 'users', for_user, 'tokens', headers=headers, ) assert r.status_code == status @@ -1424,7 +1376,7 @@ def test_token_list(app, as_user, for_user, status): assert all(token['user'] == for_user for token in reply['oauth_tokens']) # validate individual token ids for token in reply['api_tokens'] + reply['oauth_tokens']: - r = yield api_request(app, 'users', for_user, 'tokens', token['id'], + r = await api_request(app, 'users', for_user, 'tokens', token['id'], headers=headers, ) r.raise_for_status() @@ -1438,9 +1390,8 @@ def test_token_list(app, as_user, for_user, status): @mark.group -@mark.gen_test -def test_groups_list(app): - r = yield api_request(app, 'groups') +async def test_groups_list(app): + r = await api_request(app, 'groups') r.raise_for_status() reply = r.json() assert reply == [] @@ -1450,7 +1401,7 @@ def test_groups_list(app): app.db.add(group) app.db.commit() - r = yield api_request(app, 'groups') + r = await api_request(app, 'groups') r.raise_for_status() reply = r.json() assert reply == [{ @@ -1461,11 +1412,10 @@ def test_groups_list(app): @mark.group -@mark.gen_test -def test_add_multi_group(app): +async def test_add_multi_group(app): db = app.db names = ['group1', 'group2'] - r = yield api_request(app, 'groups', method='post', + r = await api_request(app, 'groups', method='post', data=json.dumps({'groups': names}), ) assert r.status_code == 201 @@ -1474,24 +1424,23 @@ def test_add_multi_group(app): assert names == r_names # try to create the same groups again - r = yield api_request(app, 'groups', method='post', + r = await api_request(app, 'groups', method='post', data=json.dumps({'groups': names}), ) assert r.status_code == 409 @mark.group -@mark.gen_test -def test_group_get(app): +async def test_group_get(app): group = orm.Group.find(app.db, name='alphaflight') user = add_user(app.db, app=app, name='sasquatch') group.users.append(user) app.db.commit() - r = yield api_request(app, 'groups/runaways') + r = await api_request(app, 'groups/runaways') assert r.status_code == 404 - r = yield api_request(app, 'groups/alphaflight') + r = await api_request(app, 'groups/alphaflight') r.raise_for_status() reply = r.json() assert reply == { @@ -1502,19 +1451,18 @@ def test_group_get(app): @mark.group -@mark.gen_test -def test_group_create_delete(app): +async def test_group_create_delete(app): db = app.db - r = yield api_request(app, 'groups/runaways', method='delete') + r = await api_request(app, 'groups/runaways', method='delete') assert r.status_code == 404 - r = yield api_request(app, 'groups/new', method='post', + r = await api_request(app, 'groups/new', method='post', data=json.dumps({'users': ['doesntexist']}), ) assert r.status_code == 400 assert orm.Group.find(db, name='new') is None - r = yield api_request(app, 'groups/omegaflight', method='post', + r = await api_request(app, 'groups/omegaflight', method='post', data=json.dumps({'users': ['sasquatch']}), ) r.raise_for_status() @@ -1525,30 +1473,29 @@ def test_group_create_delete(app): assert sasquatch in omegaflight.users # create duplicate raises 400 - r = yield api_request(app, 'groups/omegaflight', method='post') + r = await api_request(app, 'groups/omegaflight', method='post') assert r.status_code == 409 - r = yield api_request(app, 'groups/omegaflight', method='delete') + r = await api_request(app, 'groups/omegaflight', method='delete') assert r.status_code == 204 assert omegaflight not in sasquatch.groups assert orm.Group.find(db, name='omegaflight') is None # delete nonexistent gives 404 - r = yield api_request(app, 'groups/omegaflight', method='delete') + r = await api_request(app, 'groups/omegaflight', method='delete') assert r.status_code == 404 @mark.group -@mark.gen_test -def test_group_add_users(app): +async def test_group_add_users(app): db = app.db # must specify users - r = yield api_request(app, 'groups/alphaflight/users', method='post', data='{}') + r = await api_request(app, 'groups/alphaflight/users', method='post', data='{}') assert r.status_code == 400 names = ['aurora', 'guardian', 'northstar', 'sasquatch', 'shaman', 'snowbird'] users = [ find_user(db, name=name) or add_user(db, app=app, name=name) for name in names ] - r = yield api_request(app, 'groups/alphaflight/users', method='post', data=json.dumps({ + r = await api_request(app, 'groups/alphaflight/users', method='post', data=json.dumps({ 'users': names, })) r.raise_for_status() @@ -1562,16 +1509,15 @@ def test_group_add_users(app): @mark.group -@mark.gen_test -def test_group_delete_users(app): +async def test_group_delete_users(app): db = app.db # must specify users - r = yield api_request(app, 'groups/alphaflight/users', method='delete', data='{}') + r = await api_request(app, 'groups/alphaflight/users', method='delete', data='{}') assert r.status_code == 400 names = ['aurora', 'guardian', 'northstar', 'sasquatch', 'shaman', 'snowbird'] users = [ find_user(db, name=name) for name in names ] - r = yield api_request(app, 'groups/alphaflight/users', method='delete', data=json.dumps({ + r = await api_request(app, 'groups/alphaflight/users', method='delete', data=json.dumps({ 'users': names[:2], })) r.raise_for_status() @@ -1591,11 +1537,10 @@ def test_group_delete_users(app): @mark.services -@mark.gen_test -def test_get_services(app, mockservice_url): +async def test_get_services(app, mockservice_url): mockservice = mockservice_url db = app.db - r = yield api_request(app, 'services') + r = await api_request(app, 'services') r.raise_for_status() assert r.status_code == 200 @@ -1612,18 +1557,17 @@ def test_get_services(app, mockservice_url): } } - r = yield api_request(app, 'services', + r = await api_request(app, 'services', headers=auth_header(db, 'user'), ) assert r.status_code == 403 @mark.services -@mark.gen_test -def test_get_service(app, mockservice_url): +async def test_get_service(app, mockservice_url): mockservice = mockservice_url db = app.db - r = yield api_request(app, 'services/%s' % mockservice.name) + r = await api_request(app, 'services/%s' % mockservice.name) r.raise_for_status() assert r.status_code == 200 @@ -1638,27 +1582,26 @@ def test_get_service(app, mockservice_url): 'info': {}, } - r = yield api_request(app, 'services/%s' % mockservice.name, + r = await api_request(app, 'services/%s' % mockservice.name, headers={ 'Authorization': 'token %s' % mockservice.api_token } ) r.raise_for_status() - r = yield api_request(app, 'services/%s' % mockservice.name, + r = await api_request(app, 'services/%s' % mockservice.name, headers=auth_header(db, 'user'), ) assert r.status_code == 403 -@mark.gen_test -def test_root_api(app): +async def test_root_api(app): base_url = app.hub.url url = ujoin(base_url, 'api') kwargs = {} if app.internal_ssl: kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key) kwargs["verify"] = app.internal_ssl_ca - r = yield async_requests.get(url, **kwargs) + r = await async_requests.get(url, **kwargs) r.raise_for_status() expected = { 'version': jupyterhub.__version__ @@ -1666,9 +1609,8 @@ def test_root_api(app): assert r.json() == expected -@mark.gen_test -def test_info(app): - r = yield api_request(app, 'info') +async def test_info(app): + r = await api_request(app, 'info') r.raise_for_status() data = r.json() assert data['version'] == jupyterhub.__version__ @@ -1696,16 +1638,14 @@ def test_info(app): # ----------------- -@mark.gen_test -def test_options(app): - r = yield api_request(app, 'users', method='options') +async def test_options(app): + r = await api_request(app, 'users', method='options') r.raise_for_status() assert 'Access-Control-Allow-Headers' in r.headers -@mark.gen_test -def test_bad_json_body(app): - r = yield api_request(app, 'users', method='post', data='notjson') +async def test_bad_json_body(app): + r = await api_request(app, 'users', method='post', data='notjson') assert r.status_code == 400 @@ -1720,9 +1660,8 @@ def test_shutdown(app): # have to do things a little funky since we are going to stop the loop, # which makes gen_test unhappy. So we run the loop ourselves. - @gen.coroutine - def shutdown(): - r = yield api_request(app, 'shutdown', method='post', + async def shutdown(): + r = await api_request(app, 'shutdown', method='post', data=json.dumps({'servers': True, 'proxy': True,}), ) return r diff --git a/jupyterhub/tests/test_app.py b/jupyterhub/tests/test_app.py index c758489d..f34d3c5a 100644 --- a/jupyterhub/tests/test_app.py +++ b/jupyterhub/tests/test_app.py @@ -63,8 +63,7 @@ def test_generate_config(): assert 'Authenticator.whitelist' in cfg_text -@pytest.mark.gen_test -def test_init_tokens(request): +async def test_init_tokens(request): with TemporaryDirectory() as td: db_file = os.path.join(td, 'jupyterhub.sqlite') tokens = { @@ -77,7 +76,7 @@ def test_init_tokens(request): if ssl_enabled: kwargs['internal_certs_location'] = td app = MockHub(**kwargs) - yield app.initialize([]) + await app.initialize([]) db = app.db for token, username in tokens.items(): api_token = orm.APIToken.find(db, token) @@ -87,7 +86,7 @@ def test_init_tokens(request): # simulate second startup, reloading same tokens: app = MockHub(**kwargs) - yield app.initialize([]) + await app.initialize([]) db = app.db for token, username in tokens.items(): api_token = orm.APIToken.find(db, token) @@ -99,7 +98,7 @@ def test_init_tokens(request): tokens['short'] = 'gman' app = MockHub(**kwargs) with pytest.raises(ValueError): - yield app.initialize([]) + await app.initialize([]) assert orm.User.find(app.db, 'gman') is None @@ -169,8 +168,7 @@ def test_cookie_secret_env(tmpdir, request): assert not os.path.exists(hub.cookie_secret_file) -@pytest.mark.gen_test -def test_load_groups(tmpdir, request): +async def test_load_groups(tmpdir, request): to_load = { 'blue': ['cyclops', 'rogue', 'wolverine'], 'gold': ['storm', 'jean-grey', 'colossus'], @@ -181,8 +179,8 @@ def test_load_groups(tmpdir, request): kwargs['internal_certs_location'] = str(tmpdir) hub = MockHub(**kwargs) hub.init_db() - yield hub.init_users() - yield hub.init_groups() + await hub.init_users() + await hub.init_groups() db = hub.db blue = orm.Group.find(db, name='blue') assert blue is not None @@ -192,16 +190,15 @@ def test_load_groups(tmpdir, request): assert sorted([ u.name for u in gold.users ]) == sorted(to_load['gold']) -@pytest.mark.gen_test -def test_resume_spawners(tmpdir, request): +async def test_resume_spawners(tmpdir, request): if not os.getenv('JUPYTERHUB_TEST_DB_URL'): p = patch.dict(os.environ, { 'JUPYTERHUB_TEST_DB_URL': 'sqlite:///%s' % tmpdir.join('jupyterhub.sqlite'), }) p.start() request.addfinalizer(p.stop) - @gen.coroutine - def new_hub(): + + async def new_hub(): kwargs = {} ssl_enabled = getattr(request.module, "ssl_enabled", False) if ssl_enabled: @@ -209,26 +206,27 @@ def test_resume_spawners(tmpdir, request): app = MockHub(test_clean_db=False, **kwargs) app.config.ConfigurableHTTPProxy.should_start = False app.config.ConfigurableHTTPProxy.auth_token = 'unused' - yield app.initialize([]) + await app.initialize([]) return app - app = yield new_hub() + + app = await new_hub() db = app.db # spawn a user's server name = 'kurt' user = add_user(db, app, name=name) - yield user.spawn() + await user.spawn() proc = user.spawner.proc assert proc is not None # stop the Hub without cleaning up servers app.cleanup_servers = False - yield app.stop() + app.stop() # proc is still running assert proc.poll() is None # resume Hub, should still be running - app = yield new_hub() + app = await new_hub() db = app.db user = app.users[name] assert user.running @@ -236,7 +234,7 @@ def test_resume_spawners(tmpdir, request): # stop the Hub without cleaning up servers app.cleanup_servers = False - yield app.stop() + app.stop() # stop the server while the Hub is down. BAMF! proc.terminate() @@ -244,7 +242,7 @@ def test_resume_spawners(tmpdir, request): assert proc.poll() is not None # resume Hub, should be stopped - app = yield new_hub() + app = await new_hub() db = app.db user = app.users[name] assert not user.running diff --git a/jupyterhub/tests/test_auth.py b/jupyterhub/tests/test_auth.py index e872d6bc..2783af68 100644 --- a/jupyterhub/tests/test_auth.py +++ b/jupyterhub/tests/test_auth.py @@ -14,47 +14,44 @@ from jupyterhub import auth, crypto, orm from .mocking import MockPAMAuthenticator, MockStructGroup, MockStructPasswd from .test_api import add_user -@pytest.mark.gen_test -def test_pam_auth(): +async def test_pam_auth(): authenticator = MockPAMAuthenticator() - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'match', 'password': 'match', }) assert authorized['name'] == 'match' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'match', 'password': 'nomatch', }) assert authorized is None # Account check is on by default for increased security - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'notallowedmatch', 'password': 'notallowedmatch', }) assert authorized is None -@pytest.mark.gen_test -def test_pam_auth_account_check_disabled(): +async def test_pam_auth_account_check_disabled(): authenticator = MockPAMAuthenticator(check_account=False) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'allowedmatch', 'password': 'allowedmatch', }) assert authorized['name'] == 'allowedmatch' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'notallowedmatch', 'password': 'notallowedmatch', }) assert authorized['name'] == 'notallowedmatch' -@pytest.mark.gen_test -def test_pam_auth_admin_groups(): +async def test_pam_auth_admin_groups(): jh_users = MockStructGroup('jh_users', ['group_admin', 'also_group_admin', 'override_admin', 'non_admin'], 1234) jh_admins = MockStructGroup('jh_admins', ['group_admin'], 5678) wheel = MockStructGroup('wheel', ['also_group_admin'], 9999) @@ -67,10 +64,10 @@ def test_pam_auth_admin_groups(): system_users = [group_admin, also_group_admin, override_admin, non_admin] user_group_map = { - 'group_admin': [jh_users, jh_admins], - 'also_group_admin': [jh_users, wheel], - 'override_admin': [jh_users], - 'non_admin': [jh_users] + 'group_admin': [jh_users.gr_gid, jh_admins.gr_gid], + 'also_group_admin': [jh_users.gr_gid, wheel.gr_gid], + 'override_admin': [jh_users.gr_gid], + 'non_admin': [jh_users.gr_gid] } def getgrnam(name): @@ -90,7 +87,7 @@ def test_pam_auth_admin_groups(): _getgrnam=getgrnam, _getpwnam=getpwnam, _getgrouplist=getgrouplist): - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'group_admin', 'password': 'group_admin' }) @@ -102,7 +99,7 @@ def test_pam_auth_admin_groups(): _getgrnam=getgrnam, _getpwnam=getpwnam, _getgrouplist=getgrouplist): - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'also_group_admin', 'password': 'also_group_admin' }) @@ -114,7 +111,7 @@ def test_pam_auth_admin_groups(): _getgrnam=getgrnam, _getpwnam=getpwnam, _getgrouplist=getgrouplist): - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'override_admin', 'password': 'override_admin' }) @@ -126,7 +123,7 @@ def test_pam_auth_admin_groups(): _getgrnam=getgrnam, _getpwnam=getpwnam, _getgrouplist=getgrouplist): - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'non_admin', 'password': 'non_admin' }) @@ -134,55 +131,52 @@ def test_pam_auth_admin_groups(): assert authorized['admin'] is False -@pytest.mark.gen_test -def test_pam_auth_whitelist(): +async def test_pam_auth_whitelist(): authenticator = MockPAMAuthenticator(whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) assert authorized['name'] == 'kaylee' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'nomatch', }) assert authorized is None - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'mal', 'password': 'mal', }) assert authorized is None -@pytest.mark.gen_test -def test_pam_auth_group_whitelist(): +async def test_pam_auth_group_whitelist(): def getgrnam(name): return MockStructGroup('grp', ['kaylee']) authenticator = MockPAMAuthenticator(group_whitelist={'group'}) with mock.patch.object(authenticator, '_getgrnam', getgrnam): - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) assert authorized['name'] == 'kaylee' with mock.patch.object(authenticator, '_getgrnam', getgrnam): - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'mal', 'password': 'mal', }) assert authorized is None -@pytest.mark.gen_test -def test_pam_auth_blacklist(): +async def test_pam_auth_blacklist(): # Null case compared to next case authenticator = MockPAMAuthenticator() - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'wash', }) @@ -190,7 +184,7 @@ def test_pam_auth_blacklist(): # Blacklist basics authenticator = MockPAMAuthenticator(blacklist={'wash'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'wash', }) @@ -198,7 +192,7 @@ def test_pam_auth_blacklist(): # User in both white and blacklists: default deny. Make error someday? authenticator = MockPAMAuthenticator(blacklist={'wash'}, whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'wash', }) @@ -206,7 +200,7 @@ def test_pam_auth_blacklist(): # User not in blacklist can log in authenticator = MockPAMAuthenticator(blacklist={'wash'}, whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) @@ -214,7 +208,7 @@ def test_pam_auth_blacklist(): # User in whitelist, blacklist irrelevent authenticator = MockPAMAuthenticator(blacklist={'mal'}, whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'wash', }) @@ -222,7 +216,7 @@ def test_pam_auth_blacklist(): # User in neither list authenticator = MockPAMAuthenticator(blacklist={'mal'}, whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'simon', 'password': 'simon', }) @@ -230,34 +224,31 @@ def test_pam_auth_blacklist(): # blacklist == {} authenticator = MockPAMAuthenticator(blacklist=set(), whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) assert authorized['name'] == 'kaylee' -@pytest.mark.gen_test -def test_pam_auth_no_such_group(): +async def test_pam_auth_no_such_group(): authenticator = MockPAMAuthenticator(group_whitelist={'nosuchcrazygroup'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) assert authorized is None -@pytest.mark.gen_test -def test_wont_add_system_user(): +async def test_wont_add_system_user(): user = orm.User(name='lioness4321') authenticator = auth.PAMAuthenticator(whitelist={'mal'}) authenticator.create_system_users = False with pytest.raises(KeyError): - yield authenticator.add_user(user) + await authenticator.add_user(user) -@pytest.mark.gen_test -def test_cant_add_system_user(): +async def test_cant_add_system_user(): user = orm.User(name='lioness4321') authenticator = auth.PAMAuthenticator(whitelist={'mal'}) authenticator.add_user_cmd = ['jupyterhub-fake-command'] @@ -279,12 +270,11 @@ def test_cant_add_system_user(): with mock.patch.object(auth, 'Popen', DummyPopen): with pytest.raises(RuntimeError) as exc: - yield authenticator.add_user(user) + await authenticator.add_user(user) assert str(exc.value) == 'Failed to create system user lioness4321: dummy error' -@pytest.mark.gen_test -def test_add_system_user(): +async def test_add_system_user(): user = orm.User(name='lioness4321') authenticator = auth.PAMAuthenticator(whitelist={'mal'}) authenticator.create_system_users = True @@ -300,17 +290,16 @@ def test_add_system_user(): return with mock.patch.object(auth, 'Popen', DummyPopen): - yield authenticator.add_user(user) + await authenticator.add_user(user) assert record['cmd'] == ['echo', '/home/lioness4321', 'lioness4321'] -@pytest.mark.gen_test -def test_delete_user(): +async def test_delete_user(): user = orm.User(name='zoe') a = MockPAMAuthenticator(whitelist={'mal'}) assert 'zoe' not in a.whitelist - yield a.add_user(user) + await a.add_user(user) assert 'zoe' in a.whitelist a.delete_user(user) assert 'zoe' not in a.whitelist @@ -330,47 +319,43 @@ def test_handlers(app): assert handlers[0][0] == '/login' -@pytest.mark.gen_test -def test_auth_state(app, auth_state_enabled): +async def test_auth_state(app, auth_state_enabled): """auth_state enabled and available""" name = 'kiwi' user = add_user(app.db, app, name=name) assert user.encrypted_auth_state is None - cookies = yield app.login_user(name) - auth_state = yield user.get_auth_state() + cookies = await app.login_user(name) + auth_state = await user.get_auth_state() assert auth_state == app.authenticator.auth_state -@pytest.mark.gen_test -def test_auth_admin_non_admin(app): +async def test_auth_admin_non_admin(app): """admin should be passed through for non-admin users""" name = 'kiwi' user = add_user(app.db, app, name=name, admin=False) assert user.admin is False - cookies = yield app.login_user(name) + cookies = await app.login_user(name) assert user.admin is False -@pytest.mark.gen_test -def test_auth_admin_is_admin(app): +async def test_auth_admin_is_admin(app): """admin should be passed through for admin users""" # Admin user defined in MockPAMAuthenticator. name = 'admin' user = add_user(app.db, app, name=name, admin=False) assert user.admin is False - cookies = yield app.login_user(name) + cookies = await app.login_user(name) assert user.admin is True -@pytest.mark.gen_test -def test_auth_admin_retained_if_unset(app): +async def test_auth_admin_retained_if_unset(app): """admin should be unchanged if authenticator doesn't return admin value""" name = 'kiwi' # Add user as admin. user = add_user(app.db, app, name=name, admin=True) assert user.admin is True # User should remain unchanged. - cookies = yield app.login_user(name) + cookies = await app.login_user(name) assert user.admin is True @@ -384,56 +369,53 @@ def auth_state_unavailable(auth_state_enabled): yield -@pytest.mark.gen_test -def test_auth_state_disabled(app, auth_state_unavailable): +async def test_auth_state_disabled(app, auth_state_unavailable): name = 'driebus' user = add_user(app.db, app, name=name) assert user.encrypted_auth_state is None with pytest.raises(HTTPError): - cookies = yield app.login_user(name) - auth_state = yield user.get_auth_state() + cookies = await app.login_user(name) + auth_state = await user.get_auth_state() assert auth_state is None -@pytest.mark.gen_test -def test_normalize_names(): +async def test_normalize_names(): a = MockPAMAuthenticator() - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'ZOE', 'password': 'ZOE', }) assert authorized['name'] == 'zoe' - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'Glenn', 'password': 'Glenn', }) assert authorized['name'] == 'glenn' - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'hExi', 'password': 'hExi', }) assert authorized['name'] == 'hexi' - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'Test', 'password': 'Test', }) assert authorized['name'] == 'test' -@pytest.mark.gen_test -def test_username_map(): +async def test_username_map(): a = MockPAMAuthenticator(username_map={'wash': 'alpha'}) - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'WASH', 'password': 'WASH', }) assert authorized['name'] == 'alpha' - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'Inara', 'password': 'Inara', }) diff --git a/jupyterhub/tests/test_crypto.py b/jupyterhub/tests/test_crypto.py index e133745f..db7799a0 100644 --- a/jupyterhub/tests/test_crypto.py +++ b/jupyterhub/tests/test_crypto.py @@ -11,6 +11,7 @@ keys = [('%i' % i).encode('ascii') * 32 for i in range(3)] hex_keys = [ b2a_hex(key).decode('ascii') for key in keys ] b64_keys = [ b2a_base64(key).decode('ascii').strip() for key in keys ] + @pytest.mark.parametrize("key_env, keys", [ (hex_keys[0], [keys[0]]), (';'.join([b64_keys[0], hex_keys[1]]), keys[:2]), @@ -52,30 +53,27 @@ def crypt_keeper(): ck.keys = save_keys -@pytest.mark.gen_test -def test_roundtrip(crypt_keeper): +async def test_roundtrip(crypt_keeper): data = {'key': 'value'} - encrypted = yield encrypt(data) - decrypted = yield decrypt(encrypted) + encrypted = await encrypt(data) + decrypted = await decrypt(encrypted) assert decrypted == data -@pytest.mark.gen_test -def test_missing_crypto(crypt_keeper): +async def test_missing_crypto(crypt_keeper): with patch.object(crypto, 'cryptography', None): with pytest.raises(crypto.CryptographyUnavailable): - yield encrypt({}) + await encrypt({}) with pytest.raises(crypto.CryptographyUnavailable): - yield decrypt(b'whatever') + await decrypt(b'whatever') -@pytest.mark.gen_test -def test_missing_keys(crypt_keeper): +async def test_missing_keys(crypt_keeper): crypt_keeper.keys = [] with pytest.raises(crypto.NoEncryptionKeys): - yield encrypt({}) + await encrypt({}) with pytest.raises(crypto.NoEncryptionKeys): - yield decrypt(b'whatever') + await decrypt(b'whatever') diff --git a/jupyterhub/tests/test_db.py b/jupyterhub/tests/test_db.py index cc166aad..18a8e952 100644 --- a/jupyterhub/tests/test_db.py +++ b/jupyterhub/tests/test_db.py @@ -40,8 +40,7 @@ def generate_old_db(env_dir, hub_version, db_url): '0.8.1', ], ) -@pytest.mark.gen_test -def test_upgrade(tmpdir, hub_version): +async def test_upgrade(tmpdir, hub_version): db_url = os.getenv('JUPYTERHUB_TEST_DB_URL') if db_url: db_url += '_upgrade_' + hub_version.replace('.', '') @@ -69,7 +68,7 @@ def test_upgrade(tmpdir, hub_version): assert len(sqlite_files) == 1 upgradeapp = UpgradeDB(config=cfg) - yield upgradeapp.initialize([]) + upgradeapp.initialize([]) upgradeapp.start() # check that backup was created: diff --git a/jupyterhub/tests/test_dummyauth.py b/jupyterhub/tests/test_dummyauth.py index 8e419ef2..84fb480e 100644 --- a/jupyterhub/tests/test_dummyauth.py +++ b/jupyterhub/tests/test_dummyauth.py @@ -7,48 +7,45 @@ import pytest from jupyterhub.auth import DummyAuthenticator -@pytest.mark.gen_test -def test_dummy_auth_without_global_password(): +async def test_dummy_auth_without_global_password(): authenticator = DummyAuthenticator() - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'test_user', 'password': 'test_pass', }) assert authorized['name'] == 'test_user' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'test_user', 'password': '', }) assert authorized['name'] == 'test_user' -@pytest.mark.gen_test -def test_dummy_auth_without_username(): +async def test_dummy_auth_without_username(): authenticator = DummyAuthenticator() - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': '', 'password': 'test_pass', }) assert authorized is None -@pytest.mark.gen_test -def test_dummy_auth_with_global_password(): +async def test_dummy_auth_with_global_password(): authenticator = DummyAuthenticator() authenticator.password = "test_password" - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'test_user', 'password': 'test_password', }) assert authorized['name'] == 'test_user' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'test_user', 'password': 'qwerty', }) assert authorized is None - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'some_other_user', 'password': 'test_password', }) diff --git a/jupyterhub/tests/test_internal_ssl_connections.py b/jupyterhub/tests/test_internal_ssl_connections.py index 42b306aa..be5ddcca 100644 --- a/jupyterhub/tests/test_internal_ssl_connections.py +++ b/jupyterhub/tests/test_internal_ssl_connections.py @@ -39,39 +39,36 @@ def wait_for_spawner(spawner, timeout=10): yield wait() -@pytest.mark.gen_test -def test_connection_hub_wrong_certs(app): +async def test_connection_hub_wrong_certs(app): """Connecting to the internal hub url fails without correct certs""" with pytest.raises(SSLError): kwargs = {'verify': False} - r = yield async_requests.get(app.hub.url, **kwargs) + r = await async_requests.get(app.hub.url, **kwargs) r.raise_for_status() -@pytest.mark.gen_test -def test_connection_proxy_api_wrong_certs(app): +async def test_connection_proxy_api_wrong_certs(app): """Connecting to the proxy api fails without correct certs""" with pytest.raises(SSLError): kwargs = {'verify': False} - r = yield async_requests.get(app.proxy.api_url, **kwargs) + r = await async_requests.get(app.proxy.api_url, **kwargs) r.raise_for_status() -@pytest.mark.gen_test -def test_connection_notebook_wrong_certs(app): +async def test_connection_notebook_wrong_certs(app): """Connecting to a notebook fails without correct certs""" with mock.patch.dict( app.config.LocalProcessSpawner, {'cmd': [sys.executable, '-m', 'jupyterhub.tests.mocksu']} ): user = add_user(app.db, app, name='foo') - yield user.spawn() - yield wait_for_spawner(user.spawner) + await user.spawn() + await wait_for_spawner(user.spawner) spawner = user.spawner - status = yield spawner.poll() + status = await spawner.poll() assert status is None with pytest.raises(SSLError): kwargs = {'verify': False} - r = yield async_requests.get(spawner.server.url, **kwargs) + r = await async_requests.get(spawner.server.url, **kwargs) r.raise_for_status() diff --git a/jupyterhub/tests/test_named_servers.py b/jupyterhub/tests/test_named_servers.py index 8f808d33..40bca0cf 100644 --- a/jupyterhub/tests/test_named_servers.py +++ b/jupyterhub/tests/test_named_servers.py @@ -17,16 +17,15 @@ def named_servers(app): yield -@pytest.mark.gen_test -def test_default_server(app, named_servers): +async def test_default_server(app, named_servers): """Test the default /users/:user/server handler when named servers are enabled""" username = 'rosie' user = add_user(app.db, app, name=username) - r = yield api_request(app, 'users', username, 'server', method='post') + r = await api_request(app, 'users', username, 'server', method='post') assert r.status_code == 201 assert r.text == '' - r = yield api_request(app, 'users', username) + r = await api_request(app, 'users', username) r.raise_for_status() user_model = normalize_user(r.json()) @@ -51,11 +50,11 @@ def test_default_server(app, named_servers): # now stop the server - r = yield api_request(app, 'users', username, 'server', method='delete') + r = await api_request(app, 'users', username, 'server', method='delete') assert r.status_code == 204 assert r.text == '' - r = yield api_request(app, 'users', username) + r = await api_request(app, 'users', username) r.raise_for_status() user_model = normalize_user(r.json()) @@ -66,20 +65,19 @@ def test_default_server(app, named_servers): }) -@pytest.mark.gen_test -def test_create_named_server(app, named_servers): +async def test_create_named_server(app, named_servers): username = 'walnut' user = add_user(app.db, app, name=username) # assert user.allow_named_servers == True - cookies = yield app.login_user(username) + cookies = await app.login_user(username) servername = 'trevor' - r = yield api_request(app, 'users', username, 'servers', servername, method='post') + r = await api_request(app, 'users', username, 'servers', servername, method='post') r.raise_for_status() assert r.status_code == 201 assert r.text == '' url = url_path_join(public_url(app, user), servername, 'env') - r = yield async_requests.get(url, cookies=cookies) + r = await async_requests.get(url, cookies=cookies) r.raise_for_status() assert r.url == url env = r.json() @@ -87,7 +85,7 @@ def test_create_named_server(app, named_servers): assert prefix == user.spawners[servername].server.base_url assert prefix.endswith('/user/%s/%s/' % (username, servername)) - r = yield api_request(app, 'users', username) + r = await api_request(app, 'users', username) r.raise_for_status() user_model = normalize_user(r.json()) @@ -111,22 +109,21 @@ def test_create_named_server(app, named_servers): }) -@pytest.mark.gen_test -def test_delete_named_server(app, named_servers): +async def test_delete_named_server(app, named_servers): username = 'donaar' user = add_user(app.db, app, name=username) assert user.allow_named_servers cookies = app.login_user(username) servername = 'splugoth' - r = yield api_request(app, 'users', username, 'servers', servername, method='post') + r = await api_request(app, 'users', username, 'servers', servername, method='post') r.raise_for_status() assert r.status_code == 201 - r = yield api_request(app, 'users', username, 'servers', servername, method='delete') + r = await api_request(app, 'users', username, 'servers', servername, method='delete') r.raise_for_status() assert r.status_code == 204 - r = yield api_request(app, 'users', username) + r = await api_request(app, 'users', username) r.raise_for_status() user_model = normalize_user(r.json()) @@ -140,7 +137,7 @@ def test_delete_named_server(app, named_servers): # low-level record still exists assert servername in user.orm_spawners - r = yield api_request( + r = await api_request( app, 'users', username, 'servers', servername, method='delete', data=json.dumps({'remove': True}), @@ -151,11 +148,10 @@ def test_delete_named_server(app, named_servers): assert servername not in user.orm_spawners -@pytest.mark.gen_test -def test_named_server_disabled(app): +async def test_named_server_disabled(app): username = 'user' servername = 'okay' - r = yield api_request(app, 'users', username, 'servers', servername, method='post') + r = await api_request(app, 'users', username, 'servers', servername, method='post') assert r.status_code == 400 - r = yield api_request(app, 'users', username, 'servers', servername, method='delete') + r = await api_request(app, 'users', username, 'servers', servername, method='delete') assert r.status_code == 400 diff --git a/jupyterhub/tests/test_orm.py b/jupyterhub/tests/test_orm.py index cb442f42..5c8eff93 100644 --- a/jupyterhub/tests/test_orm.py +++ b/jupyterhub/tests/test_orm.py @@ -205,8 +205,7 @@ def test_token_find(db): assert found is None -@pytest.mark.gen_test -def test_spawn_fails(db): +async def test_spawn_fails(db): orm_user = orm.User(name='aeofel') db.add(orm_user) db.commit() @@ -223,7 +222,7 @@ def test_spawn_fails(db): }) with pytest.raises(RuntimeError) as exc: - yield user.spawn() + await user.spawn() assert user.spawners[''].server is None assert not user.running @@ -246,8 +245,7 @@ def test_groups(db): assert group.users == [] -@pytest.mark.gen_test -def test_auth_state(db): +async def test_auth_state(db): orm_user = orm.User(name='eve') db.add(orm_user) db.commit() @@ -262,51 +260,51 @@ def test_auth_state(db): state = {'key': 'value'} ck.keys = [] with pytest.raises(crypto.EncryptionUnavailable): - yield user.save_auth_state(state) + await user.save_auth_state(state) assert user.encrypted_auth_state is None # saving/loading None doesn't require keys - yield user.save_auth_state(None) - current = yield user.get_auth_state() + await user.save_auth_state(None) + current = await user.get_auth_state() assert current is None first_key = os.urandom(32) second_key = os.urandom(32) ck.keys = [first_key] - yield user.save_auth_state(state) + await user.save_auth_state(state) assert user.encrypted_auth_state is not None - decrypted_state = yield user.get_auth_state() + decrypted_state = await user.get_auth_state() assert decrypted_state == state # can't read auth_state without keys ck.keys = [] - auth_state = yield user.get_auth_state() + auth_state = await user.get_auth_state() assert auth_state is None # key rotation works db.rollback() ck.keys = [second_key, first_key] - decrypted_state = yield user.get_auth_state() + decrypted_state = await user.get_auth_state() assert decrypted_state == state new_state = {'key': 'newvalue'} - yield user.save_auth_state(new_state) + await user.save_auth_state(new_state) db.commit() ck.keys = [first_key] db.rollback() # can't read anymore with new-key after encrypting with second-key - decrypted_state = yield user.get_auth_state() + decrypted_state = await user.get_auth_state() assert decrypted_state is None - yield user.save_auth_state(new_state) - decrypted_state = yield user.get_auth_state() + await user.save_auth_state(new_state) + decrypted_state = await user.get_auth_state() assert decrypted_state == new_state ck.keys = [] db.rollback() - decrypted_state = yield user.get_auth_state() + decrypted_state = await user.get_auth_state() assert decrypted_state is None diff --git a/jupyterhub/tests/test_pages.py b/jupyterhub/tests/test_pages.py index 6c65a63e..cd810f6c 100644 --- a/jupyterhub/tests/test_pages.py +++ b/jupyterhub/tests/test_pages.py @@ -30,91 +30,81 @@ def get_page(path, app, hub=True, **kw): return async_requests.get(ujoin(base_url, path), **kw) -@pytest.mark.gen_test -def test_root_no_auth(app): +async def test_root_no_auth(app): url = ujoin(public_host(app), app.hub.base_url) - r = yield async_requests.get(url) + r = await async_requests.get(url) r.raise_for_status() assert r.url == ujoin(url, 'login') -@pytest.mark.gen_test -def test_root_auth(app): - cookies = yield app.login_user('river') - r = yield async_requests.get(public_url(app), cookies=cookies) +async def test_root_auth(app): + cookies = await app.login_user('river') + r = await async_requests.get(public_url(app), cookies=cookies) r.raise_for_status() assert r.url.startswith(public_url(app, app.users['river'])) -@pytest.mark.gen_test -def test_root_redirect(app): +async def test_root_redirect(app): name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) next_url = ujoin(app.base_url, 'user/other/test.ipynb') url = '/?' + urlencode({'next': next_url}) - r = yield get_page(url, app, cookies=cookies) + r = await get_page(url, app, cookies=cookies) r.raise_for_status() path = urlparse(r.url).path assert path == ujoin(app.base_url, 'user/%s/test.ipynb' % name) -@pytest.mark.gen_test -def test_root_default_url_noauth(app): +async def test_root_default_url_noauth(app): with mock.patch.dict(app.tornado_settings, {'default_url': '/foo/bar'}): - r = yield get_page('/', app, allow_redirects=False) + r = await get_page('/', app, allow_redirects=False) r.raise_for_status() url = r.headers.get('Location', '') path = urlparse(url).path assert path == '/foo/bar' -@pytest.mark.gen_test -def test_root_default_url_auth(app): +async def test_root_default_url_auth(app): name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) with mock.patch.dict(app.tornado_settings, {'default_url': '/foo/bar'}): - r = yield get_page('/', app, cookies=cookies, allow_redirects=False) + r = await get_page('/', app, cookies=cookies, allow_redirects=False) r.raise_for_status() url = r.headers.get('Location', '') path = urlparse(url).path assert path == '/foo/bar' -@pytest.mark.gen_test -def test_home_no_auth(app): - r = yield get_page('home', app, allow_redirects=False) +async def test_home_no_auth(app): + r = await get_page('home', app, allow_redirects=False) r.raise_for_status() assert r.status_code == 302 assert '/hub/login' in r.headers['Location'] -@pytest.mark.gen_test -def test_home_auth(app): - cookies = yield app.login_user('river') - r = yield get_page('home', app, cookies=cookies) +async def test_home_auth(app): + cookies = await app.login_user('river') + r = await get_page('home', app, cookies=cookies) r.raise_for_status() assert r.url.endswith('home') -@pytest.mark.gen_test -def test_admin_no_auth(app): - r = yield get_page('admin', app) +async def test_admin_no_auth(app): + r = await get_page('admin', app) assert r.status_code == 403 -@pytest.mark.gen_test -def test_admin_not_admin(app): - cookies = yield app.login_user('wash') - r = yield get_page('admin', app, cookies=cookies) +async def test_admin_not_admin(app): + cookies = await app.login_user('wash') + r = await get_page('admin', app, cookies=cookies) assert r.status_code == 403 -@pytest.mark.gen_test -def test_admin(app): - cookies = yield app.login_user('admin') - r = yield get_page('admin', app, cookies=cookies, allow_redirects=False) +async def test_admin(app): + cookies = await app.login_user('admin') + r = await get_page('admin', app, cookies=cookies, allow_redirects=False) r.raise_for_status() assert r.url.endswith('/admin') @@ -125,127 +115,120 @@ def test_admin(app): 'admin', 'name', ]) -@pytest.mark.gen_test -def test_admin_sort(app, sort): - cookies = yield app.login_user('admin') - r = yield get_page('admin?sort=%s' % sort, app, cookies=cookies) +async def test_admin_sort(app, sort): + cookies = await app.login_user('admin') + r = await get_page('admin?sort=%s' % sort, app, cookies=cookies) r.raise_for_status() assert r.status_code == 200 -@pytest.mark.gen_test -def test_spawn_redirect(app): +async def test_spawn_redirect(app): name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) u = app.users[orm.User.find(app.db, name)] - status = yield u.spawner.poll() + status = await u.spawner.poll() assert status is not None # test spawn page when no server is running - r = yield get_page('spawn', app, cookies=cookies) + r = await get_page('spawn', app, cookies=cookies) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, 'user/%s/' % name) # should have started server - status = yield u.spawner.poll() + status = await u.spawner.poll() assert status is None # test spawn page when server is already running (just redirect) - r = yield get_page('spawn', app, cookies=cookies) + r = await get_page('spawn', app, cookies=cookies) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/' % name) # stop server to ensure /user/name is handled by the Hub - r = yield api_request(app, 'users', name, 'server', method='delete', cookies=cookies) + r = await api_request(app, 'users', name, 'server', method='delete', cookies=cookies) r.raise_for_status() # test handing of trailing slash on `/user/name` - r = yield get_page('user/' + name, app, hub=False, cookies=cookies) + r = await get_page('user/' + name, app, hub=False, cookies=cookies) r.raise_for_status() path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/' % name) -@pytest.mark.gen_test -def test_spawn_handler_access(app): +async def test_spawn_handler_access(app): name = 'winston' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) u = app.users[orm.User.find(app.db, name)] - status = yield u.spawner.poll() + status = await u.spawner.poll() assert status is not None # spawn server via browser link with ?arg=value - r = yield get_page('spawn', app, cookies=cookies, params={'arg': 'value'}) + r = await get_page('spawn', app, cookies=cookies, params={'arg': 'value'}) r.raise_for_status() # verify that request params got passed down # implemented in MockSpawner - r = yield async_requests.get(ujoin(public_url(app, u), 'env')) + r = await async_requests.get(ujoin(public_url(app, u), 'env')) env = r.json() assert 'HANDLER_ARGS' in env assert env['HANDLER_ARGS'] == 'arg=value' # stop server - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() -@pytest.mark.gen_test -def test_spawn_admin_access(app, admin_access): +async def test_spawn_admin_access(app, admin_access): """GET /user/:name as admin with admin-access spawns user's server""" - cookies = yield app.login_user('admin') + cookies = await app.login_user('admin') name = 'mariel' user = add_user(app.db, app=app, name=name) app.db.commit() - r = yield get_page('user/' + name, app, cookies=cookies) + r = await get_page('user/' + name, app, cookies=cookies) r.raise_for_status() assert (r.url.split('?')[0] + '/').startswith(public_url(app, user)) - r = yield get_page('user/{}/env'.format(name), app, hub=False, cookies=cookies) + r = await get_page('user/{}/env'.format(name), app, hub=False, cookies=cookies) r.raise_for_status() env = r.json() assert env['JUPYTERHUB_USER'] == name -@pytest.mark.gen_test -def test_spawn_page(app): +async def test_spawn_page(app): with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): - cookies = yield app.login_user('jones') - r = yield get_page('spawn', app, cookies=cookies) + cookies = await app.login_user('jones') + r = await get_page('spawn', app, cookies=cookies) assert r.url.endswith('/spawn') assert FormSpawner.options_form in r.text - r = yield get_page('spawn?next=foo', app, cookies=cookies) + r = await get_page('spawn?next=foo', app, cookies=cookies) assert r.url.endswith('/spawn?next=foo') assert FormSpawner.options_form in r.text -@pytest.mark.gen_test -def test_spawn_page_admin(app, admin_access): +async def test_spawn_page_admin(app, admin_access): with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): - cookies = yield app.login_user('admin') + cookies = await app.login_user('admin') u = add_user(app.db, app=app, name='melanie') - r = yield get_page('spawn/' + u.name, app, cookies=cookies) + r = await get_page('spawn/' + u.name, app, cookies=cookies) assert r.url.endswith('/spawn/' + u.name) assert FormSpawner.options_form in r.text assert "Spawning server for {}".format(u.name) in r.text -@pytest.mark.gen_test -def test_spawn_form(app): +async def test_spawn_form(app): with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): base_url = ujoin(public_host(app), app.hub.base_url) - cookies = yield app.login_user('jones') + cookies = await app.login_user('jones') orm_u = orm.User.find(app.db, 'jones') u = app.users[orm_u] - yield u.stop() + await u.stop() next_url = ujoin(app.base_url, 'user/jones/tree') - r = yield async_requests.post( + r = await async_requests.post( url_concat(ujoin(base_url, 'spawn'), {'next': next_url}), cookies=cookies, data={'bounds': ['-1', '1'], 'energy': '511keV'}, @@ -259,15 +242,14 @@ def test_spawn_form(app): } -@pytest.mark.gen_test -def test_spawn_form_admin_access(app, admin_access): +async def test_spawn_form_admin_access(app, admin_access): with mock.patch.dict(app.tornado_settings, {'spawner_class': FormSpawner}): base_url = ujoin(public_host(app), app.hub.base_url) - cookies = yield app.login_user('admin') + cookies = await app.login_user('admin') u = add_user(app.db, app=app, name='martha') next_url = ujoin(app.base_url, 'user', u.name, 'tree') - r = yield async_requests.post( + r = await async_requests.post( url_concat(ujoin(base_url, 'spawn', u.name), {'next': next_url}), cookies=cookies, data={'bounds': ['-3', '3'], 'energy': '938MeV'}, @@ -282,16 +264,15 @@ def test_spawn_form_admin_access(app, admin_access): } -@pytest.mark.gen_test -def test_spawn_form_with_file(app): +async def test_spawn_form_with_file(app): with mock.patch.dict(app.tornado_settings, {'spawner_class': FormSpawner}): base_url = ujoin(public_host(app), app.hub.base_url) - cookies = yield app.login_user('jones') + cookies = await app.login_user('jones') orm_u = orm.User.find(app.db, 'jones') u = app.users[orm_u] - yield u.stop() + await u.stop() - r = yield async_requests.post(ujoin(base_url, 'spawn'), + r = await async_requests.post(ujoin(base_url, 'spawn'), cookies=cookies, data={ 'bounds': ['-1', '1'], @@ -310,12 +291,11 @@ def test_spawn_form_with_file(app): } -@pytest.mark.gen_test -def test_user_redirect(app): +async def test_user_redirect(app): name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) - r = yield get_page('/user-redirect/tree/top/', app) + r = await get_page('/user-redirect/tree/top/', app) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path @@ -325,32 +305,31 @@ def test_user_redirect(app): 'next': ujoin(app.hub.base_url, '/user-redirect/tree/top/') }) - r = yield get_page('/user-redirect/notebooks/test.ipynb', app, cookies=cookies) + r = await get_page('/user-redirect/notebooks/test.ipynb', app, cookies=cookies) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/notebooks/test.ipynb' % name) -@pytest.mark.gen_test -def test_user_redirect_deprecated(app): +async def test_user_redirect_deprecated(app): """redirecting from /user/someonelse/ URLs (deprecated)""" name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) - r = yield get_page('/user/baduser', app, cookies=cookies, hub=False) + r = await get_page('/user/baduser', app, cookies=cookies, hub=False) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/' % name) - r = yield get_page('/user/baduser/test.ipynb', app, cookies=cookies, hub=False) + r = await get_page('/user/baduser/test.ipynb', app, cookies=cookies, hub=False) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/test.ipynb' % name) - r = yield get_page('/user/baduser/test.ipynb', app, hub=False) + r = await get_page('/user/baduser/test.ipynb', app, hub=False) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path @@ -361,11 +340,10 @@ def test_user_redirect_deprecated(app): }) -@pytest.mark.gen_test -def test_login_fail(app): +async def test_login_fail(app): name = 'wash' base_url = public_url(app) - r = yield async_requests.post(base_url + 'hub/login', + r = await async_requests.post(base_url + 'hub/login', data={ 'username': name, 'password': 'wrong', @@ -375,8 +353,7 @@ def test_login_fail(app): assert not r.cookies -@pytest.mark.gen_test -def test_login_strip(app): +async def test_login_strip(app): """Test that login form doesn't strip whitespace from passwords""" form_data = { 'username': 'spiff', @@ -389,7 +366,7 @@ def test_login_strip(app): called_with.append(data) with mock.patch.object(app.authenticator, 'authenticate', mock_authenticate): - yield async_requests.post(base_url + 'hub/login', + await async_requests.post(base_url + 'hub/login', data=form_data, allow_redirects=False, ) @@ -415,9 +392,8 @@ def test_login_strip(app): (False, '//other.domain', ''), ] ) -@pytest.mark.gen_test -def test_login_redirect(app, running, next_url, location): - cookies = yield app.login_user('river') +async def test_login_redirect(app, running, next_url, location): + cookies = await app.login_user('river') user = app.users['river'] if location: location = ujoin(app.base_url, location) @@ -433,18 +409,17 @@ def test_login_redirect(app, running, next_url, location): if running and not user.active: # ensure running - yield user.spawn() + await user.spawn() elif user.active and not running: # ensure not running - yield user.stop() - r = yield get_page(url, app, cookies=cookies, allow_redirects=False) + await user.stop() + r = await get_page(url, app, cookies=cookies, allow_redirects=False) r.raise_for_status() assert r.status_code == 302 assert location == r.headers['Location'] -@pytest.mark.gen_test -def test_auto_login(app, request): +async def test_auto_login(app, request): class DummyLoginHandler(BaseHandler): def get(self): self.write('ok!') @@ -453,7 +428,7 @@ def test_auto_login(app, request): (ujoin(app.hub.base_url, 'dummy'), DummyLoginHandler), ]) # no auto_login: end up at /hub/login - r = yield async_requests.get(base_url) + r = await async_requests.get(base_url) assert r.url == public_url(app, path='hub/login') # enable auto_login: redirect from /hub/login to /hub/dummy authenticator = Authenticator(auto_login=True) @@ -462,28 +437,28 @@ def test_auto_login(app, request): with mock.patch.dict(app.tornado_settings, { 'authenticator': authenticator, }): - r = yield async_requests.get(base_url) + r = await async_requests.get(base_url) assert r.url == public_url(app, path='hub/dummy') -@pytest.mark.gen_test -def test_auto_login_logout(app): + +async def test_auto_login_logout(app): name = 'burnham' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) with mock.patch.dict(app.tornado_settings, { 'authenticator': Authenticator(auto_login=True), }): - r = yield async_requests.get(public_host(app) + app.tornado_settings['logout_url'], cookies=cookies) + r = await async_requests.get(public_host(app) + app.tornado_settings['logout_url'], cookies=cookies) r.raise_for_status() logout_url = public_host(app) + app.tornado_settings['logout_url'] assert r.url == logout_url assert r.cookies == {} -@pytest.mark.gen_test -def test_logout(app): + +async def test_logout(app): name = 'wash' - cookies = yield app.login_user(name) - r = yield async_requests.get(public_host(app) + app.tornado_settings['logout_url'], cookies=cookies) + cookies = await app.login_user(name) + r = await async_requests.get(public_host(app) + app.tornado_settings['logout_url'], cookies=cookies) r.raise_for_status() login_url = public_host(app) + app.tornado_settings['login_url'] assert r.url == login_url @@ -491,7 +466,6 @@ def test_logout(app): @pytest.mark.parametrize('shutdown_on_logout', [True, False]) -@pytest.mark.gen_test async def test_shutdown_on_logout(app, shutdown_on_logout): name = 'shutitdown' cookies = await app.login_user(name) @@ -535,50 +509,46 @@ async def test_shutdown_on_logout(app, shutdown_on_logout): assert spawner.ready == (not shutdown_on_logout) -@pytest.mark.gen_test -def test_login_no_whitelist_adds_user(app): +async def test_login_no_whitelist_adds_user(app): auth = app.authenticator mock_add_user = mock.Mock() with mock.patch.object(auth, 'add_user', mock_add_user): - cookies = yield app.login_user('jubal') + cookies = await app.login_user('jubal') user = app.users['jubal'] assert mock_add_user.mock_calls == [mock.call(user)] -@pytest.mark.gen_test -def test_static_files(app): +async def test_static_files(app): base_url = ujoin(public_host(app), app.hub.base_url) - r = yield async_requests.get(ujoin(base_url, 'logo')) + r = await async_requests.get(ujoin(base_url, 'logo')) r.raise_for_status() assert r.headers['content-type'] == 'image/png' - r = yield async_requests.get(ujoin(base_url, 'static', 'images', 'jupyter.png')) + r = await async_requests.get(ujoin(base_url, 'static', 'images', 'jupyter.png')) r.raise_for_status() assert r.headers['content-type'] == 'image/png' - r = yield async_requests.get(ujoin(base_url, 'static', 'css', 'style.min.css')) + r = await async_requests.get(ujoin(base_url, 'static', 'css', 'style.min.css')) r.raise_for_status() assert r.headers['content-type'] == 'text/css' -@pytest.mark.gen_test -def test_token_auth(app): - cookies = yield app.login_user('token') - r = yield get_page('token', app, cookies=cookies) +async def test_token_auth(app): + cookies = await app.login_user('token') + r = await get_page('token', app, cookies=cookies) r.raise_for_status() assert r.status_code == 200 -@pytest.mark.gen_test -def test_oauth_token_page(app): +async def test_oauth_token_page(app): name = 'token' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) user = app.users[orm.User.find(app.db, name)] client = orm.OAuthClient(identifier='token') app.db.add(client) oauth_token = orm.OAuthAccessToken(client=client, user=user, grant_type=orm.GrantType.authorization_code) app.db.add(oauth_token) app.db.commit() - r = yield get_page('token', app, cookies=cookies) + r = await get_page('token', app, cookies=cookies) r.raise_for_status() assert r.status_code == 200 @@ -587,14 +557,11 @@ def test_oauth_token_page(app): 503, 404, ]) - -@pytest.mark.gen_test -def test_proxy_error(app, error_status): - r = yield get_page('/error/%i' % error_status, app) +async def test_proxy_error(app, error_status): + r = await get_page('/error/%i' % error_status, app) assert r.status_code == 200 -@pytest.mark.gen_test @pytest.mark.parametrize( "announcements", [ @@ -604,7 +571,7 @@ def test_proxy_error(app, error_status): "login,logout", ] ) -def test_announcements(app, announcements): +async def test_announcements(app, announcements): """Test announcements on various pages""" # Default announcement - same on all pages ann01 = "ANNOUNCE01" @@ -620,26 +587,26 @@ def test_announcements(app, announcements): else: assert ann01 in text - cookies = yield app.login_user("jones") + cookies = await app.login_user("jones") with mock.patch.dict( app.tornado_settings, {"template_vars": template_vars, "spawner_class": FormSpawner}, ): - r = yield get_page("login", app) + r = await get_page("login", app) r.raise_for_status() assert_announcement("login", r.text) - r = yield get_page("spawn", app, cookies=cookies) + r = await get_page("spawn", app, cookies=cookies) r.raise_for_status() assert_announcement("spawn", r.text) - r = yield get_page("home", app, cookies=cookies) # hub/home + r = await get_page("home", app, cookies=cookies) # hub/home r.raise_for_status() assert_announcement("home", r.text) # need auto_login=True to get logout page auto_login = app.authenticator.auto_login app.authenticator.auto_login = True try: - r = yield get_page("logout", app, cookies=cookies) + r = await get_page("logout", app, cookies=cookies) finally: app.authenticator.auto_login = auto_login r.raise_for_status() @@ -654,18 +621,16 @@ def test_announcements(app, announcements): "redirect_uri=ok&client_id=nosuchthing", ] ) -@pytest.mark.gen_test -def test_bad_oauth_get(app, params): - cookies = yield app.login_user("authorizer") - r = yield get_page("hub/api/oauth2/authorize?" + params, app, hub=False, cookies=cookies) +async def test_bad_oauth_get(app, params): + cookies = await app.login_user("authorizer") + r = await get_page("hub/api/oauth2/authorize?" + params, app, hub=False, cookies=cookies) assert r.status_code == 400 -@pytest.mark.gen_test -def test_token_page(app): +async def test_token_page(app): name = "cake" - cookies = yield app.login_user(name) - r = yield get_page("token", app, cookies=cookies) + cookies = await app.login_user(name) + r = await get_page("token", app, cookies=cookies) r.raise_for_status() assert urlparse(r.url).path.endswith('/hub/token') def extract_body(r): @@ -684,7 +649,7 @@ def test_token_page(app): token = user.new_api_token(expires_in=60, note="my-test-token") app.db.commit() - r = yield get_page("token", app, cookies=cookies) + r = await get_page("token", app, cookies=cookies) r.raise_for_status() body = extract_body(r) assert "API Tokens" in body, body @@ -695,10 +660,10 @@ def test_token_page(app): # spawn the user to trigger oauth, etc. # request an oauth token user.spawner.cmd = [sys.executable, '-m', 'jupyterhub.singleuser'] - r = yield get_page("spawn", app, cookies=cookies) + r = await get_page("spawn", app, cookies=cookies) r.raise_for_status() - r = yield get_page("token", app, cookies=cookies) + r = await get_page("token", app, cookies=cookies) r.raise_for_status() body = extract_body(r) assert "API Tokens" in body, body @@ -706,31 +671,27 @@ def test_token_page(app): assert "Authorized Applications" in body, body -@pytest.mark.gen_test -def test_server_not_running_api_request(app): - cookies = yield app.login_user("bees") - r = yield get_page("user/bees/api/status", app, hub=False, cookies=cookies) +async def test_server_not_running_api_request(app): + cookies = await app.login_user("bees") + r = await get_page("user/bees/api/status", app, hub=False, cookies=cookies) assert r.status_code == 404 assert r.headers["content-type"] == "application/json" assert r.json() == {"message": "bees is not running"} -@pytest.mark.gen_test -def test_metrics_no_auth(app): - r = yield get_page("metrics", app) +async def test_metrics_no_auth(app): + r = await get_page("metrics", app) assert r.status_code == 403 -@pytest.mark.gen_test -def test_metrics_auth(app): - cookies = yield app.login_user('river') +async def test_metrics_auth(app): + cookies = await app.login_user('river') metrics_url = ujoin(public_host(app), app.hub.base_url, 'metrics') - r = yield get_page("metrics", app, cookies=cookies) + r = await get_page("metrics", app, cookies=cookies) assert r.status_code == 200 assert r.url == metrics_url -@pytest.mark.gen_test -def test_health_check_request(app): - r = yield get_page('health', app) +async def test_health_check_request(app): + r = await get_page('health', app) assert r.status_code == 200 diff --git a/jupyterhub/tests/test_proxy.py b/jupyterhub/tests/test_proxy.py index 1e4816b7..dd493a86 100644 --- a/jupyterhub/tests/test_proxy.py +++ b/jupyterhub/tests/test_proxy.py @@ -16,6 +16,7 @@ from .mocking import MockHub from .test_api import api_request, add_user from ..utils import wait_for_http_server, url_path_join as ujoin + @pytest.fixture def disable_check_routes(app): # disable periodic check_routes while we are testing @@ -25,9 +26,8 @@ def disable_check_routes(app): finally: app.last_activity_callback.start() -@pytest.mark.gen_test -def test_external_proxy(request): +async def test_external_proxy(request): auth_token = 'secret!' proxy_ip = '127.0.0.1' proxy_port = 54321 @@ -71,23 +71,23 @@ def test_external_proxy(request): def wait_for_proxy(): return wait_for_http_server('http://%s:%i' % (proxy_ip, proxy_port)) - yield wait_for_proxy() + await wait_for_proxy() - yield app.initialize([]) - yield app.start() + await app.initialize([]) + await app.start() assert app.proxy.proxy_process is None # test if api service has a root route '/' - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() assert list(routes.keys()) == [app.hub.routespec] - + # add user to the db and start a single user server name = 'river' add_user(app.db, app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - - routes = yield app.proxy.get_all_routes() + + routes = await app.proxy.get_all_routes() # sets the desired path result user_path = ujoin(app.base_url, 'user/river') + '/' print(app.base_url, user_path) @@ -101,18 +101,18 @@ def test_external_proxy(request): proxy.terminate() proxy.wait(timeout=10) proxy = Popen(cmd, env=env) - yield wait_for_proxy() + await wait_for_proxy() - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() assert list(routes.keys()) == [] # poke the server to update the proxy - r = yield api_request(app, 'proxy', method='post') + r = await api_request(app, 'proxy', method='post') r.raise_for_status() # check that the routes are correct - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() assert sorted(routes.keys()) == [app.hub.routespec, user_spec] # teardown the proxy, and start a new one with different auth and port @@ -131,11 +131,11 @@ def test_external_proxy(request): if app.subdomain_host: cmd.append('--host-routing') proxy = Popen(cmd, env=env) - yield wait_for_proxy() + await wait_for_proxy() # tell the hub where the new proxy is new_api_url = 'http://{}:{}'.format(proxy_ip, proxy_port) - r = yield api_request(app, 'proxy', method='patch', data=json.dumps({ + r = await api_request(app, 'proxy', method='patch', data=json.dumps({ 'api_url': new_api_url, 'auth_token': new_auth_token, })) @@ -145,11 +145,10 @@ def test_external_proxy(request): assert app.proxy.auth_token == new_auth_token # check that the routes are correct - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() assert sorted(routes.keys()) == [app.hub.routespec, user_spec] -@pytest.mark.gen_test @pytest.mark.parametrize("username", [ 'zoe', '50fia', @@ -157,27 +156,27 @@ def test_external_proxy(request): '~TestJH', 'has@', ]) -def test_check_routes(app, username, disable_check_routes): +async def test_check_routes(app, username, disable_check_routes): proxy = app.proxy test_user = add_user(app.db, app, name=username) - r = yield api_request(app, 'users/%s/server' % username, method='post') + r = await api_request(app, 'users/%s/server' % username, method='post') r.raise_for_status() # check a valid route exists for user - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() before = sorted(routes) assert test_user.proxy_spec in before # check if a route is removed when user deleted - yield app.proxy.check_routes(app.users, app._service_map) - yield proxy.delete_user(test_user) - routes = yield app.proxy.get_all_routes() + await app.proxy.check_routes(app.users, app._service_map) + await proxy.delete_user(test_user) + routes = await app.proxy.get_all_routes() during = sorted(routes) assert test_user.proxy_spec not in during # check if a route exists for user - yield app.proxy.check_routes(app.users, app._service_map) - routes = yield app.proxy.get_all_routes() + await app.proxy.check_routes(app.users, app._service_map) + routes = await app.proxy.get_all_routes() after = sorted(routes) assert test_user.proxy_spec in after @@ -185,7 +184,6 @@ def test_check_routes(app, username, disable_check_routes): assert before == after -@pytest.mark.gen_test @pytest.mark.parametrize("routespec", [ '/has%20space/foo/', '/missing-trailing/slash', @@ -194,11 +192,11 @@ def test_check_routes(app, username, disable_check_routes): 'host.name/path/', 'other.host/path/no/slash', ]) -def test_add_get_delete(app, routespec, disable_check_routes): +async def test_add_get_delete(app, routespec, disable_check_routes): arg = routespec if not routespec.endswith('/'): routespec = routespec + '/' - + # host-routes when not host-routing raises an error # and vice versa expect_value_error = bool(app.subdomain_host) ^ (not routespec.startswith('/')) @@ -213,26 +211,25 @@ def test_add_get_delete(app, routespec, disable_check_routes): proxy = app.proxy target = 'https://localhost:1234' with context(): - yield proxy.add_route(arg, target, {}) - routes = yield proxy.get_all_routes() + await proxy.add_route(arg, target, {}) + routes = await proxy.get_all_routes() if not expect_value_error: assert routespec in routes.keys() with context(): - route = yield proxy.get_route(arg) + route = await proxy.get_route(arg) assert route == { 'target': target, 'routespec': routespec, 'data': route.get('data'), } with context(): - yield proxy.delete_route(arg) + await proxy.delete_route(arg) with context(): - route = yield proxy.get_route(arg) + route = await proxy.get_route(arg) assert route is None -@pytest.mark.gen_test @pytest.mark.parametrize("test_data", [None, 'notjson', json.dumps([])]) -def test_proxy_patch_bad_request_data(app, test_data): - r = yield api_request(app, 'proxy', method='patch', data=test_data) +async def test_proxy_patch_bad_request_data(app, test_data): + r = await api_request(app, 'proxy', method='patch', data=test_data) assert r.status_code == 400 diff --git a/jupyterhub/tests/test_services.py b/jupyterhub/tests/test_services.py index 78069078..428e69ea 100644 --- a/jupyterhub/tests/test_services.py +++ b/jupyterhub/tests/test_services.py @@ -1,5 +1,6 @@ """Tests for services""" +import asyncio from binascii import hexlify from contextlib import contextmanager import os @@ -8,13 +9,14 @@ import sys from threading import Event import time +from async_generator import asynccontextmanager, async_generator, yield_ import pytest import requests from tornado import gen from tornado.ioloop import IOLoop from .mocking import public_url -from ..utils import url_path_join, wait_for_http_server, random_port +from ..utils import url_path_join, wait_for_http_server, random_port, maybe_future from .utils import async_requests mockservice_path = os.path.dirname(os.path.abspath(__file__)) @@ -22,8 +24,9 @@ mockservice_py = os.path.join(mockservice_path, 'mockservice.py') mockservice_cmd = [sys.executable, mockservice_py] -@contextmanager -def external_service(app, name='mockservice'): +@asynccontextmanager +@async_generator +async def external_service(app, name='mockservice'): env = { 'JUPYTERHUB_API_TOKEN': hexlify(os.urandom(5)), 'JUPYTERHUB_SERVICE_NAME': name, @@ -31,17 +34,14 @@ def external_service(app, name='mockservice'): 'JUPYTERHUB_SERVICE_URL': 'http://127.0.0.1:%i' % random_port(), } proc = Popen(mockservice_cmd, env=env) - IOLoop().run_sync( - lambda: wait_for_http_server(env['JUPYTERHUB_SERVICE_URL']) - ) try: - yield env + await wait_for_http_server(env['JUPYTERHUB_SERVICE_URL']) + await yield_(env) finally: proc.terminate() -@pytest.mark.gen_test -def test_managed_service(mockservice): +async def test_managed_service(mockservice): service = mockservice proc = service.proc assert isinstance(proc.pid, object) @@ -58,19 +58,18 @@ def test_managed_service(mockservice): if service.proc is not proc: break else: - yield gen.sleep(0.2) - + await asyncio.sleep(0.2) + assert service.proc.pid != first_pid assert service.proc.poll() is None -@pytest.mark.gen_test -def test_proxy_service(app, mockservice_url): +async def test_proxy_service(app, mockservice_url): service = mockservice_url name = service.name - yield app.proxy.get_all_routes() + await app.proxy.get_all_routes() url = public_url(app, service) + '/foo' - r = yield async_requests.get(url, allow_redirects=False) + r = await async_requests.get(url, allow_redirects=False) path = '/services/{}/foo'.format(name) r.raise_for_status() @@ -78,23 +77,22 @@ def test_proxy_service(app, mockservice_url): assert r.text.endswith(path) -@pytest.mark.gen_test -def test_external_service(app): +async def test_external_service(app): name = 'external' - with external_service(app, name=name) as env: + async with external_service(app, name=name) as env: app.services = [{ 'name': name, 'admin': True, 'url': env['JUPYTERHUB_SERVICE_URL'], 'api_token': env['JUPYTERHUB_API_TOKEN'], }] - yield app.init_services() - yield app.init_api_tokens() - yield app.proxy.add_all_services(app._service_map) + await maybe_future(app.init_services()) + await app.init_api_tokens() + await app.proxy.add_all_services(app._service_map) service = app._service_map[name] url = public_url(app, service) + '/api/users' - r = yield async_requests.get(url, allow_redirects=False) + r = await async_requests.get(url, allow_redirects=False) r.raise_for_status() assert r.status_code == 200 resp = r.json() diff --git a/jupyterhub/tests/test_services_auth.py b/jupyterhub/tests/test_services_auth.py index 64326fa2..30fbec0d 100644 --- a/jupyterhub/tests/test_services_auth.py +++ b/jupyterhub/tests/test_services_auth.py @@ -227,11 +227,10 @@ def test_hub_authenticated(request): assert r.status_code == 403 -@pytest.mark.gen_test -def test_hubauth_cookie(app, mockservice_url): +async def test_hubauth_cookie(app, mockservice_url): """Test HubAuthenticated service with user cookies""" - cookies = yield app.login_user('badger') - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/', cookies=cookies) + cookies = await app.login_user('badger') + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/', cookies=cookies) r.raise_for_status() print(r.text) reply = r.json() @@ -242,15 +241,14 @@ def test_hubauth_cookie(app, mockservice_url): } -@pytest.mark.gen_test -def test_hubauth_token(app, mockservice_url): +async def test_hubauth_token(app, mockservice_url): """Test HubAuthenticated service with user API tokens""" u = add_user(app.db, name='river') token = u.new_api_token() app.db.commit() # token in Authorization header - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/', + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/', headers={ 'Authorization': 'token %s' % token, }) @@ -262,7 +260,7 @@ def test_hubauth_token(app, mockservice_url): } # token in ?token parameter - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=%s' % token) + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=%s' % token) r.raise_for_status() reply = r.json() sub_reply = { key: reply.get(key, 'missing') for key in ['name', 'admin']} @@ -271,7 +269,7 @@ def test_hubauth_token(app, mockservice_url): 'admin': False, } - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=no-such-token', + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=no-such-token', allow_redirects=False, ) assert r.status_code == 302 @@ -281,17 +279,16 @@ def test_hubauth_token(app, mockservice_url): assert path.endswith('/hub/login') -@pytest.mark.gen_test -def test_hubauth_service_token(app, mockservice_url): +async def test_hubauth_service_token(app, mockservice_url): """Test HubAuthenticated service with service API tokens""" - + token = hexlify(os.urandom(5)).decode('utf8') name = 'test-api-service' app.service_tokens[token] = name - yield app.init_api_tokens() + await app.init_api_tokens() # token in Authorization header - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/', + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/', headers={ 'Authorization': 'token %s' % token, }) @@ -305,7 +302,7 @@ def test_hubauth_service_token(app, mockservice_url): assert not r.cookies # token in ?token parameter - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=%s' % token) + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=%s' % token) r.raise_for_status() reply = r.json() assert reply == { @@ -314,7 +311,7 @@ def test_hubauth_service_token(app, mockservice_url): 'admin': False, } - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=no-such-token', + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=no-such-token', allow_redirects=False, ) assert r.status_code == 302 @@ -324,16 +321,15 @@ def test_hubauth_service_token(app, mockservice_url): assert path.endswith('/hub/login') -@pytest.mark.gen_test -def test_oauth_service(app, mockservice_url): +async def test_oauth_service(app, mockservice_url): service = mockservice_url url = url_path_join(public_url(app, mockservice_url) + 'owhoami/?arg=x') # first request is only going to login and get us to the oauth form page s = AsyncSession() name = 'link' - s.cookies = yield app.login_user(name) + s.cookies = await app.login_user(name) - r = yield s.get(url) + r = await s.get(url) r.raise_for_status() # we should be looking at the oauth confirmation page assert urlparse(r.url).path == app.base_url + 'hub/api/oauth2/authorize' @@ -341,7 +337,7 @@ def test_oauth_service(app, mockservice_url): assert set(r.history[0].cookies.keys()) == {'service-%s-oauth-state' % service.name} # submit the oauth form to complete authorization - r = yield s.post(r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}) + r = await s.post(r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}) r.raise_for_status() assert r.url == url # verify oauth cookie is set @@ -350,7 +346,7 @@ def test_oauth_service(app, mockservice_url): assert 'service-%s-oauth-state' % service.name not in set(s.cookies.keys()) # second request should be authenticated, which means no redirects - r = yield s.get(url, allow_redirects=False) + r = await s.get(url, allow_redirects=False) r.raise_for_status() assert r.status_code == 200 reply = r.json() @@ -363,7 +359,7 @@ def test_oauth_service(app, mockservice_url): # token-authenticated request to HubOAuth token = app.users[name].new_api_token() # token in ?token parameter - r = yield async_requests.get(url_concat(url, {'token': token})) + r = await async_requests.get(url_concat(url, {'token': token})) r.raise_for_status() reply = r.json() assert reply['name'] == name @@ -371,7 +367,7 @@ def test_oauth_service(app, mockservice_url): # verify that ?token= requests set a cookie assert len(r.cookies) != 0 # ensure cookie works in future requests - r = yield async_requests.get( + r = await async_requests.get( url, cookies=r.cookies, allow_redirects=False, @@ -382,17 +378,16 @@ def test_oauth_service(app, mockservice_url): assert reply['name'] == name -@pytest.mark.gen_test -def test_oauth_cookie_collision(app, mockservice_url): +async def test_oauth_cookie_collision(app, mockservice_url): service = mockservice_url url = url_path_join(public_url(app, mockservice_url), 'owhoami/') print(url) s = AsyncSession() name = 'mypha' - s.cookies = yield app.login_user(name) + s.cookies = await app.login_user(name) state_cookie_name = 'service-%s-oauth-state' % service.name service_cookie_name = 'service-%s' % service.name - oauth_1 = yield s.get(url) + oauth_1 = await s.get(url) print(oauth_1.headers) print(oauth_1.cookies, oauth_1.url, url) assert state_cookie_name in s.cookies @@ -402,7 +397,7 @@ def test_oauth_cookie_collision(app, mockservice_url): state_1 = s.cookies[state_cookie_name] # start second oauth login before finishing the first - oauth_2 = yield s.get(url) + oauth_2 = await s.get(url) state_cookies = [ c for c in s.cookies.keys() if c.startswith(state_cookie_name) ] assert len(state_cookies) == 2 # get the random-suffix cookie name @@ -412,7 +407,7 @@ def test_oauth_cookie_collision(app, mockservice_url): # finish oauth 2 # submit the oauth form to complete authorization - r = yield s.post( + r = await s.post( oauth_2.url, data={'scopes': ['identify']}, headers={'Referer': oauth_2.url}, @@ -426,7 +421,7 @@ def test_oauth_cookie_collision(app, mockservice_url): service_cookie_2 = s.cookies[service_cookie_name] # finish oauth 1 - r = yield s.post( + r = await s.post( oauth_1.url, data={'scopes': ['identify']}, headers={'Referer': oauth_1.url}, @@ -445,8 +440,7 @@ def test_oauth_cookie_collision(app, mockservice_url): assert state_cookies == [] -@pytest.mark.gen_test -def test_oauth_logout(app, mockservice_url): +async def test_oauth_logout(app, mockservice_url): """Verify that logout via the Hub triggers logout for oauth services 1. clears session id cookie @@ -471,18 +465,18 @@ def test_oauth_logout(app, mockservice_url): # ensure we start empty assert auth_tokens() == [] - s.cookies = yield app.login_user(name) + s.cookies = await app.login_user(name) assert 'jupyterhub-session-id' in s.cookies - r = yield s.get(url) + r = await s.get(url) r.raise_for_status() assert urlparse(r.url).path.endswith('oauth2/authorize') # submit the oauth form to complete authorization - r = yield s.post(r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}) + r = await s.post(r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}) r.raise_for_status() assert r.url == url # second request should be authenticated - r = yield s.get(url, allow_redirects=False) + r = await s.get(url, allow_redirects=False) r.raise_for_status() assert r.status_code == 200 reply = r.json() @@ -501,13 +495,13 @@ def test_oauth_logout(app, mockservice_url): assert len(auth_tokens()) == 1 # hit hub logout URL - r = yield s.get(public_url(app, path='hub/logout')) + r = await s.get(public_url(app, path='hub/logout')) r.raise_for_status() # verify that all cookies other than the service cookie are cleared assert list(s.cookies.keys()) == [service_cookie_name] # verify that clearing session id invalidates service cookie # i.e. redirect back to login page - r = yield s.get(url) + r = await s.get(url) r.raise_for_status() assert r.url.split('?')[0] == public_url(app, path='hub/login') @@ -524,7 +518,7 @@ def test_oauth_logout(app, mockservice_url): # check that we got the old session id back assert session_id == s.cookies['jupyterhub-session-id'] - r = yield s.get(url, allow_redirects=False) + r = await s.get(url, allow_redirects=False) r.raise_for_status() assert r.status_code == 200 reply = r.json() diff --git a/jupyterhub/tests/test_singleuser.py b/jupyterhub/tests/test_singleuser.py index c9ab77d6..f098cbf7 100644 --- a/jupyterhub/tests/test_singleuser.py +++ b/jupyterhub/tests/test_singleuser.py @@ -13,42 +13,41 @@ from ..utils import url_path_join from .utils import async_requests, AsyncSession -@pytest.mark.gen_test -def test_singleuser_auth(app): +async def test_singleuser_auth(app): # use StubSingleUserSpawner to launch a single-user app in a thread app.spawner_class = StubSingleUserSpawner app.tornado_settings['spawner_class'] = StubSingleUserSpawner # login, start the server - cookies = yield app.login_user('nandy') + cookies = await app.login_user('nandy') user = app.users['nandy'] if not user.running: - yield user.spawn() + await user.spawn() url = public_url(app, user) # no cookies, redirects to login page - r = yield async_requests.get(url) + r = await async_requests.get(url) r.raise_for_status() assert '/hub/login' in r.url # with cookies, login successful - r = yield async_requests.get(url, cookies=cookies) + r = await async_requests.get(url, cookies=cookies) r.raise_for_status() assert urlparse(r.url).path.rstrip('/').endswith('/user/nandy/tree') assert r.status_code == 200 # logout - r = yield async_requests.get(url_path_join(url, 'logout'), cookies=cookies) + r = await async_requests.get(url_path_join(url, 'logout'), cookies=cookies) assert len(r.cookies) == 0 # accessing another user's server hits the oauth confirmation page - cookies = yield app.login_user('burgess') + cookies = await app.login_user('burgess') s = AsyncSession() s.cookies = cookies - r = yield s.get(url) + r = await s.get(url) assert urlparse(r.url).path.endswith('/oauth2/authorize') # submit the oauth form to complete authorization - r = yield s.post( + r = await s.post( r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}, @@ -59,28 +58,27 @@ def test_singleuser_auth(app): assert 'burgess' in r.text -@pytest.mark.gen_test -def test_disable_user_config(app): +async def test_disable_user_config(app): # use StubSingleUserSpawner to launch a single-user app in a thread app.spawner_class = StubSingleUserSpawner app.tornado_settings['spawner_class'] = StubSingleUserSpawner # login, start the server - cookies = yield app.login_user('nandy') + cookies = await app.login_user('nandy') user = app.users['nandy'] # stop spawner, if running: if user.running: print("stopping") - yield user.stop() + await user.stop() # start with new config: user.spawner.debug = True user.spawner.disable_user_config = True - yield user.spawn() - yield app.proxy.add_user(user) - + await user.spawn() + await app.proxy.add_user(user) + url = public_url(app, user) - + # with cookies, login successful - r = yield async_requests.get(url, cookies=cookies) + r = await async_requests.get(url, cookies=cookies) r.raise_for_status() assert r.url.rstrip('/').endswith('/user/nandy/tree') assert r.status_code == 200 @@ -90,6 +88,7 @@ def test_help_output(): out = check_output([sys.executable, '-m', 'jupyterhub.singleuser', '--help-all']).decode('utf8', 'replace') assert 'JupyterHub' in out + def test_version(): out = check_output([sys.executable, '-m', 'jupyterhub.singleuser', '--version']).decode('utf8', 'replace') assert jupyterhub.__version__ in out diff --git a/jupyterhub/tests/test_spawner.py b/jupyterhub/tests/test_spawner.py index b6709acf..18b100ce 100644 --- a/jupyterhub/tests/test_spawner.py +++ b/jupyterhub/tests/test_spawner.py @@ -60,10 +60,9 @@ def new_spawner(db, **kwargs): return user._new_spawner('', spawner_class=LocalProcessSpawner, **kwargs) -@pytest.mark.gen_test -def test_spawner(db, request): +async def test_spawner(db, request): spawner = new_spawner(db) - ip, port = yield spawner.start() + ip, port = await spawner.start() assert ip == '127.0.0.1' assert isinstance(port, int) assert port > 0 @@ -72,115 +71,110 @@ def test_spawner(db, request): # wait for the process to get to the while True: loop time.sleep(1) - status = yield spawner.poll() + status = await spawner.poll() assert status is None - yield spawner.stop() - status = yield spawner.poll() + await spawner.stop() + status = await spawner.poll() assert status == 1 -@gen.coroutine -def wait_for_spawner(spawner, timeout=10): +async def wait_for_spawner(spawner, timeout=10): """Wait for an http server to show up - + polling at shorter intervals for early termination """ deadline = time.monotonic() + timeout def wait(): return spawner.server.wait_up(timeout=1, http=True) while time.monotonic() < deadline: - status = yield spawner.poll() + status = await spawner.poll() assert status is None try: - yield wait() + await wait() except TimeoutError: continue else: break - yield wait() + await wait() -@pytest.mark.gen_test -def test_single_user_spawner(app, request): +async def test_single_user_spawner(app, request): user = next(iter(app.users.values()), None) spawner = user.spawner spawner.cmd = ['jupyterhub-singleuser'] - yield user.spawn() + await user.spawn() assert spawner.server.ip == '127.0.0.1' assert spawner.server.port > 0 - yield wait_for_spawner(spawner) - status = yield spawner.poll() + await wait_for_spawner(spawner) + status = await spawner.poll() assert status is None - yield spawner.stop() - status = yield spawner.poll() + await spawner.stop() + status = await spawner.poll() assert status == 0 -@pytest.mark.gen_test -def test_stop_spawner_sigint_fails(db): +async def test_stop_spawner_sigint_fails(db): spawner = new_spawner(db, cmd=[sys.executable, '-c', _uninterruptible]) - yield spawner.start() + await spawner.start() # wait for the process to get to the while True: loop - yield gen.sleep(1) + await gen.sleep(1) - status = yield spawner.poll() + status = await spawner.poll() assert status is None - - yield spawner.stop() - status = yield spawner.poll() + + await spawner.stop() + status = await spawner.poll() assert status == -signal.SIGTERM -@pytest.mark.gen_test -def test_stop_spawner_stop_now(db): +async def test_stop_spawner_stop_now(db): spawner = new_spawner(db) - yield spawner.start() + await spawner.start() # wait for the process to get to the while True: loop - yield gen.sleep(1) + await gen.sleep(1) - status = yield spawner.poll() + status = await spawner.poll() assert status is None - - yield spawner.stop(now=True) - status = yield spawner.poll() + + await spawner.stop(now=True) + status = await spawner.poll() assert status == -signal.SIGTERM -@pytest.mark.gen_test -def test_spawner_poll(db): +async def test_spawner_poll(db): first_spawner = new_spawner(db) user = first_spawner.user - yield first_spawner.start() + await first_spawner.start() proc = first_spawner.proc - status = yield first_spawner.poll() + status = await first_spawner.poll() assert status is None if user.state is None: user.state = {} first_spawner.orm_spawner.state = first_spawner.get_state() assert 'pid' in first_spawner.orm_spawner.state - + # create a new Spawner, loading from state of previous spawner = new_spawner(db, user=first_spawner.user) spawner.start_polling() - + # wait for the process to get to the while True: loop - yield gen.sleep(1) - status = yield spawner.poll() + await gen.sleep(1) + status = await spawner.poll() assert status is None - + # kill the process proc.terminate() for i in range(10): if proc.poll() is None: - yield gen.sleep(1) + await gen.sleep(1) else: break assert proc.poll() is not None - yield gen.sleep(2) - status = yield spawner.poll() + await gen.sleep(2) + status = await spawner.poll() assert status is not None @@ -213,8 +207,7 @@ def test_string_formatting(db): assert s.format_string(s.default_url) == '/base/%s' % name -@pytest.mark.gen_test -def test_popen_kwargs(db): +async def test_popen_kwargs(db): mock_proc = mock.Mock(spec=Popen) def mock_popen(*args, **kwargs): mock_proc.args = args @@ -224,14 +217,13 @@ def test_popen_kwargs(db): s = new_spawner(db, popen_kwargs={'shell': True}, cmd='jupyterhub-singleuser') with mock.patch.object(spawnermod, 'Popen', mock_popen): - yield s.start() + await s.start() assert mock_proc.kwargs['shell'] == True assert mock_proc.args[0][:1] == (['jupyterhub-singleuser']) -@pytest.mark.gen_test -def test_shell_cmd(db, tmpdir, request): +async def test_shell_cmd(db, tmpdir, request): f = tmpdir.join('bashrc') f.write('export TESTVAR=foo\n') s = new_spawner(db, @@ -243,17 +235,17 @@ def test_shell_cmd(db, tmpdir, request): db.commit() s.server = Server.from_orm(server) db.commit() - (ip, port) = yield s.start() + (ip, port) = await s.start() request.addfinalizer(s.stop) s.server.ip = ip s.server.port = port db.commit() - yield wait_for_spawner(s) - r = yield async_requests.get('http://%s:%i/env' % (ip, port)) + await wait_for_spawner(s) + r = await async_requests.get('http://%s:%i/env' % (ip, port)) r.raise_for_status() env = r.json() assert env['TESTVAR'] == 'foo' - yield s.stop() + await s.stop() def test_inherit_overwrite(): @@ -277,8 +269,7 @@ def test_inherit_ok(): pass -@pytest.mark.gen_test -def test_spawner_reuse_api_token(db, app): +async def test_spawner_reuse_api_token(db, app): # setup: user with no tokens, whose spawner has set the .will_resume flag user = add_user(app.db, app, name='snoopy') spawner = user.spawner @@ -286,26 +277,25 @@ def test_spawner_reuse_api_token(db, app): # will_resume triggers reuse of tokens spawner.will_resume = True # first start: gets a new API token - yield user.spawn() + await user.spawn() api_token = spawner.api_token found = orm.APIToken.find(app.db, api_token) assert found assert found.user.name == user.name assert user.api_tokens == [found] - yield user.stop() + await user.stop() # stop now deletes unused spawners. # put back the mock spawner! user.spawners[''] = spawner # second start: should reuse the token - yield user.spawn() + await user.spawn() # verify re-use of API token assert spawner.api_token == api_token # verify that a new token was not created assert user.api_tokens == [found] -@pytest.mark.gen_test -def test_spawner_insert_api_token(app): +async def test_spawner_insert_api_token(app): """Token provided by spawner is not in the db Insert token into db as a user-provided token. @@ -322,17 +312,16 @@ def test_spawner_insert_api_token(app): # The spawner's provided API token would already be in the db # unless there is a bug somewhere else (in the Spawner), # but handle it anyway. - yield user.spawn() + await user.spawn() assert spawner.api_token == api_token found = orm.APIToken.find(app.db, api_token) assert found assert found.user.name == user.name assert user.api_tokens == [found] - yield user.stop() + await user.stop() -@pytest.mark.gen_test -def test_spawner_bad_api_token(app): +async def test_spawner_bad_api_token(app): """Tokens are revoked when a Spawner gets another user's token""" # we need two users for this one user = add_user(app.db, app, name='antimone') @@ -349,13 +338,12 @@ def test_spawner_bad_api_token(app): # starting a user's server with another user's token # should revoke it with pytest.raises(ValueError): - yield user.spawn() + await user.spawn() assert orm.APIToken.find(app.db, other_token) is None assert other_user.api_tokens == [] -@pytest.mark.gen_test -def test_spawner_delete_server(app): +async def test_spawner_delete_server(app): """Test deleting spawner.server This can occur during app startup if their server has been deleted. @@ -393,22 +381,21 @@ def test_spawner_delete_server(app): "has%40x", ] ) -@pytest.mark.gen_test -def test_spawner_routing(app, name): +async def test_spawner_routing(app, name): """Test routing of names with special characters""" db = app.db with mock.patch.dict(app.config.LocalProcessSpawner, {'cmd': [sys.executable, '-m', 'jupyterhub.tests.mocksu']}): user = add_user(app.db, app, name=name) - yield user.spawn() - yield wait_for_spawner(user.spawner) - yield app.proxy.add_user(user) + await user.spawn() + await wait_for_spawner(user.spawner) + await app.proxy.add_user(user) kwargs = {'allow_redirects': False} if app.internal_ssl: kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key) kwargs["verify"] = app.internal_ssl_ca url = url_path_join(public_url(app, user), "test/url") - r = yield async_requests.get(url, **kwargs) + r = await async_requests.get(url, **kwargs) r.raise_for_status() assert r.url == url assert r.text == urlparse(url).path - yield user.stop() + await user.stop() diff --git a/jupyterhub/tests/test_utils.py b/jupyterhub/tests/test_utils.py index 418f4795..7301b420 100644 --- a/jupyterhub/tests/test_utils.py +++ b/jupyterhub/tests/test_utils.py @@ -26,7 +26,6 @@ def schedule_future(io_loop, *, delay, result=None): return f -@pytest.mark.gen_test @pytest.mark.parametrize("deadline, n, delay, expected", [ (0, 3, 1, []), (0, 3, 0, [0, 1, 2]), @@ -43,7 +42,6 @@ async def test_iterate_until(io_loop, deadline, n, delay, expected): assert yielded == expected -@pytest.mark.gen_test async def test_iterate_until_ready_after_deadline(io_loop): f = schedule_future(io_loop, delay=0) diff --git a/jupyterhub/tests/utils.py b/jupyterhub/tests/utils.py index e2b4c1b1..a59bf87e 100644 --- a/jupyterhub/tests/utils.py +++ b/jupyterhub/tests/utils.py @@ -13,11 +13,15 @@ class _AsyncRequests: def __init__(self): self.executor = ThreadPoolExecutor(1) + real_submit = self.executor.submit + self.executor.submit = lambda *args, **kwargs: asyncio.wrap_future( + real_submit(*args, **kwargs) + ) def __getattr__(self, name): requests_method = getattr(requests, name) - return lambda *args, **kwargs: asyncio.wrap_future( - self.executor.submit(requests_method, *args, **kwargs) + return lambda *args, **kwargs: self.executor.submit( + requests_method, *args, **kwargs ) diff --git a/jupyterhub/user.py b/jupyterhub/user.py index 74003ed2..b5b80e16 100644 --- a/jupyterhub/user.py +++ b/jupyterhub/user.py @@ -531,7 +531,7 @@ class User: self.settings['statsd'].incr('spawner.failure.error') e.reason = 'error' try: - await self.stop() + await self.stop(spawner.name) except Exception: self.log.error("Failed to cleanup {user}'s server that failed to start".format( user=self.name, @@ -550,6 +550,15 @@ class User: spawner.orm_spawner.state = spawner.get_state() db.commit() spawner._waiting_for_response = True + await self._wait_up(spawner) + + async def _wait_up(self, spawner): + """Wait for a server to finish starting. + + Shuts the server down if it doesn't respond within + spawner.http_timeout. + """ + server = spawner.server key = self.settings.get('internal_ssl_key') cert = self.settings.get('internal_ssl_cert') ca = self.settings.get('internal_ssl_ca') @@ -578,7 +587,7 @@ class User: )) self.settings['statsd'].incr('spawner.failure.http_error') try: - await self.stop() + await self.stop(spawner.name) except Exception: self.log.error("Failed to cleanup {user}'s server that failed to start".format( user=self.name, @@ -594,7 +603,7 @@ class User: finally: spawner._waiting_for_response = False spawner._start_pending = False - return self + return spawner async def stop(self, server_name=''): """Stop the user's spawner