diff --git a/.travis.yml b/.travis.yml index 8c03233a..152a2f6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ before_install: fi install: - pip install --upgrade pip - - pip install --pre -r dev-requirements.txt . + - pip install --upgrade --pre -r dev-requirements.txt . - pip freeze # running tests @@ -48,7 +48,8 @@ script: # build docs if [[ "$TEST" == "docs" ]]; then pushd docs - pip install -r requirements.txt + pip install --upgrade -r requirements.txt + pip install --upgrade alabaster_jupyterhub make html popd fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 07e6ecb0..a19b07e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,5 @@ # Contributing -See the [contribution guide](https://jupyterhub.readthedocs.io/en/latest/index.html#contributor) section -at the JupyterHub documentation. \ No newline at end of file +See the [contribution guide](https://jupyterhub.readthedocs.io/en/latest/index.html#contributing) section +at the JupyterHub documentation. diff --git a/README.md b/README.md index d12e6515..e868ef9d 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,12 @@ [![PyPI](https://img.shields.io/pypi/v/jupyterhub.svg)](https://pypi.python.org/pypi/jupyterhub) [![Documentation Status](https://readthedocs.org/projects/jupyterhub/badge/?version=latest)](https://jupyterhub.readthedocs.org/en/latest/?badge=latest) -[![Documentation Status](http://readthedocs.org/projects/jupyterhub/badge/?version=0.7.2)](https://jupyterhub.readthedocs.io/en/0.7.2/?badge=0.7.2) [![Build Status](https://travis-ci.org/jupyterhub/jupyterhub.svg?branch=master)](https://travis-ci.org/jupyterhub/jupyterhub) [![Circle CI](https://circleci.com/gh/jupyterhub/jupyterhub.svg?style=shield&circle-token=b5b65862eb2617b9a8d39e79340b0a6b816da8cc)](https://circleci.com/gh/jupyterhub/jupyterhub) [![codecov.io](https://codecov.io/github/jupyterhub/jupyterhub/coverage.svg?branch=master)](https://codecov.io/github/jupyterhub/jupyterhub?branch=master) -[![Google Group](https://img.shields.io/badge/google-group-blue.svg)](https://groups.google.com/forum/#!forum/jupyter) +[![GitHub](https://img.shields.io/badge/issue_tracking-github-blue.svg)](https://github.com/jupyterhub/jupyterhub/issues) +[![Discourse](https://img.shields.io/badge/help_forum-discourse-blue.svg)](https://discourse.jupyter.org/c/jupyterhub) +[![Gitter](https://img.shields.io/badge/social_chat-gitter-blue.svg)](https://gitter.im/jupyterhub/jupyterhub) With [JupyterHub](https://jupyterhub.readthedocs.io) you can create a **multi-user Hub** which spawns, manages, and proxies multiple instances of the @@ -204,6 +205,9 @@ and the [`CONTRIBUTING.md`](CONTRIBUTING.md). The `CONTRIBUTING.md` file explains how to set up a development installation, how to run the test suite, and how to contribute to documentation. +For a high-level view of the vision and next directions of the project, see the +[JupyterHub community roadmap](docs/source/contributing/roadmap.md). + ### A note about platform support JupyterHub is supported on Linux/Unix based systems. diff --git a/dev-requirements.txt b/dev-requirements.txt index 4c90579a..0fc2169d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,7 +6,7 @@ coverage cryptography html5lib # needed for beautifulsoup pytest-cov -pytest-tornado +pytest-asyncio pytest>=3.3 notebook requests-mock diff --git a/docs/environment.yml b/docs/environment.yml index 59bfc599..dfe35e06 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -21,3 +21,4 @@ dependencies: - prometheus_client - attrs>=17.4.0 - sphinx-copybutton + - alabaster_jupyterhub diff --git a/docs/requirements.txt b/docs/requirements.txt index e008b40e..5edce880 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -4,3 +4,4 @@ sphinx>=1.7 recommonmark==0.4.0 sphinx-copybutton +alabaster_jupyterhub diff --git a/docs/rest-api.yml b/docs/rest-api.yml index 1eff3a53..573e886b 100644 --- a/docs/rest-api.yml +++ b/docs/rest-api.yml @@ -89,7 +89,7 @@ paths: post: summary: Create multiple users parameters: - - name: data + - name: body in: body required: true schema: @@ -147,7 +147,7 @@ paths: in: path required: true type: string - - name: data + - name: body in: body required: true description: Updated user info. At least one key to be updated (name or admin) is required. @@ -176,6 +176,60 @@ paths: responses: '204': description: The user has been deleted + /users/{name}/activity: + post: + summary: + Notify Hub of activity for a given user. + description: + Notify the Hub of activity by the user, + e.g. accessing a service or (more likely) + actively using a server. + parameters: + - name: name + description: username + in: path + required: true + type: string + - body: + in: body + schema: + type: object + properties: + last_activity: + type: string + format: date-time + description: | + Timestamp of last-seen activity for this user. + Only needed if this is not activity associated + with using a given server. + required: false + servers: + description: | + Register activity for specific servers by name. + The keys of this dict are the names of servers. + The default server has an empty name (''). + required: false + type: object + properties: + '': + description: | + Activity for a single server. + type: object + properties: + last_activity: + required: true + type: string + format: date-time + description: | + Timestamp of last-seen activity on this server. + example: + last_activity: '2019-02-06T12:54:14Z' + servers: + '': + last_activity: '2019-02-06T12:54:14Z' + gpu: + last_activity: '2019-02-06T12:54:14Z' + /users/{name}/server: post: summary: Start a user's single-user notebook server @@ -185,6 +239,15 @@ paths: in: path required: true type: string + - options: + description: | + Spawn options can be passed as a JSON body + when spawning via the API instead of spawn form. + The structure of the options + will depend on the Spawner's configuration. + in: body + required: false + type: object responses: '201': description: The user's notebook server has started @@ -217,13 +280,15 @@ paths: in: path required: true type: string - - name: remove + - options: description: | - Whether to fully remove the server, rather than just stop it. - Removing a server deletes things like the state of the stopped server. + Spawn options can be passed as a JSON body + when spawning via the API instead of spawn form. + The structure of the options + will depend on the Spawner's configuration. in: body required: false - type: boolean + type: object responses: '201': description: The user's notebook named-server has started @@ -242,6 +307,13 @@ paths: in: path required: true type: string + - name: remove + description: | + Whether to fully remove the server, rather than just stop it. + Removing a server deletes things like the state of the stopped server. + in: body + required: false + type: boolean responses: '204': description: The user's notebook named-server has stopped @@ -352,7 +424,7 @@ paths: in: path required: true type: string - - name: data + - name: body in: body required: true description: The users to add to the group @@ -377,7 +449,7 @@ paths: in: path required: true type: string - - name: data + - name: body in: body required: true description: The users to remove from the group @@ -435,7 +507,7 @@ paths: summary: Notify the Hub about a new proxy description: Notifies the Hub of a new proxy to use. parameters: - - name: data + - name: body in: body required: true description: Any values that have changed for the new proxy. All keys are optional. diff --git a/docs/source/conf.py b/docs/source/conf.py index 9f3bb4b6..eeaa7020 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -67,7 +67,9 @@ source_suffix = ['.rst', '.md'] # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. -html_theme = 'alabaster' +import alabaster_jupyterhub +html_theme = 'alabaster_jupyterhub' +html_theme_path = [alabaster_jupyterhub.get_html_theme_path()] html_logo = '_static/images/logo/logo.png' html_favicon = '_static/images/logo/favicon.ico' diff --git a/docs/source/contributing/roadmap.md b/docs/source/contributing/roadmap.md new file mode 100644 index 00000000..b0efbec1 --- /dev/null +++ b/docs/source/contributing/roadmap.md @@ -0,0 +1,98 @@ +# The JupyterHub roadmap + +This roadmap collects "next steps" for JupyterHub. It is about creating a +shared understanding of the project's vision and direction amongst +the community of users, contributors, and maintainers. +The goal is to communicate priorities and upcoming release plans. +It is not a aimed at limiting contributions to what is listed here. + + +## Using the roadmap +### Sharing Feedback on the Roadmap + +All of the community is encouraged to provide feedback as well as share new +ideas with the community. Please do so by submitting an issue. If you want to +have an informal conversation first use one of the other communication channels. +After submitting the issue, others from the community will probably +respond with questions or comments they have to clarify the issue. The +maintainers will help identify what a good next step is for the issue. + +### What do we mean by "next step"? + +When submitting an issue, think about what "next step" category best describes +your issue: + +* **now**, concrete/actionable step that is ready for someone to start work on. +These might be items that have a link to an issue or more abstract like +"decrease typos and dead links in the documentation" +* **soon**, less concrete/actionable step that is going to happen soon, +discussions around the topic are coming close to an end at which point it can +move into the "now" category +* **later**, abstract ideas or tasks, need a lot of discussion or +experimentation to shape the idea so that it can be executed. Can also +contain concrete/actionable steps that have been postponed on purpose +(these are steps that could be in "now" but the decision was taken to work on +them later) + +### Reviewing and Updating the Roadmap + +The roadmap will get updated as time passes (next review by 1st December) based +on discussions and ideas captured as issues. +This means this list should not be exhaustive, it should only represent +the "top of the stack" of ideas. It should +not function as a wish list, collection of feature requests or todo list. +For those please create a +[new issue](https://github.com/jupyterhub/jupyterhub/issues/new). + +The roadmap should give the reader an idea of what is happening next, what needs +input and discussion before it can happen and what has been postponed. + + +## The roadmap proper +### Project vision + +JupyterHub is a dependable tool used by humans that reduces the complexity of +creating the environment in which a piece of software can be executed. + +### Now + +These "Now" items are considered active areas of focus for the project: + +* HubShare - a sharing service for use with JupyterHub. + * Users should be able to: + - Push a project to other users. + - Get a checkout of a project from other users. + - Push updates to a published project. + - Pull updates from a published project. + - Manage conflicts/merges by simply picking a version (our/theirs) + - Get a checkout of a project from the internet. These steps are completely different from saving notebooks/files. + - Have directories that are managed by git completely separately from our stuff. + - Look at pushed content that they have access to without an explicit pull. + - Define and manage teams of users. + - Adding/removing a user to/from a team gives/removes them access to all projects that team has access to. + - Build other services, such as static HTML publishing and dashboarding on top of these things. + + +### Soon + +These "Soon" items are under discussion. Once an item reaches the point of an +actionable plan, the item will be moved to the "Now" section. Typically, +these will be moved at a future review of the roadmap. + +* resource monitoring and management: + - (prometheus?) API for resource monitoring + - tracking activity on single-user servers instead of the proxy + - notes and activity tracking per API token + - UI for managing named servers + + +### Later + +The "Later" items are things that are at the back of the project's mind. At this +time there is no active plan for an item. The project would like to find the +resources and time to discuss these ideas. + +- real-time collaboration + - Enter into real-time collaboration mode for a project that starts a shared execution context. + - Once the single-user notebook package supports realtime collaboration, + implement sharing mechanism integrated into the Hub. \ No newline at end of file diff --git a/docs/source/gallery-jhub-deployments.md b/docs/source/gallery-jhub-deployments.md index 44de1cdb..9baa8ce3 100644 --- a/docs/source/gallery-jhub-deployments.md +++ b/docs/source/gallery-jhub-deployments.md @@ -85,8 +85,13 @@ easy to do with RStudio too. - https://datascience.business.illinois.edu +### IllustrisTNG Simulation Project + +- [JupyterHub/Lab-based analysis platform, part of the TNG public data release](http://www.tng-project.org/data/) + ### MIT and Lincoln Labs +- https://supercloud.mit.edu/ ### Michigan State University @@ -146,7 +151,6 @@ easy to do with RStudio too. [Everware](https://github.com/everware) Reproducible and reusable science powered by jupyterhub and docker. Like nbviewer, but executable. CERN, Geneva [website](http://everware.xyz/) - ### Microsoft Azure - https://docs.microsoft.com/en-us/azure/machine-learning/machine-learning-data-science-linux-dsvm-intro @@ -156,9 +160,7 @@ easy to do with RStudio too. - https://getcarina.com/blog/learning-how-to-whale/ - http://carolynvanslyck.com/talk/carina/jupyterhub/#/ -### jcloud.io -- Open to public JupyterHub server - - https://jcloud.io + ## Miscellaneous - https://medium.com/@ybarraud/setting-up-jupyterhub-with-sudospawner-and-anaconda-844628c0dbee#.rm3yt87e1 diff --git a/docs/source/index.rst b/docs/source/index.rst index 1047f6d3..54d221e5 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -30,7 +30,7 @@ For convenient administration of the Hub, its users, and services, JupyterHub also provides a `REST API`_. The JupyterHub team and Project Jupyter value our community, and JupyterHub -follows the Jupyter [Community Guides](https://jupyter.readthedocs.io/en/latest/community/content-community.html). +follows the Jupyter `Community Guides `_. Contents ======== @@ -115,6 +115,7 @@ helps keep our community welcoming to as many people as possible. contributing/setup contributing/docs contributing/tests + contributing/roadmap Upgrading JupyterHub -------------------- diff --git a/docs/source/reference/config-proxy.md b/docs/source/reference/config-proxy.md index 093d4ee2..6d30e966 100644 --- a/docs/source/reference/config-proxy.md +++ b/docs/source/reference/config-proxy.md @@ -190,3 +190,24 @@ Listen 443 ``` + + +In case of the need to run the jupyterhub under /jhub/ or other location please use the below configurations: +- JupyterHub running locally at http://127.0.0.1:8000/jhub/ or other location + +httpd.conf amendments: +```bash + RewriteRule /jhub/(.*) ws://127.0.0.1:8000/jhub/$1 [P,L] + RewriteRule /jhub/(.*) http://127.0.0.1:8000/jhub/$1 [P,L] + + ProxyPass /jhub/ http://127.0.0.1:8000/jhub/ + ProxyPassReverse /jhub/ http://127.0.0.1:8000/jhub/ + ``` + +jupyterhub_config.py amendments: + ```bash + --The public facing URL of the whole JupyterHub application. + --This is the address on which the proxy will bind. Sets protocol, ip, base_url + c.JupyterHub.bind_url = 'http://127.0.0.1:8000/jhub/' + ``` + diff --git a/docs/source/reference/templates.md b/docs/source/reference/templates.md index fefca625..61784403 100644 --- a/docs/source/reference/templates.md +++ b/docs/source/reference/templates.md @@ -25,19 +25,19 @@ supplement the material in the block. The make extensive use of blocks, which allows you to customize parts of the interface easily. -In general, a child template can extend a base template, `base.html`, by beginning with: +In general, a child template can extend a base template, `page.html`, by beginning with: ```html -{% extends "base.html" %} +{% extends "page.html" %} ``` This works, unless you are trying to extend the default template for the same file name. Starting in version 0.9, you may refer to the base file with a -`templates/` prefix. Thus, if you are writing a custom `base.html`, start the +`templates/` prefix. Thus, if you are writing a custom `page.html`, start the file with this block: ```html -{% extends "templates/base.html" %} +{% extends "templates/page.html" %} ``` By defining `block`s with same name as in the base template, child templates diff --git a/jupyterhub/apihandlers/users.py b/jupyterhub/apihandlers/users.py index b12320c1..dbdfc9f3 100644 --- a/jupyterhub/apihandlers/users.py +++ b/jupyterhub/apihandlers/users.py @@ -4,16 +4,17 @@ # Distributed under the terms of the Modified BSD License. import asyncio -from datetime import datetime +from datetime import datetime, timedelta, timezone import json from async_generator import aclosing +from dateutil.parser import parse as parse_date from tornado import web from tornado.iostream import StreamClosedError from .. import orm from ..user import User -from ..utils import admin_only, iterate_until, maybe_future, url_path_join +from ..utils import admin_only, isoformat, iterate_until, maybe_future, url_path_join from .base import APIHandler @@ -242,7 +243,10 @@ class UserTokenListAPIHandler(APIHandler): # defer to Authenticator for identifying the user # can be username+password or an upstream auth token try: - name = await self.authenticator.authenticate(self, body.get('auth')) + name = await self.authenticate(body.get('auth')) + if isinstance(name, dict): + # not a simple string so it has to be a dict + name = name.get('name') except web.HTTPError as e: # turn any authentication error into 403 raise web.HTTPError(403) @@ -347,8 +351,19 @@ class UserServerAPIHandler(APIHandler): @admin_or_self async def post(self, name, server_name=''): user = self.find_user(name) - if server_name and not self.allow_named_servers: - raise web.HTTPError(400, "Named servers are not enabled.") + if server_name: + if not self.allow_named_servers: + raise web.HTTPError(400, "Named servers are not enabled.") + if self.named_server_limit_per_user > 0 and server_name not in user.orm_spawners: + named_spawners = list(user.all_spawners(include_default=False)) + if self.named_server_limit_per_user <= len(named_spawners): + raise web.HTTPError( + 400, + "User {} already has the maximum of {} named servers." + " One must be deleted before a new server can be created".format( + name, + self.named_server_limit_per_user + )) spawner = user.spawners[server_name] pending = spawner.pending if pending == 'spawn': @@ -573,6 +588,139 @@ class SpawnProgressAPIHandler(APIHandler): await self.send_event(failed_event) +def _parse_timestamp(timestamp): + """Parse and return a utc timestamp + + - raise HTTPError(400) on parse error + - handle and strip tz info for internal consistency + (we use naïve utc timestamps everywhere) + """ + try: + dt = parse_date(timestamp) + except Exception: + raise web.HTTPError(400, "Not a valid timestamp: %r", timestamp) + if dt.tzinfo: + # strip timezone info to naïve UTC datetime + dt = dt.astimezone(timezone.utc).replace(tzinfo=None) + + now = datetime.utcnow() + if (dt - now) > timedelta(minutes=59): + raise web.HTTPError( + 400, + "Rejecting activity from more than an hour in the future: {}".format( + isoformat(dt) + ) + ) + return dt + + +class ActivityAPIHandler(APIHandler): + + def _validate_servers(self, user, servers): + """Validate servers dict argument + + - types are correct + - each server exists + - last_activity fields are parsed into datetime objects + """ + msg = "servers must be a dict of the form {server_name: {last_activity: timestamp}}" + if not isinstance(servers, dict): + raise web.HTTPError(400, msg) + + spawners = user.orm_spawners + for server_name, server_info in servers.items(): + if server_name not in spawners: + raise web.HTTPError( + 400, + "No such server '{}' for user {}".format( + server_name, + user.name, + ) + ) + # check that each per-server field is a dict + if not isinstance(server_info, dict): + raise web.HTTPError(400, msg) + # check that last_activity is defined for each per-server dict + if 'last_activity' not in server_info: + raise web.HTTPError(400, msg) + # parse last_activity timestamps + # _parse_timestamp above is responsible for raising errors + server_info['last_activity'] = _parse_timestamp(server_info['last_activity']) + return servers + + @admin_or_self + def post(self, username): + user = self.find_user(username) + if user is None: + # no such user + raise web.HTTPError(404, "No such user: %r", username) + + body = self.get_json_body() + if not isinstance(body, dict): + raise web.HTTPError(400, "body must be a json dict") + + last_activity_timestamp = body.get('last_activity') + servers = body.get('servers') + if not last_activity_timestamp and not servers: + raise web.HTTPError( + 400, + "body must contain at least one of `last_activity` or `servers`" + ) + + if servers: + # validate server args + servers = self._validate_servers(user, servers) + # at this point we know that the servers dict + # is valid and contains only servers that exist + # and last_activity is defined and a valid datetime object + + # update user.last_activity if specified + if last_activity_timestamp: + last_activity = _parse_timestamp(last_activity_timestamp) + if ( + (not user.last_activity) + or last_activity > user.last_activity + ): + self.log.debug("Activity for user %s: %s", + user.name, + isoformat(last_activity), + ) + user.last_activity = last_activity + else: + self.log.debug( + "Not updating activity for %s: %s < %s", + user, + isoformat(last_activity), + isoformat(user.last_activity), + ) + + if servers: + for server_name, server_info in servers.items(): + last_activity = server_info['last_activity'] + spawner = user.orm_spawners[server_name] + + if ( + (not spawner.last_activity) + or last_activity > spawner.last_activity + ): + self.log.debug("Activity on server %s/%s: %s", + user.name, + server_name, + isoformat(last_activity), + ) + spawner.last_activity = last_activity + else: + self.log.debug( + "Not updating server activity on %s/%s: %s < %s", + user.name, + server_name, + isoformat(last_activity), + isoformat(user.last_activity), + ) + + self.db.commit() + + default_handlers = [ (r"/api/user", SelfAPIHandler), (r"/api/users", UserListAPIHandler), @@ -583,5 +731,6 @@ default_handlers = [ (r"/api/users/([^/]+)/tokens/([^/]*)", UserTokenAPIHandler), (r"/api/users/([^/]+)/servers/([^/]*)", UserServerAPIHandler), (r"/api/users/([^/]+)/servers/([^/]*)/progress", SpawnProgressAPIHandler), + (r"/api/users/([^/]+)/activity", ActivityAPIHandler), (r"/api/users/([^/]+)/admin-access", UserAdminAccessAPIHandler), ] diff --git a/jupyterhub/app.py b/jupyterhub/app.py index e0cfa4dc..8ceacff0 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -803,6 +803,16 @@ class JupyterHub(Application): help="Allow named single-user servers per user" ).tag(config=True) + named_server_limit_per_user = Integer(0, + help=""" + Maximum number of concurrent named servers that can be created by a user at a time. + + Setting this can limit the total resources a user can consume. + + If set to 0, no limit is enforced. + """ + ).tag(config=True) + # class for spawning single-user servers spawner_class = EntryPointType( default_value=LocalProcessSpawner, @@ -1020,6 +1030,11 @@ class JupyterHub(Application): statsd = Any(allow_none=False, help="The statsd client, if any. A mock will be used if we aren't using statsd") + shutdown_on_logout = Bool( + False, + help="""Shuts down all user servers on logout""" + ).tag(config=True) + @default('statsd') def _statsd(self): if self.statsd_host: @@ -1687,6 +1702,40 @@ class JupyterHub(Application): spawner._log_name) status = -1 + if status is None: + # poll claims it's running. + # Check if it's really there + url_in_db = spawner.server.url + url = await spawner.get_url() + if url != url_in_db: + self.log.warning( + "%s had invalid url %s. Updating to %s", + spawner._log_name, url_in_db, url, + ) + urlinfo = urlparse(url) + spawner.server.protocol = urlinfo.scheme + spawner.server.ip = urlinfo.hostname + if urlinfo.port: + spawner.server.port = urlinfo.port + elif urlinfo.scheme == 'http': + spawner.server.port = 80 + elif urlinfo.scheme == 'https': + spawner.server.port = 443 + self.db.commit() + + self.log.debug( + "Verifying that %s is running at %s", + spawner._log_name, url, + ) + try: + await user._wait_up(spawner) + except TimeoutError: + self.log.error( + "%s does not appear to be running at %s, shutting it down.", + spawner._log_name, url, + ) + status = -1 + if status is None: self.log.info("%s still running", user.name) spawner.add_poll_callback(user_stopped, user, name) @@ -1836,6 +1885,7 @@ class JupyterHub(Application): domain=self.domain, statsd=self.statsd, allow_named_servers=self.allow_named_servers, + named_server_limit_per_user=self.named_server_limit_per_user, oauth_provider=self.oauth_provider, concurrent_spawn_limit=self.concurrent_spawn_limit, spawn_throttle_retry_range=self.spawn_throttle_retry_range, @@ -1849,6 +1899,7 @@ class JupyterHub(Application): internal_ssl_cert=self.internal_ssl_cert, internal_ssl_ca=self.internal_ssl_ca, trusted_alt_names=self.trusted_alt_names, + shutdown_on_logout=self.shutdown_on_logout ) # allow configured settings to have priority settings.update(self.tornado_settings) diff --git a/jupyterhub/auth.py b/jupyterhub/auth.py index 9d0eb570..5152e97a 100644 --- a/jupyterhub/auth.py +++ b/jupyterhub/auth.py @@ -19,40 +19,15 @@ except Exception as e: _pamela_error = e from tornado.concurrent import run_on_executor -from tornado import gen from traitlets.config import LoggingConfigurable -from traitlets import Bool, Set, Unicode, Dict, Any, default, observe +from traitlets import Bool, Integer, Set, Unicode, Dict, Any, default, observe from .handlers.login import LoginHandler from .utils import maybe_future, url_path_join from .traitlets import Command -def getgrnam(name): - """Wrapper function to protect against `grp` not being available - on Windows - """ - import grp - return grp.getgrnam(name) - - -def getpwnam(name): - """Wrapper function to protect against `pwd` not being available - on Windows - """ - import pwd - return pwd.getpwnam(name) - - -def getgrouplist(name, group): - """Wrapper function to protect against `os.getgrouplist` not being available - on Windows - """ - import os - return os.getgrouplist(name, group) - - class Authenticator(LoggingConfigurable): """Base class for implementing an authentication provider for JupyterHub""" @@ -77,6 +52,35 @@ class Authenticator(LoggingConfigurable): """, ) + auth_refresh_age = Integer( + 300, + config=True, + help="""The max age (in seconds) of authentication info + before forcing a refresh of user auth info. + + Refreshing auth info allows, e.g. requesting/re-validating auth tokens. + + See :meth:`.refresh_user` for what happens when user auth info is refreshed + (nothing by default). + """ + ) + + refresh_pre_spawn = Bool( + False, + config=True, + help="""Force refresh of auth prior to spawn. + + This forces :meth:`.refresh_user` to be called prior to launching + a server, to ensure that auth state is up-to-date. + + This can be important when e.g. auth tokens that may have expired + are passed to the spawner via environment variables from auth_state. + + If refresh_user cannot refresh the user auth data, + launch will fail until the user logs in again. + """ + ) + admin_users = Set( help=""" Set of users that will have admin rights on this JupyterHub. @@ -329,7 +333,7 @@ class Authenticator(LoggingConfigurable): self.log.warning("User %r not in whitelist.", username) return - async def refresh_user(self, user): + async def refresh_user(self, user, handler=None): """Refresh auth data for a given user Allows refreshing or invalidating auth data. @@ -341,6 +345,7 @@ class Authenticator(LoggingConfigurable): Args: user (User): the user to refresh + handler (tornado.web.RequestHandler or None): the current request handler Returns: auth_data (bool or dict): Return **True** if auth data for the user is up-to-date @@ -594,7 +599,7 @@ class LocalAuthenticator(Authenticator): return False for grnam in self.group_whitelist: try: - group = getgrnam(grnam) + group = self._getgrnam(grnam) except KeyError: self.log.error('No such group: [%s]' % grnam) continue @@ -622,11 +627,33 @@ class LocalAuthenticator(Authenticator): await maybe_future(super().add_user(user)) @staticmethod - def system_user_exists(user): - """Check if the user exists on the system""" + def _getgrnam(name): + """Wrapper function to protect against `grp` not being available + on Windows + """ + import grp + return grp.getgrnam(name) + + @staticmethod + def _getpwnam(name): + """Wrapper function to protect against `pwd` not being available + on Windows + """ import pwd + return pwd.getpwnam(name) + + @staticmethod + def _getgrouplist(name, group): + """Wrapper function to protect against `os._getgrouplist` not being available + on Windows + """ + import os + return os.getgrouplist(name, group) + + def system_user_exists(self, user): + """Check if the user exists on the system""" try: - pwd.getpwnam(user.name) + self._getpwnam(user.name) except KeyError: return False else: @@ -727,8 +754,8 @@ class PAMAuthenticator(LocalAuthenticator): # fail to authenticate and raise instead of soft-failing and not changing admin status # (returning None instead of just the username) as this indicates some sort of system failure - admin_group_gids = {getgrnam(x).gr_gid for x in self.admin_groups} - user_group_gids = {x.gr_gid for x in getgrouplist(username, getpwnam(username).pw_gid)} + admin_group_gids = {self._getgrnam(x).gr_gid for x in self.admin_groups} + user_group_gids = set(self._getgrouplist(username, self._getpwnam(username).pw_gid)) admin_status = len(admin_group_gids & user_group_gids) != 0 except Exception as e: diff --git a/jupyterhub/handlers/base.py b/jupyterhub/handlers/base.py index 802516f9..8c6ec5ca 100644 --- a/jupyterhub/handlers/base.py +++ b/jupyterhub/handlers/base.py @@ -28,6 +28,7 @@ from .. import __version__ from .. import orm from ..objects import Server from ..spawner import LocalProcessSpawner +from ..user import User from ..utils import maybe_future, url_path_join from ..metrics import ( SERVER_SPAWN_DURATION_SECONDS, ServerSpawnStatus, @@ -103,6 +104,10 @@ class BaseHandler(RequestHandler): def allow_named_servers(self): return self.settings.get('allow_named_servers', False) + @property + def named_server_limit_per_user(self): + return self.settings.get('named_server_limit_per_user', 0) + @property def domain(self): return self.settings['domain'] @@ -236,7 +241,7 @@ class BaseHandler(RequestHandler): self.db.commit() return self._user_from_orm(orm_token.user) - async def refresh_user_auth(self, user, force=False): + async def refresh_auth(self, user, force=False): """Refresh user authentication info Calls `authenticator.refresh_user(user)` @@ -250,7 +255,12 @@ class BaseHandler(RequestHandler): user (User): the user having been refreshed, or None if the user must login again to refresh auth info. """ - if not force: # TODO: and it's sufficiently recent + refresh_age = self.authenticator.auth_refresh_age + if not refresh_age: + return user + now = time.monotonic() + if not force and user._auth_refreshed and (now - user._auth_refreshed < refresh_age): + # auth up-to-date return user # refresh a user at most once per request @@ -262,7 +272,7 @@ class BaseHandler(RequestHandler): self._refreshed_users.add(user.name) self.log.debug("Refreshing auth for %s", user.name) - auth_info = await self.authenticator.refresh_user(user) + auth_info = await self.authenticator.refresh_user(user, self) if not auth_info: self.log.warning( @@ -271,6 +281,8 @@ class BaseHandler(RequestHandler): ) return + user._auth_refreshed = now + if auth_info == True: # refresh_user confirmed that it's up-to-date, # nothing to refresh @@ -298,7 +310,11 @@ class BaseHandler(RequestHandler): now = datetime.utcnow() orm_token.last_activity = now if orm_token.user: - orm_token.user.last_activity = now + # FIXME: scopes should give us better control than this + # don't consider API requests originating from a server + # to be activity from the user + if not orm_token.note.startswith("Server at "): + orm_token.user.last_activity = now self.db.commit() if orm_token.service: @@ -351,8 +367,8 @@ class BaseHandler(RequestHandler): user = self.get_current_user_token() if user is None: user = self.get_current_user_cookie() - if user: - user = await self.refresh_user_auth(user) + if user and isinstance(user, User): + user = await self.refresh_auth(user) self._jupyterhub_user = user except Exception: # don't let errors here raise more than once @@ -606,6 +622,7 @@ class BaseHandler(RequestHandler): self.statsd.incr('login.success') self.statsd.timing('login.authenticate.success', auth_timer.ms) self.log.info("User logged in: %s", user.name) + user._auth_refreshed = time.monotonic() return user else: self.statsd.incr('login.failure') @@ -639,6 +656,11 @@ class BaseHandler(RequestHandler): async def spawn_single_user(self, user, server_name='', options=None): # in case of error, include 'try again from /hub/home' message + if self.authenticator.refresh_pre_spawn: + auth_user = await self.refresh_auth(user, force=True) + if auth_user is None: + raise web.HTTPError(403, "auth has expired for %s, login again", user.name) + spawn_start_time = time.perf_counter() self.extra_error_html = self.spawn_home_error @@ -1050,6 +1072,10 @@ class PrefixRedirectHandler(BaseHandler): path = self.request.uri[len(self.base_url):] else: path = self.request.path + if not path: + # default / -> /hub/ redirect + # avoiding extra hop through /hub + path = '/' self.redirect(url_path_join( self.hub.base_url, path, ), permanent=False) @@ -1098,7 +1124,7 @@ class UserSpawnHandler(BaseHandler): # otherwise redirect users to their own server should_spawn = (current_user and current_user.name == user_name) - if "api" in user_path.split("/") and not user.active: + if "api" in user_path.split("/") and user and not user.active: # API request for not-running server (e.g. notebook UI left open) # Avoid triggering a spawn. self._fail_api_request(user) @@ -1187,7 +1213,7 @@ class UserSpawnHandler(BaseHandler): status = 0 # server is not running, trigger spawn if status is not None: - if spawner.options_form: + if await spawner.get_options_form(): url_parts = [self.hub.base_url, 'spawn'] if current_user.name != user.name: # spawning on behalf of another user diff --git a/jupyterhub/handlers/login.py b/jupyterhub/handlers/login.py index 666a5a00..f806eae0 100644 --- a/jupyterhub/handlers/login.py +++ b/jupyterhub/handlers/login.py @@ -3,19 +3,41 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +import asyncio + from tornado.escape import url_escape -from tornado import gen from tornado.httputil import url_concat from tornado import web from .base import BaseHandler +from ..utils import maybe_future class LogoutHandler(BaseHandler): """Log a user out by clearing their login cookie.""" - def get(self): + + @property + def shutdown_on_logout(self): + return self.settings.get('shutdown_on_logout', False) + + async def get(self): user = self.current_user if user: + if self.shutdown_on_logout: + active_servers = [ + name + for (name, spawner) in user.spawners.items() + if spawner.active and not spawner.pending + ] + if active_servers: + self.log.info("Shutting down %s's servers", user.name) + futures = [] + for server_name in active_servers: + futures.append( + maybe_future(self.stop_single_user(user, server_name)) + ) + await asyncio.gather(*futures) + self.log.info("User logged out: %s", user.name) self.clear_login_cookie() self.statsd.incr('logout') diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index 9a6a2d39..53e4487a 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -58,6 +58,7 @@ class HomeHandler(BaseHandler): user=user, url=url, allow_named_servers=self.allow_named_servers, + named_server_limit_per_user=self.named_server_limit_per_user, url_path_join=url_path_join, # can't use user.spawners because the stop method of User pops named servers from user.spawners when they're stopped spawners = user.orm_user._orm_spawners, @@ -73,11 +74,7 @@ class SpawnHandler(BaseHandler): Only enabled when Spawner.options_form is defined. """ - async def _render_form(self, message='', for_user=None): - # Note that 'user' is the authenticated user making the request and - # 'for_user' is the user whose server is being spawned. - user = for_user or self.get_current_user() - spawner_options_form = await user.spawner.get_options_form() + def _render_form(self, for_user, spawner_options_form, message=''): return self.render_template('spawn.html', for_user=for_user, spawner_options_form=spawner_options_form, @@ -106,10 +103,12 @@ class SpawnHandler(BaseHandler): self.log.debug("User is running: %s", url) self.redirect(url) return - if user.spawner.options_form: + + spawner_options_form = await user.spawner.get_options_form() + if spawner_options_form: # Add handler to spawner here so you can access query params in form rendering. user.spawner.handler = self - form = await self._render_form(for_user=user) + form = self._render_form(for_user=user, spawner_options_form=spawner_options_form) self.finish(form) else: # Explicit spawn request: clear _spawn_future @@ -153,7 +152,8 @@ class SpawnHandler(BaseHandler): await self.spawn_single_user(user, options=options) except Exception as e: self.log.error("Failed to spawn single-user server with form", exc_info=True) - form = await self._render_form(message=str(e), for_user=user) + spawner_options_form = await user.spawner.get_options_form() + form = self._render_form(for_user=user, spawner_options_form=spawner_options_form, message=str(e)) self.finish(form) return if current_user is user: @@ -230,6 +230,7 @@ class AdminHandler(BaseHandler): running=running, sort={s:o for s,o in zip(sorts, orders)}, allow_named_servers=self.allow_named_servers, + named_server_limit_per_user=self.named_server_limit_per_user, ) self.finish(html) diff --git a/jupyterhub/orm.py b/jupyterhub/orm.py index f0fe39bc..59c0fce1 100644 --- a/jupyterhub/orm.py +++ b/jupyterhub/orm.py @@ -12,14 +12,13 @@ import alembic.command from alembic.script import ScriptDirectory from tornado.log import app_log -from sqlalchemy.types import TypeDecorator, TEXT, LargeBinary +from sqlalchemy.types import TypeDecorator, Text, LargeBinary from sqlalchemy import ( create_engine, event, exc, inspect, or_, select, Column, Integer, ForeignKey, Unicode, Boolean, DateTime, Enum, Table, ) from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.interfaces import PoolListener from sqlalchemy.orm import ( Session, interfaces, object_session, relationship, sessionmaker, @@ -46,7 +45,7 @@ class JSONDict(TypeDecorator): """ - impl = TEXT + impl = Text def process_bind_param(self, value, dialect): if value is not None: @@ -559,10 +558,13 @@ class DatabaseSchemaMismatch(Exception): """ -class ForeignKeysListener(PoolListener): - """Enable foreign keys on sqlite""" - def connect(self, dbapi_con, con_record): - dbapi_con.execute('pragma foreign_keys=ON') +def register_foreign_keys(engine): + """register PRAGMA foreign_keys=on on connection""" + @event.listens_for(engine, "connect") + def connect(dbapi_con, con_record): + cursor = dbapi_con.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() def _expire_relationship(target, relationship_prop): @@ -735,8 +737,6 @@ def new_session_factory(url="sqlite:///:memory:", """Create a new session at url""" if url.startswith('sqlite'): kwargs.setdefault('connect_args', {'check_same_thread': False}) - listeners = kwargs.setdefault('listeners', []) - listeners.append(ForeignKeysListener()) elif url.startswith('mysql'): kwargs.setdefault('pool_recycle', 60) @@ -747,6 +747,9 @@ def new_session_factory(url="sqlite:///:memory:", kwargs.setdefault('poolclass', StaticPool) engine = create_engine(url, **kwargs) + if url.startswith('sqlite'): + register_foreign_keys(engine) + # enable pessimistic disconnect handling register_ping_connection(engine) diff --git a/jupyterhub/services/auth.py b/jupyterhub/services/auth.py index ef80799f..90627bc8 100644 --- a/jupyterhub/services/auth.py +++ b/jupyterhub/services/auth.py @@ -99,6 +99,11 @@ class _ExpiringDict(dict): except KeyError: return default + def clear(self): + """Clear the cache""" + self.values.clear() + self.timestamps.clear() + class HubAuth(SingletonConfigurable): """A class for authenticating with JupyterHub @@ -277,11 +282,10 @@ class HubAuth(SingletonConfigurable): if cache_key is None: raise ValueError("cache_key is required when using cache") # check for a cached reply, so we don't check with the Hub if we don't have to - cached = self.cache.get(cache_key) - if cached is not None: - return cached - else: - app_log.debug("Cache miss: %s" % cache_key) + try: + return self.cache[cache_key] + except KeyError: + app_log.debug("HubAuth cache miss: %s", cache_key) data = self._api_request('GET', url, allow_404=True) if data is None: @@ -848,8 +852,8 @@ class HubAuthenticated(object): self._hub_auth_user_cache = None raise - # store ?token=... tokens passed via url in a cookie for future requests - url_token = self.get_argument('token', '') + # store tokens passed via url or header in a cookie for future requests + url_token = self.hub_auth.get_token(self) if ( user_model and url_token diff --git a/jupyterhub/services/service.py b/jupyterhub/services/service.py index 9ddd4b45..11e05ece 100644 --- a/jupyterhub/services/service.py +++ b/jupyterhub/services/service.py @@ -107,6 +107,8 @@ class _ServiceSpawner(LocalProcessSpawner): def start(self): """Start the process""" env = self.get_env() + # no activity url for services + env.pop('JUPYTERHUB_ACTIVITY_URL', None) if os.name == 'nt': env['SYSTEMROOT'] = os.environ['SYSTEMROOT'] cmd = self.cmd @@ -258,7 +260,6 @@ class Service(LoggingConfigurable): def _default_redirect_uri(self): if self.server is None: return '' - print(self.domain, self.host, self.server) return self.host + url_path_join(self.prefix, 'oauth_callback') @property diff --git a/jupyterhub/singleuser.py b/jupyterhub/singleuser.py index 84038173..e6bf07f2 100755 --- a/jupyterhub/singleuser.py +++ b/jupyterhub/singleuser.py @@ -4,13 +4,17 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +import asyncio +from datetime import datetime, timezone +import json import os +import random from textwrap import dedent from urllib.parse import urlparse from jinja2 import ChoiceLoader, FunctionLoader -from tornado.httpclient import AsyncHTTPClient +from tornado.httpclient import AsyncHTTPClient, HTTPRequest from tornado import gen from tornado import ioloop from tornado.web import HTTPError, RequestHandler @@ -21,8 +25,10 @@ except ImportError: raise ImportError("JupyterHub single-user server requires notebook >= 4.0") from traitlets import ( + Any, Bool, Bytes, + Integer, Unicode, CUnicode, default, @@ -43,7 +49,7 @@ from notebook.base.handlers import IPythonHandler from ._version import __version__, _check_version from .log import log_request from .services.auth import HubOAuth, HubOAuthenticated, HubOAuthCallbackHandler -from .utils import url_path_join, make_ssl_context +from .utils import isoformat, url_path_join, make_ssl_context, exponential_backoff # Authenticate requests with the Hub @@ -385,20 +391,32 @@ class SingleUserNotebookApp(NotebookApp): path = list(_exclude_home(path)) return path + # create dynamic default http client, + # configured with any relevant ssl config + hub_http_client = Any() + @default('hub_http_client') + def _default_client(self): + ssl_context = make_ssl_context( + self.keyfile, + self.certfile, + cafile=self.client_ca, + ) + AsyncHTTPClient.configure( + None, + defaults={ + "ssl_options": ssl_context, + }, + ) + return AsyncHTTPClient() + + async def check_hub_version(self): """Test a connection to my Hub - exit if I can't connect at all - check version and warn on sufficient mismatch """ - ssl_context = make_ssl_context( - self.keyfile, - self.certfile, - cafile=self.client_ca, - ) - AsyncHTTPClient.configure(None, defaults={"ssl_options" : ssl_context}) - - client = AsyncHTTPClient() + client = self.hub_http_client RETRIES = 5 for i in range(1, RETRIES+1): try: @@ -415,6 +433,112 @@ class SingleUserNotebookApp(NotebookApp): hub_version = resp.headers.get('X-JupyterHub-Version') _check_version(hub_version, __version__, self.log) + server_name = Unicode() + @default('server_name') + def _server_name_default(self): + return os.environ.get('JUPYTERHUB_SERVER_NAME', '') + + hub_activity_url = Unicode( + config=True, + help="URL for sending JupyterHub activity updates", + ) + @default('hub_activity_url') + def _default_activity_url(self): + return os.environ.get('JUPYTERHUB_ACTIVITY_URL', '') + + hub_activity_interval = Integer( + 300, + config=True, + help=""" + Interval (in seconds) on which to update the Hub + with our latest activity. + """ + ) + @default('hub_activity_interval') + def _default_activity_interval(self): + env_value = os.environ.get('JUPYTERHUB_ACTIVITY_INTERVAL') + if env_value: + return int(env_value) + else: + return 300 + + _last_activity_sent = Any(allow_none=True) + + async def notify_activity(self): + """Notify jupyterhub of activity""" + client = self.hub_http_client + last_activity = self.web_app.last_activity() + if not last_activity: + self.log.debug("No activity to send to the Hub") + return + if last_activity: + # protect against mixed timezone comparisons + if not last_activity.tzinfo: + # assume naive timestamps are utc + self.log.warning("last activity is using naïve timestamps") + last_activity = last_activity.replace(tzinfo=timezone.utc) + + if ( + self._last_activity_sent + and last_activity < self._last_activity_sent + ): + self.log.debug("No activity since %s", self._last_activity_sent) + return + + last_activity_timestamp = isoformat(last_activity) + + async def notify(): + self.log.debug("Notifying Hub of activity %s", last_activity_timestamp) + req = HTTPRequest( + url=self.hub_activity_url, + method='POST', + headers={ + "Authorization": "token {}".format(self.hub_auth.api_token), + "Content-Type": "application/json", + }, + body=json.dumps({ + 'servers': { + self.server_name: { + 'last_activity': last_activity_timestamp, + }, + }, + 'last_activity': last_activity_timestamp, + }) + ) + try: + await client.fetch(req) + except Exception: + self.log.exception("Error notifying Hub of activity") + return False + else: + return True + + await exponential_backoff( + notify, + fail_message="Failed to notify Hub of activity", + start_wait=1, + max_wait=15, + timeout=60, + ) + self._last_activity_sent = last_activity + + async def keep_activity_updated(self): + if not self.hub_activity_url or not self.hub_activity_interval: + self.log.warning("Activity events disabled") + return + self.log.info("Updating Hub with activity every %s seconds", + self.hub_activity_interval + ) + while True: + try: + await self.notify_activity() + except Exception as e: + self.log.exception("Error notifying Hub of activity") + # add 20% jitter to the interval to avoid alignment + # of lots of requests from user servers + t = self.hub_activity_interval * (1 + 0.2 * (random.random() - 0.5)) + await asyncio.sleep(t) + def initialize(self, argv=None): # disable trash by default # this can be re-enabled by config @@ -425,6 +549,7 @@ class SingleUserNotebookApp(NotebookApp): self.log.info("Starting jupyterhub-singleuser server version %s", __version__) # start by hitting Hub to check version ioloop.IOLoop.current().run_sync(self.check_hub_version) + ioloop.IOLoop.current().add_callback(self.keep_activity_updated) super(SingleUserNotebookApp, self).start() def init_hub_auth(self): diff --git a/jupyterhub/spawner.py b/jupyterhub/spawner.py index 3d8311f3..b89723f5 100644 --- a/jupyterhub/spawner.py +++ b/jupyterhub/spawner.py @@ -5,6 +5,7 @@ Contains base Spawner class & default implementation # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +import ast import asyncio import errno import json @@ -14,7 +15,6 @@ import shutil import signal import sys import warnings -import pwd from subprocess import Popen from tempfile import mkdtemp @@ -36,6 +36,23 @@ from .traitlets import Command, ByteSpecification, Callable from .utils import iterate_until, maybe_future, random_port, url_path_join, exponential_backoff +def _quote_safe(s): + """pass a string that is safe on the command-line + + traitlets may parse literals on the command-line, e.g. `--ip=123` will be the number 123 instead of the *string* 123. + wrap valid literals in repr to ensure they are safe + """ + + try: + val = ast.literal_eval(s) + except Exception: + # not valid, leave it alone + return s + else: + # it's a valid literal, wrap it in repr (usually just quotes, but with proper escapes) + # to avoid getting interpreted by traitlets + return repr(s) + class Spawner(LoggingConfigurable): """Base class for spawning single-user notebook servers. @@ -636,7 +653,15 @@ class Spawner(LoggingConfigurable): # Info previously passed on args env['JUPYTERHUB_USER'] = self.user.name + env['JUPYTERHUB_SERVER_NAME'] = self.name env['JUPYTERHUB_API_URL'] = self.hub.api_url + env['JUPYTERHUB_ACTIVITY_URL'] = url_path_join( + self.hub.api_url, + 'users', + # tolerate mocks defining only user.name + getattr(self.user, 'escaped_name', self.user.name), + 'activity', + ) env['JUPYTERHUB_BASE_URL'] = self.hub.base_url[:-4] if self.server: env['JUPYTERHUB_SERVICE_PREFIX'] = self.server.base_url @@ -661,6 +686,22 @@ class Spawner(LoggingConfigurable): return env + async def get_url(self): + """Get the URL to connect to the server + + Sometimes JupyterHub may ask the Spawner for its url. + This can occur e.g. when JupyterHub has restarted while a server was not finished starting, + giving Spawners a chance to recover the URL where their server is running. + + The default is to trust that JupyterHub has the right information. + Only override this method in Spawners that know how to + check the correct URL for the servers they start. + + This will only be asked of Spawners that claim to be running + (`poll()` returns `None`). + """ + return self.server.url + def template_namespace(self): """Return the template namespace for format-string formatting. @@ -816,7 +857,7 @@ class Spawner(LoggingConfigurable): args = [] if self.ip: - args.append('--ip="%s"' % self.ip) + args.append('--ip=%s' % _quote_safe(self.ip)) if self.port: args.append('--port=%i' % self.port) @@ -826,10 +867,10 @@ class Spawner(LoggingConfigurable): if self.notebook_dir: notebook_dir = self.format_string(self.notebook_dir) - args.append('--notebook-dir="%s"' % notebook_dir) + args.append('--notebook-dir=%s' % _quote_safe(notebook_dir)) if self.default_url: default_url = self.format_string(self.default_url) - args.append('--NotebookApp.default_url="%s"' % default_url) + args.append('--NotebookApp.default_url=%s' % _quote_safe(default_url)) if self.debug: args.append('--debug') @@ -1051,6 +1092,7 @@ def set_user_setuid(username, chdir=True): home directory. """ import grp + import pwd user = pwd.getpwnam(username) uid = user.pw_uid gid = user.pw_gid @@ -1224,6 +1266,7 @@ class LocalProcessSpawner(Spawner): Stage certificates into a private home directory and make them readable by the user. """ + import pwd key = paths['keyfile'] cert = paths['certfile'] ca = paths['cafile'] @@ -1383,3 +1426,52 @@ class LocalProcessSpawner(Spawner): if status is None: # it all failed, zombie process self.log.warning("Process %i never died", self.pid) + + +class SimpleLocalProcessSpawner(LocalProcessSpawner): + """ + A version of LocalProcessSpawner that doesn't require users to exist on + the system beforehand. + + Only use this for testing. + + Note: DO NOT USE THIS FOR PRODUCTION USE CASES! It is very insecure, and + provides absolutely no isolation between different users! + """ + + home_dir_template = Unicode( + '/tmp/{username}', + config=True, + help=""" + Template to expand to set the user home. + {username} is expanded to the jupyterhub username. + """ + ) + + home_dir = Unicode(help="The home directory for the user") + @default('home_dir') + def _default_home_dir(self): + return self.home_dir_template.format( + username=self.user.name, + ) + + def make_preexec_fn(self, name): + home = self.home_dir + def preexec(): + try: + os.makedirs(home, 0o755, exist_ok=True) + os.chdir(home) + except Exception as e: + self.log.exception("Error in preexec for %s", name) + return preexec + + def user_env(self, env): + env['USER'] = self.user.name + env['HOME'] = self.home_dir + env['SHELL'] = '/bin/bash' + return env + + def move_certs(self, paths): + """No-op for installing certs""" + return paths + diff --git a/jupyterhub/tests/conftest.py b/jupyterhub/tests/conftest.py index e44dd60c..c5f208f8 100644 --- a/jupyterhub/tests/conftest.py +++ b/jupyterhub/tests/conftest.py @@ -29,6 +29,7 @@ Fixtures to add functionality or spawning behavior import asyncio from getpass import getuser +import inspect import logging import os import sys @@ -46,7 +47,7 @@ from ..utils import random_port from . import mocking from .mocking import MockHub -from .utils import ssl_setup +from .utils import ssl_setup, add_user from .test_services import mockservice_cmd import jupyterhub.services.service @@ -55,6 +56,16 @@ import jupyterhub.services.service _db = None +def pytest_collection_modifyitems(items): + """add asyncio marker to all async tests""" + for item in items: + if inspect.iscoroutinefunction(item.obj): + item.add_marker('asyncio') + if hasattr(inspect, 'isasyncgenfunction'): + # double-check that we aren't mixing yield and async def + assert not inspect.isasyncgenfunction(item.obj) + + @fixture(scope='module') def ssl_tmpdir(tmpdir_factory): return tmpdir_factory.mktemp('ssl') @@ -65,7 +76,7 @@ def app(request, io_loop, ssl_tmpdir): """Mock a jupyterhub app for testing""" mocked_app = None ssl_enabled = getattr(request.module, "ssl_enabled", False) - kwargs = dict(log_level=logging.DEBUG) + kwargs = dict() if ssl_enabled: kwargs.update( dict( @@ -126,15 +137,21 @@ def db(): @fixture(scope='module') -def io_loop(request): +def event_loop(request): + """Same as pytest-asyncio.event_loop, but re-scoped to module-level""" + event_loop = asyncio.new_event_loop() + asyncio.set_event_loop(event_loop) + return event_loop + + +@fixture(scope='module') +def io_loop(event_loop, request): """Same as pytest-tornado.io_loop, but re-scoped to module-level""" ioloop.IOLoop.configure(AsyncIOMainLoop) - aio_loop = asyncio.new_event_loop() - asyncio.set_event_loop(aio_loop) - io_loop = ioloop.IOLoop() + io_loop = AsyncIOMainLoop() io_loop.make_current() - assert asyncio.get_event_loop() is aio_loop - assert io_loop.asyncio_loop is aio_loop + assert asyncio.get_event_loop() is event_loop + assert io_loop.asyncio_loop is event_loop def _close(): io_loop.clear_current() @@ -168,6 +185,43 @@ def cleanup_after(request, io_loop): app.db.commit() +_username_counter = 0 + + +def new_username(prefix='testuser'): + """Return a new unique username""" + global _username_counter + _username_counter += 1 + return '{}-{}'.format(prefix, _username_counter) + + +@fixture +def username(): + """allocate a temporary username + + unique each time the fixture is used + """ + yield new_username() + + +@fixture +def user(app): + """Fixture for creating a temporary user + + Each time the fixture is used, a new user is created + """ + user = add_user(app.db, app, name=new_username()) + yield user + + +@fixture +def admin_user(app, username): + """Fixture for creating a temporary admin user""" + user = add_user(app.db, app, name=new_username('testadmin'), admin=True) + yield user + + + class MockServiceSpawner(jupyterhub.services.service._ServiceSpawner): """mock services for testing. diff --git a/jupyterhub/tests/mocking.py b/jupyterhub/tests/mocking.py index b8749ba3..d2a2bb9a 100644 --- a/jupyterhub/tests/mocking.py +++ b/jupyterhub/tests/mocking.py @@ -40,16 +40,16 @@ from tornado import gen from tornado.concurrent import Future from tornado.ioloop import IOLoop -from traitlets import Bool, default +from traitlets import Bool, Dict, default from ..app import JupyterHub from ..auth import PAMAuthenticator from .. import orm from ..objects import Server -from ..spawner import LocalProcessSpawner +from ..spawner import LocalProcessSpawner, SimpleLocalProcessSpawner from ..singleuser import SingleUserNotebookApp from ..utils import random_port, url_path_join -from .utils import async_requests, ssl_setup +from .utils import async_requests, ssl_setup, public_host, public_url from pamela import PAMError @@ -73,20 +73,14 @@ def mock_open_session(username, service, encoding): pass -class MockSpawner(LocalProcessSpawner): +class MockSpawner(SimpleLocalProcessSpawner): """Base mock spawner - disables user-switching that we need root permissions to do - spawns `jupyterhub.tests.mocksu` instead of a full single-user server """ - def make_preexec_fn(self, *a, **kw): - # skip the setuid stuff - return - - def _set_user_changed(self, name, old, new): - pass - def user_env(self, env): + env = super().user_env(env) if self.handler: env['HANDLER_ARGS'] = self.handler.request.query return env @@ -95,10 +89,6 @@ class MockSpawner(LocalProcessSpawner): def _cmd_default(self): return [sys.executable, '-m', 'jupyterhub.tests.mocksu'] - def move_certs(self, paths): - """Return the paths unmodified""" - return paths - use_this_api_token = None def start(self): if self.use_this_api_token: @@ -177,6 +167,11 @@ class FormSpawner(MockSpawner): options['hello'] = form_data['hello_file'][0] return options +class FalsyCallableFormSpawner(FormSpawner): + """A spawner that has a callable options form defined returning a falsy value""" + + options_form = lambda a, b: "" + class MockStructGroup: """Mock grp.struct_group""" @@ -229,11 +224,17 @@ class MockPAMAuthenticator(PAMAuthenticator): class MockHub(JupyterHub): """Hub with various mock bits""" + # disable some inherited traits with hardcoded values db_file = None last_activity_interval = 2 log_datefmt = '%M:%S' - external_certs = None - log_level = 10 + + @default('log_level') + def _default_log_level(self): + return 10 + + # MockHub additional traits + external_certs = Dict() def __init__(self, *args, **kwargs): if 'internal_certs_location' in kwargs: @@ -361,31 +362,6 @@ class MockHub(JupyterHub): return r.cookies -def public_host(app): - """Return the public *host* (no URL prefix) of the given JupyterHub instance.""" - if app.subdomain_host: - return app.subdomain_host - else: - return Server.from_url(app.proxy.public_url).host - - -def public_url(app, user_or_service=None, path=''): - """Return the full, public base URL (including prefix) of the given JupyterHub instance.""" - if user_or_service: - if app.subdomain_host: - host = user_or_service.host - else: - host = public_host(app) - prefix = user_or_service.prefix - else: - host = public_host(app) - prefix = Server.from_url(app.proxy.public_url).base_url - if path: - return host + url_path_join(prefix, path) - else: - return host + prefix - - # single-user-server mocking: class MockSingleUserServer(SingleUserNotebookApp): diff --git a/jupyterhub/tests/test_api.py b/jupyterhub/tests/test_api.py index d4e91cd0..48cf2eda 100644 --- a/jupyterhub/tests/test_api.py +++ b/jupyterhub/tests/test_api.py @@ -1,6 +1,7 @@ """Tests for the REST API.""" -from datetime import datetime +import asyncio +from datetime import datetime, timedelta from concurrent.futures import Future import json import re @@ -15,100 +16,15 @@ from tornado import gen import jupyterhub from .. import orm -from ..utils import url_path_join as ujoin +from ..utils import url_path_join as ujoin, utcnow from .mocking import public_host, public_url -from .utils import async_requests - - -def check_db_locks(func): - """Decorator that verifies no locks are held on database upon exit. - - This decorator for test functions verifies no locks are held on the - application's database upon exit by creating and dropping a dummy table. - - The decorator relies on an instance of JupyterHubApp being the first - argument to the decorated function. - - Example - ------- - - @check_db_locks - def api_request(app, *api_path, **kwargs): - - """ - def new_func(app, *args, **kwargs): - retval = func(app, *args, **kwargs) - - temp_session = app.session_factory() - temp_session.execute('CREATE TABLE dummy (foo INT)') - temp_session.execute('DROP TABLE dummy') - temp_session.close() - - return retval - - return new_func - - -def find_user(db, name, app=None): - """Find user in database.""" - orm_user = db.query(orm.User).filter(orm.User.name == name).first() - if app is None: - return orm_user - else: - return app.users[orm_user.id] - - -def add_user(db, app=None, **kwargs): - """Add a user to the database.""" - orm_user = find_user(db, name=kwargs.get('name')) - if orm_user is None: - orm_user = orm.User(**kwargs) - db.add(orm_user) - else: - for attr, value in kwargs.items(): - setattr(orm_user, attr, value) - db.commit() - if app: - return app.users[orm_user.id] - else: - return orm_user - - -def auth_header(db, name): - """Return header with user's API authorization token.""" - user = find_user(db, name) - if user is None: - user = add_user(db, name=name) - token = user.new_api_token() - return {'Authorization': 'token %s' % token} - - -@check_db_locks -@gen.coroutine -def api_request(app, *api_path, **kwargs): - """Make an API request""" - base_url = app.hub.url - headers = kwargs.setdefault('headers', {}) - - if 'Authorization' not in headers and not kwargs.pop('noauth', False): - # make a copy to avoid modifying arg in-place - kwargs['headers'] = h = {} - h.update(headers) - h.update(auth_header(app.db, 'admin')) - - url = ujoin(base_url, 'api', *api_path) - method = kwargs.pop('method', 'get') - f = getattr(async_requests, method) - if app.internal_ssl: - kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key) - kwargs["verify"] = app.internal_ssl_ca - resp = yield f(url, **kwargs) - assert "frame-ancestors 'self'" in resp.headers['Content-Security-Policy'] - assert ujoin(app.hub.base_url, "security/csp-report") in resp.headers['Content-Security-Policy'] - assert 'http' not in resp.headers['Content-Security-Policy'] - if not kwargs.get('stream', False) and resp.content: - assert resp.headers.get('content-type') == 'application/json' - return resp +from .utils import ( + add_user, + api_request, + async_requests, + auth_header, + find_user, +) # -------------------- @@ -116,10 +32,9 @@ def api_request(app, *api_path, **kwargs): # -------------------- -@mark.gen_test -def test_auth_api(app): +async def test_auth_api(app): db = app.db - r = yield api_request(app, 'authorizations', 'gobbledygook') + r = await api_request(app, 'authorizations', 'gobbledygook') assert r.status_code == 404 # make a new cookie token @@ -127,33 +42,32 @@ def test_auth_api(app): api_token = user.new_api_token() # check success: - r = yield api_request(app, 'authorizations/token', api_token) + r = await api_request(app, 'authorizations/token', api_token) assert r.status_code == 200 reply = r.json() assert reply['name'] == user.name # check fail - r = yield api_request(app, 'authorizations/token', api_token, + r = await api_request(app, 'authorizations/token', api_token, headers={'Authorization': 'no sir'}, ) assert r.status_code == 403 - r = yield api_request(app, 'authorizations/token', api_token, + r = await api_request(app, 'authorizations/token', api_token, headers={'Authorization': 'token: %s' % user.cookie_id}, ) assert r.status_code == 403 -@mark.gen_test -def test_referer_check(app): +async def test_referer_check(app): url = ujoin(public_host(app), app.hub.base_url) host = urlparse(url).netloc user = find_user(app.db, 'admin') if user is None: user = add_user(app.db, name='admin', admin=True) - cookies = yield app.login_user('admin') + cookies = await app.login_user('admin') - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers={ 'Authorization': '', 'Referer': 'null', @@ -161,7 +75,7 @@ def test_referer_check(app): ) assert r.status_code == 403 - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers={ 'Authorization': '', 'Referer': 'http://attack.com/csrf/vulnerability', @@ -169,7 +83,7 @@ def test_referer_check(app): ) assert r.status_code == 403 - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers={ 'Authorization': '', 'Referer': url, @@ -178,7 +92,7 @@ def test_referer_check(app): ) assert r.status_code == 200 - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers={ 'Authorization': '', 'Referer': ujoin(url, 'foo/bar/baz/bat'), @@ -199,7 +113,7 @@ def normalize_timestamp(ts): """ if ts is None: return - return re.sub('\d(\.\d+)?', '0', ts) + return re.sub(r'\d(\.\d+)?', '0', ts) def normalize_user(user): @@ -240,10 +154,9 @@ def fill_user(model): TIMESTAMP = normalize_timestamp(datetime.now().isoformat() + 'Z') @mark.user -@mark.gen_test -def test_get_users(app): +async def test_get_users(app): db = app.db - r = yield api_request(app, 'users') + r = await api_request(app, 'users') assert r.status_code == 200 users = sorted(r.json(), key=lambda d: d['name']) @@ -260,19 +173,18 @@ def test_get_users(app): }), ] - r = yield api_request(app, 'users', + r = await api_request(app, 'users', headers=auth_header(db, 'user'), ) assert r.status_code == 403 @mark.user -@mark.gen_test -def test_get_self(app): +async def test_get_self(app): db = app.db # basic get self - r = yield api_request(app, 'user') + r = await api_request(app, 'user') r.raise_for_status() assert r.json()['kind'] == 'user' @@ -290,7 +202,7 @@ def test_get_self(app): ) db.add(oauth_token) db.commit() - r = yield api_request(app, 'user', headers={ + r = await api_request(app, 'user', headers={ 'Authorization': 'token ' + token, }) r.raise_for_status() @@ -298,18 +210,17 @@ def test_get_self(app): assert model['name'] == u.name # invalid auth gets 403 - r = yield api_request(app, 'user', headers={ + r = await api_request(app, 'user', headers={ 'Authorization': 'token notvalid', }) assert r.status_code == 403 @mark.user -@mark.gen_test -def test_add_user(app): +async def test_add_user(app): db = app.db name = 'newuser' - r = yield api_request(app, 'users', name, method='post') + r = await api_request(app, 'users', name, method='post') assert r.status_code == 201 user = find_user(db, name) assert user is not None @@ -318,10 +229,9 @@ def test_add_user(app): @mark.user -@mark.gen_test -def test_get_user(app): +async def test_get_user(app): name = 'user' - r = yield api_request(app, 'users', name) + r = await api_request(app, 'users', name) assert r.status_code == 200 user = normalize_user(r.json()) @@ -329,21 +239,19 @@ def test_get_user(app): @mark.user -@mark.gen_test -def test_add_multi_user_bad(app): - r = yield api_request(app, 'users', method='post') +async def test_add_multi_user_bad(app): + r = await api_request(app, 'users', method='post') assert r.status_code == 400 - r = yield api_request(app, 'users', method='post', data='{}') + r = await api_request(app, 'users', method='post', data='{}') assert r.status_code == 400 - r = yield api_request(app, 'users', method='post', data='[]') + r = await api_request(app, 'users', method='post', data='[]') assert r.status_code == 400 @mark.user -@mark.gen_test -def test_add_multi_user_invalid(app): +async def test_add_multi_user_invalid(app): app.authenticator.username_pattern = r'w.*' - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': ['Willow', 'Andrew', 'Tara']}) ) app.authenticator.username_pattern = '' @@ -352,11 +260,10 @@ def test_add_multi_user_invalid(app): @mark.user -@mark.gen_test -def test_add_multi_user(app): +async def test_add_multi_user(app): db = app.db names = ['a', 'b'] - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': names}), ) assert r.status_code == 201 @@ -371,7 +278,7 @@ def test_add_multi_user(app): assert not user.admin # try to create the same users again - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': names}), ) assert r.status_code == 409 @@ -379,7 +286,7 @@ def test_add_multi_user(app): names = ['a', 'b', 'ab'] # try to create the same users again - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': names}), ) assert r.status_code == 201 @@ -389,11 +296,10 @@ def test_add_multi_user(app): @mark.user -@mark.gen_test -def test_add_multi_user_admin(app): +async def test_add_multi_user_admin(app): db = app.db names = ['c', 'd'] - r = yield api_request(app, 'users', method='post', + r = await api_request(app, 'users', method='post', data=json.dumps({'usernames': names, 'admin': True}), ) assert r.status_code == 201 @@ -409,35 +315,32 @@ def test_add_multi_user_admin(app): @mark.user -@mark.gen_test -def test_add_user_bad(app): +async def test_add_user_bad(app): db = app.db name = 'dne_newuser' - r = yield api_request(app, 'users', name, method='post') + r = await api_request(app, 'users', name, method='post') assert r.status_code == 400 user = find_user(db, name) assert user is None @mark.user -@mark.gen_test -def test_add_user_duplicate(app): +async def test_add_user_duplicate(app): db = app.db name = 'user' user = find_user(db, name) # double-check that it exists assert user is not None - r = yield api_request(app, 'users', name, method='post') + r = await api_request(app, 'users', name, method='post') # special 409 conflict for creating a user that already exists assert r.status_code == 409 @mark.user -@mark.gen_test -def test_add_admin(app): +async def test_add_admin(app): db = app.db name = 'newadmin' - r = yield api_request(app, 'users', name, method='post', + r = await api_request(app, 'users', name, method='post', data=json.dumps({'admin': True}), ) assert r.status_code == 201 @@ -448,27 +351,25 @@ def test_add_admin(app): @mark.user -@mark.gen_test -def test_delete_user(app): +async def test_delete_user(app): db = app.db mal = add_user(db, name='mal') - r = yield api_request(app, 'users', 'mal', method='delete') + r = await api_request(app, 'users', 'mal', method='delete') assert r.status_code == 204 @mark.user -@mark.gen_test -def test_make_admin(app): +async def test_make_admin(app): db = app.db name = 'admin2' - r = yield api_request(app, 'users', name, method='post') + r = await api_request(app, 'users', name, method='post') assert r.status_code == 201 user = find_user(db, name) assert user is not None assert user.name == name assert not user.admin - r = yield api_request(app, 'users', name, method='patch', + r = await api_request(app, 'users', name, method='patch', data=json.dumps({'admin': True}) ) assert r.status_code == 200 @@ -479,8 +380,7 @@ def test_make_admin(app): @mark.user -@mark.gen_test -def test_set_auth_state(app, auth_state_enabled): +async def test_set_auth_state(app, auth_state_enabled): auth_state = {'secret': 'hello'} db = app.db name = 'admin' @@ -488,58 +388,55 @@ def test_set_auth_state(app, auth_state_enabled): assert user is not None assert user.name == name - r = yield api_request(app, 'users', name, method='patch', + r = await api_request(app, 'users', name, method='patch', data=json.dumps({'auth_state': auth_state}) ) assert r.status_code == 200 - users_auth_state = yield user.get_auth_state() + users_auth_state = await user.get_auth_state() assert users_auth_state == auth_state @mark.user -@mark.gen_test -def test_user_set_auth_state(app, auth_state_enabled): +async def test_user_set_auth_state(app, auth_state_enabled): auth_state = {'secret': 'hello'} db = app.db name = 'user' user = find_user(db, name, app=app) assert user is not None assert user.name == name - user_auth_state = yield user.get_auth_state() + user_auth_state = await user.get_auth_state() assert user_auth_state is None - r = yield api_request( + r = await api_request( app, 'users', name, method='patch', data=json.dumps({'auth_state': auth_state}), headers=auth_header(app.db, name), ) assert r.status_code == 403 - user_auth_state = yield user.get_auth_state() + user_auth_state = await user.get_auth_state() assert user_auth_state is None @mark.user -@mark.gen_test -def test_admin_get_auth_state(app, auth_state_enabled): +async def test_admin_get_auth_state(app, auth_state_enabled): auth_state = {'secret': 'hello'} db = app.db name = 'admin' user = find_user(db, name, app=app) assert user is not None assert user.name == name - yield user.save_auth_state(auth_state) + await user.save_auth_state(auth_state) - r = yield api_request(app, 'users', name) + r = await api_request(app, 'users', name) assert r.status_code == 200 assert r.json()['auth_state'] == auth_state @mark.user -@mark.gen_test -def test_user_get_auth_state(app, auth_state_enabled): +async def test_user_get_auth_state(app, auth_state_enabled): # explicitly check that a user will not get their own auth state via the API auth_state = {'secret': 'hello'} db = app.db @@ -547,17 +444,16 @@ def test_user_get_auth_state(app, auth_state_enabled): user = find_user(db, name, app=app) assert user is not None assert user.name == name - yield user.save_auth_state(auth_state) + await user.save_auth_state(auth_state) - r = yield api_request(app, 'users', name, + r = await api_request(app, 'users', name, headers=auth_header(app.db, name)) assert r.status_code == 200 assert 'auth_state' not in r.json() -@mark.gen_test -def test_spawn(app): +async def test_spawn(app): db = app.db name = 'wash' user = add_user(db, app=app, name=name) @@ -566,7 +462,7 @@ def test_spawn(app): 'i': 5, } before_servers = sorted(db.query(orm.Server), key=lambda s: s.url) - r = yield api_request(app, 'users', name, 'server', method='post', + r = await api_request(app, 'users', name, 'server', method='post', data=json.dumps(options), ) assert r.status_code == 201 @@ -576,7 +472,7 @@ def test_spawn(app): spawner = app_user.spawner assert app_user.spawner.user_options == options assert not app_user.spawner._spawn_pending - status = yield app_user.spawner.poll() + status = await app_user.spawner.poll() assert status is None assert spawner.server.base_url == ujoin(app.base_url, 'user/%s' % name) + '/' @@ -585,26 +481,26 @@ def test_spawn(app): if app.internal_ssl: kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key) kwargs["verify"] = app.internal_ssl_ca - r = yield async_requests.get(url, **kwargs) + r = await async_requests.get(url, **kwargs) assert r.status_code == 200 assert r.text == spawner.server.base_url - r = yield async_requests.get(ujoin(url, 'args'), **kwargs) + r = await async_requests.get(ujoin(url, 'args'), **kwargs) assert r.status_code == 200 argv = r.json() assert '--port' in ' '.join(argv) - r = yield async_requests.get(ujoin(url, 'env'), **kwargs) + r = await async_requests.get(ujoin(url, 'env'), **kwargs) env = r.json() for expected in ['JUPYTERHUB_USER', 'JUPYTERHUB_BASE_URL', 'JUPYTERHUB_API_TOKEN']: assert expected in env if app.subdomain_host: assert env['JUPYTERHUB_HOST'] == app.subdomain_host - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') assert r.status_code == 204 assert 'pid' not in user.orm_spawners[''].state - status = yield app_user.spawner.poll() + status = await app_user.spawner.poll() assert status == 0 # check that we cleaned up after ourselves @@ -616,8 +512,7 @@ def test_spawn(app): assert app.users.count_active_users()['pending'] == 0 -@mark.gen_test -def test_spawn_handler(app): +async def test_spawn_handler(app): """Test that the requesting Handler is passed to Spawner.handler""" db = app.db name = 'salmon' @@ -625,7 +520,7 @@ def test_spawn_handler(app): app_user = app.users[name] # spawn via API with ?foo=bar - r = yield api_request(app, 'users', name, 'server', method='post', params={'foo': 'bar'}) + r = await api_request(app, 'users', name, 'server', method='post', params={'foo': 'bar'}) r.raise_for_status() # verify that request params got passed down @@ -634,24 +529,23 @@ def test_spawn_handler(app): if app.external_certs: kwargs['verify'] = app.external_certs['files']['ca'] url = public_url(app, user) - r = yield async_requests.get(ujoin(url, 'env'), **kwargs) + r = await async_requests.get(ujoin(url, 'env'), **kwargs) env = r.json() assert 'HANDLER_ARGS' in env assert env['HANDLER_ARGS'] == 'foo=bar' # make user spawner.handler doesn't persist after spawn finishes assert app_user.spawner.handler is None - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() @mark.slow -@mark.gen_test -def test_slow_spawn(app, no_patience, slow_spawn): +async def test_slow_spawn(app, no_patience, slow_spawn): db = app.db name = 'zoe' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() assert r.status_code == 202 assert app_user.spawner is not None @@ -659,83 +553,78 @@ def test_slow_spawn(app, no_patience, slow_spawn): assert not app_user.spawner._stop_pending assert app.users.count_active_users()['pending'] == 1 - @gen.coroutine - def wait_spawn(): + async def wait_spawn(): while not app_user.running: - yield gen.sleep(0.1) + await gen.sleep(0.1) - yield wait_spawn() + await wait_spawn() assert not app_user.spawner._spawn_pending - status = yield app_user.spawner.poll() + status = await app_user.spawner.poll() assert status is None - @gen.coroutine - def wait_stop(): + async def wait_stop(): while app_user.spawner._stop_pending: - yield gen.sleep(0.1) + await gen.sleep(0.1) - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() assert r.status_code == 202 assert app_user.spawner is not None assert app_user.spawner._stop_pending - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() assert r.status_code == 202 assert app_user.spawner is not None assert app_user.spawner._stop_pending - yield wait_stop() + await wait_stop() assert not app_user.spawner._stop_pending assert app_user.spawner is not None - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') # 204 deleted if there's no such server assert r.status_code == 204 assert app.users.count_active_users()['pending'] == 0 assert app.users.count_active_users()['active'] == 0 -@mark.gen_test -def test_never_spawn(app, no_patience, never_spawn): +async def test_never_spawn(app, no_patience, never_spawn): db = app.db name = 'badger' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert app_user.spawner is not None assert app_user.spawner._spawn_pending assert app.users.count_active_users()['pending'] == 1 while app_user.spawner.pending: - yield gen.sleep(0.1) + await gen.sleep(0.1) print(app_user.spawner.pending) assert not app_user.spawner._spawn_pending - status = yield app_user.spawner.poll() + status = await app_user.spawner.poll() assert status is not None # failed spawn should decrements pending count assert app.users.count_active_users()['pending'] == 0 -@mark.gen_test -def test_bad_spawn(app, no_patience, bad_spawn): +async def test_bad_spawn(app, bad_spawn): db = app.db name = 'prim' user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 500 assert app.users.count_active_users()['pending'] == 0 -@mark.gen_test -def test_slow_bad_spawn(app, no_patience, slow_bad_spawn): +async def test_slow_bad_spawn(app, no_patience, slow_bad_spawn): db = app.db name = 'zaphod' user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() while user.spawner.pending: - yield gen.sleep(0.1) + await gen.sleep(0.1) # spawn failed assert not user.running assert app.users.count_active_users()['pending'] == 0 @@ -752,31 +641,30 @@ def next_event(it): return json.loads(line.split(':', 1)[1]) @mark.slow -@mark.gen_test -def test_progress(request, app, no_patience, slow_spawn): +async def test_progress(request, app, no_patience, slow_spawn): db = app.db name = 'martin' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) assert r.headers['content-type'] == 'text/event-stream' ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 0, 'message': 'Server requested', } - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 50, 'message': 'Spawning server...', } - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) url = app_user.url assert evt == { 'progress': 100, @@ -787,32 +675,29 @@ def test_progress(request, app, no_patience, slow_spawn): } -@mark.gen_test -def test_progress_not_started(request, app): +async def test_progress_not_started(request, app): db = app.db name = 'nope' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server/progress') + r = await api_request(app, 'users', name, 'server/progress') assert r.status_code == 404 -@mark.gen_test -def test_progress_not_found(request, app): +async def test_progress_not_found(request, app): db = app.db name = 'noserver' - r = yield api_request(app, 'users', 'nosuchuser', 'server/progress') + r = await api_request(app, 'users', 'nosuchuser', 'server/progress') assert r.status_code == 404 app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server/progress') + r = await api_request(app, 'users', name, 'server/progress') assert r.status_code == 404 -@mark.gen_test -def test_progress_ready(request, app): +async def test_progress_ready(request, app): """Test progress API when spawner is already started e.g. a race between requesting progress and progress already being complete @@ -820,35 +705,34 @@ def test_progress_ready(request, app): db = app.db name = 'saga' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) assert r.headers['content-type'] == 'text/event-stream' ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 100 assert evt['ready'] assert evt['url'] == app_user.url -@mark.gen_test -def test_progress_bad(request, app, no_patience, bad_spawn): +async def test_progress_bad(request, app, bad_spawn): """Test progress API when spawner has already failed""" db = app.db name = 'simon' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 500 - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) assert r.headers['content-type'] == 'text/event-stream' ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 100, 'failed': True, @@ -856,25 +740,24 @@ def test_progress_bad(request, app, no_patience, bad_spawn): } -@mark.gen_test -def test_progress_bad_slow(request, app, no_patience, slow_bad_spawn): +async def test_progress_bad_slow(request, app, no_patience, slow_bad_spawn): """Test progress API when spawner fails while watching""" db = app.db name = 'eugene' app_user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 202 - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) assert r.headers['content-type'] == 'text/event-stream' ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 0 - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 50 - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 100, 'failed': True, @@ -910,8 +793,7 @@ async def progress_forever_native(): """, globals()) -@mark.gen_test -def test_spawn_progress_cutoff(request, app, no_patience, slow_spawn): +async def test_spawn_progress_cutoff(request, app, no_patience, slow_spawn): """Progress events stop when Spawner finishes even if progress iterator is still going. @@ -926,26 +808,25 @@ def test_spawn_progress_cutoff(request, app, no_patience, slow_spawn): app_user.spawner.progress = progress_forever app_user.spawner.delay = 1 - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - r = yield api_request(app, 'users', name, 'server/progress', stream=True) + r = await api_request(app, 'users', name, 'server/progress', stream=True) r.raise_for_status() request.addfinalizer(r.close) ex = async_requests.executor line_iter = iter(r.iter_lines(decode_unicode=True)) - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 0 - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt == { 'progress': 1, 'message': 'Stage 1', } - evt = yield ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) assert evt['progress'] == 100 -@mark.gen_test -def test_spawn_limit(app, no_patience, slow_spawn, request): +async def test_spawn_limit(app, no_patience, slow_spawn, request): db = app.db p = mock.patch.dict(app.tornado_settings, {'concurrent_spawn_limit': 2}) @@ -958,24 +839,24 @@ def test_spawn_limit(app, no_patience, slow_spawn, request): users[0].spawner._start_future = Future() users[1].spawner._start_future = Future() for name in names: - yield api_request(app, 'users', name, 'server', method='post') + await api_request(app, 'users', name, 'server', method='post') assert app.users.count_active_users()['pending'] == 2 # ykka and hjarka's spawns are both pending. Essun should fail with 429 name = 'essun' user = add_user(db, app=app, name=name) user.spawner._start_future = Future() - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 429 # allow ykka to start users[0].spawner._start_future.set_result(None) # wait for ykka to finish while not users[0].running: - yield gen.sleep(0.1) + await gen.sleep(0.1) assert app.users.count_active_users()['pending'] == 1 - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() assert app.users.count_active_users()['pending'] == 2 users.append(user) @@ -983,20 +864,19 @@ def test_spawn_limit(app, no_patience, slow_spawn, request): for user in users[1:]: user.spawner._start_future.set_result(None) while not all(u.running for u in users): - yield gen.sleep(0.1) + await gen.sleep(0.1) # everybody's running, pending count should be back to 0 assert app.users.count_active_users()['pending'] == 0 for u in users: u.spawner.delay = 0 - r = yield api_request(app, 'users', u.name, 'server', method='delete') + r = await api_request(app, 'users', u.name, 'server', method='delete') r.raise_for_status() while any(u.spawner.active for u in users): - yield gen.sleep(0.1) + await gen.sleep(0.1) @mark.slow -@mark.gen_test -def test_active_server_limit(app, request): +async def test_active_server_limit(app, request): db = app.db p = mock.patch.dict(app.tornado_settings, {'active_server_limit': 2}) @@ -1007,7 +887,7 @@ def test_active_server_limit(app, request): names = ['ykka', 'hjarka'] users = [ add_user(db, app=app, name=name) for name in names ] for name in names: - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() counts = app.users.count_active_users() assert counts['active'] == 2 @@ -1017,7 +897,7 @@ def test_active_server_limit(app, request): # ykka and hjarka's servers are running. Essun should fail with 429 name = 'essun' user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 429 counts = app.users.count_active_users() assert counts['active'] == 2 @@ -1025,13 +905,13 @@ def test_active_server_limit(app, request): assert counts['pending'] == 0 # stop one server - yield api_request(app, 'users', names[0], 'server', method='delete') + await api_request(app, 'users', names[0], 'server', method='delete') counts = app.users.count_active_users() assert counts['active'] == 1 assert counts['ready'] == 1 assert counts['pending'] == 0 - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') r.raise_for_status() counts = app.users.count_active_users() assert counts['active'] == 2 @@ -1044,7 +924,7 @@ def test_active_server_limit(app, request): for u in users: if not u.spawner.active: continue - r = yield api_request(app, 'users', u.name, 'server', method='delete') + r = await api_request(app, 'users', u.name, 'server', method='delete') r.raise_for_status() counts = app.users.count_active_users() @@ -1053,78 +933,75 @@ def test_active_server_limit(app, request): assert counts['pending'] == 0 @mark.slow -@mark.gen_test -def test_start_stop_race(app, no_patience, slow_spawn): +async def test_start_stop_race(app, no_patience, slow_spawn): user = add_user(app.db, app, name='panda') spawner = user.spawner # start the server - r = yield api_request(app, 'users', user.name, 'server', method='post') + r = await api_request(app, 'users', user.name, 'server', method='post') assert r.status_code == 202 assert spawner.pending == 'spawn' # additional spawns while spawning shouldn't trigger a new spawn with mock.patch.object(spawner, 'start') as m: - r = yield api_request(app, 'users', user.name, 'server', method='post') + r = await api_request(app, 'users', user.name, 'server', method='post') assert r.status_code == 202 assert m.call_count == 0 # stop while spawning is not okay - r = yield api_request(app, 'users', user.name, 'server', method='delete') + r = await api_request(app, 'users', user.name, 'server', method='delete') assert r.status_code == 400 while not spawner.ready: - yield gen.sleep(0.1) + await gen.sleep(0.1) spawner.delay = 3 # stop the spawner - r = yield api_request(app, 'users', user.name, 'server', method='delete') + r = await api_request(app, 'users', user.name, 'server', method='delete') assert r.status_code == 202 assert spawner.pending == 'stop' # make sure we get past deleting from the proxy - yield gen.sleep(1) + await gen.sleep(1) # additional stops while stopping shouldn't trigger a new stop with mock.patch.object(spawner, 'stop') as m: - r = yield api_request(app, 'users', user.name, 'server', method='delete') + r = await api_request(app, 'users', user.name, 'server', method='delete') assert r.status_code == 202 assert m.call_count == 0 # start while stopping is not allowed with mock.patch.object(spawner, 'start') as m: - r = yield api_request(app, 'users', user.name, 'server', method='post') + r = await api_request(app, 'users', user.name, 'server', method='post') assert r.status_code == 400 while spawner.active: - yield gen.sleep(0.1) + await gen.sleep(0.1) # start after stop is okay - r = yield api_request(app, 'users', user.name, 'server', method='post') + r = await api_request(app, 'users', user.name, 'server', method='post') assert r.status_code == 202 -@mark.gen_test -def test_get_proxy(app): - r = yield api_request(app, 'proxy') +async def test_get_proxy(app): + r = await api_request(app, 'proxy') r.raise_for_status() reply = r.json() assert list(reply.keys()) == [app.hub.routespec] -@mark.gen_test -def test_cookie(app): +async def test_cookie(app): db = app.db name = 'patience' user = add_user(db, app=app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post') assert r.status_code == 201 assert 'pid' in user.orm_spawners[''].state app_user = app.users[name] - cookies = yield app.login_user(name) + cookies = await app.login_user(name) cookie_name = app.hub.cookie_name # cookie jar gives '"cookie-value"', we want 'cookie-value' cookie = cookies[cookie_name][1:-1] - r = yield api_request(app, 'authorizations/cookie', + r = await api_request(app, 'authorizations/cookie', cookie_name, "nothintoseehere", ) assert r.status_code == 404 - r = yield api_request(app, 'authorizations/cookie', + r = await api_request(app, 'authorizations/cookie', cookie_name, quote(cookie, safe=''), ) r.raise_for_status() @@ -1132,7 +1009,7 @@ def test_cookie(app): assert reply['name'] == name # deprecated cookie in body: - r = yield api_request(app, 'authorizations/cookie', + r = await api_request(app, 'authorizations/cookie', cookie_name, data=cookie, ) r.raise_for_status() @@ -1146,27 +1023,25 @@ def normalize_token(token): return token -@mark.gen_test -def test_check_token(app): +async def test_check_token(app): name = 'book' user = add_user(app.db, app=app, name=name) token = user.new_api_token() - r = yield api_request(app, 'authorizations/token', token) + r = await api_request(app, 'authorizations/token', token) r.raise_for_status() user_model = r.json() assert user_model['name'] == name - r = yield api_request(app, 'authorizations/token', 'notauthorized') + r = await api_request(app, 'authorizations/token', 'notauthorized') assert r.status_code == 404 -@mark.gen_test @mark.parametrize("headers, status", [ ({}, 200), ({'Authorization': 'token bad'}, 403), ]) -def test_get_new_token_deprecated(app, headers, status): +async def test_get_new_token_deprecated(app, headers, status): # request a new token - r = yield api_request(app, 'authorizations', 'token', + r = await api_request(app, 'authorizations', 'token', method='post', headers=headers, ) @@ -1175,20 +1050,19 @@ def test_get_new_token_deprecated(app, headers, status): return reply = r.json() assert 'token' in reply - r = yield api_request(app, 'authorizations', 'token', reply['token']) + r = await api_request(app, 'authorizations', 'token', reply['token']) r.raise_for_status() reply = r.json() assert reply['name'] == 'admin' -@mark.gen_test -def test_token_formdata_deprecated(app): +async def test_token_formdata_deprecated(app): """Create a token for a user with formdata and no auth header""" data = { 'username': 'fake', 'password': 'fake', } - r = yield api_request(app, 'authorizations', 'token', + r = await api_request(app, 'authorizations', 'token', method='post', data=json.dumps(data) if data else None, noauth=True, @@ -1196,20 +1070,19 @@ def test_token_formdata_deprecated(app): assert r.status_code == 200 reply = r.json() assert 'token' in reply - r = yield api_request(app, 'authorizations', 'token', reply['token']) + r = await api_request(app, 'authorizations', 'token', reply['token']) r.raise_for_status() reply = r.json() assert reply['name'] == data['username'] -@mark.gen_test @mark.parametrize("as_user, for_user, status", [ ('admin', 'other', 200), ('admin', 'missing', 400), ('user', 'other', 403), ('user', 'user', 200), ]) -def test_token_as_user_deprecated(app, as_user, for_user, status): +async def test_token_as_user_deprecated(app, as_user, for_user, status): # ensure both users exist u = add_user(app.db, app, name=as_user) if for_user != 'missing': @@ -1218,7 +1091,7 @@ def test_token_as_user_deprecated(app, as_user, for_user, status): headers = { 'Authorization': 'token %s' % u.new_api_token(), } - r = yield api_request(app, 'authorizations', 'token', + r = await api_request(app, 'authorizations', 'token', method='post', data=json.dumps(data), headers=headers, @@ -1228,19 +1101,18 @@ def test_token_as_user_deprecated(app, as_user, for_user, status): if status != 200: return assert 'token' in reply - r = yield api_request(app, 'authorizations', 'token', reply['token']) + r = await api_request(app, 'authorizations', 'token', reply['token']) r.raise_for_status() reply = r.json() assert reply['name'] == data['username'] -@mark.gen_test @mark.parametrize("headers, status, note, expires_in", [ ({}, 200, 'test note', None), ({}, 200, '', 100), ({'Authorization': 'token bad'}, 403, '', None), ]) -def test_get_new_token(app, headers, status, note, expires_in): +async def test_get_new_token(app, headers, status, note, expires_in): options = {} if note: options['note'] = note @@ -1251,7 +1123,7 @@ def test_get_new_token(app, headers, status, note, expires_in): else: body = '' # request a new token - r = yield api_request(app, 'users/admin/tokens', + r = await api_request(app, 'users/admin/tokens', method='post', headers=headers, data=body, @@ -1279,28 +1151,27 @@ def test_get_new_token(app, headers, status, note, expires_in): initial.pop('token') # check the validity of the new token - r = yield api_request(app, 'users/admin/tokens', token_id) + r = await api_request(app, 'users/admin/tokens', token_id) r.raise_for_status() reply = r.json() assert normalize_token(reply) == initial # delete the token - r = yield api_request(app, 'users/admin/tokens', token_id, + r = await api_request(app, 'users/admin/tokens', token_id, method='delete') assert r.status_code == 204 # verify deletion - r = yield api_request(app, 'users/admin/tokens', token_id) + r = await api_request(app, 'users/admin/tokens', token_id) assert r.status_code == 404 -@mark.gen_test @mark.parametrize("as_user, for_user, status", [ ('admin', 'other', 200), ('admin', 'missing', 404), ('user', 'other', 403), ('user', 'user', 200), ]) -def test_token_for_user(app, as_user, for_user, status): +async def test_token_for_user(app, as_user, for_user, status): # ensure both users exist u = add_user(app.db, app, name=as_user) if for_user != 'missing': @@ -1309,7 +1180,7 @@ def test_token_for_user(app, as_user, for_user, status): headers = { 'Authorization': 'token %s' % u.new_api_token(), } - r = yield api_request(app, 'users', for_user, 'tokens', + r = await api_request(app, 'users', for_user, 'tokens', method='post', data=json.dumps(data), headers=headers, @@ -1320,7 +1191,7 @@ def test_token_for_user(app, as_user, for_user, status): return assert 'token' in reply token_id = reply['id'] - r = yield api_request(app, 'users', for_user, 'tokens', token_id, + r = await api_request(app, 'users', for_user, 'tokens', token_id, headers=headers, ) r.raise_for_status() @@ -1334,20 +1205,19 @@ def test_token_for_user(app, as_user, for_user, status): # delete the token - r = yield api_request(app, 'users', for_user, 'tokens', token_id, + r = await api_request(app, 'users', for_user, 'tokens', token_id, method='delete', headers=headers, ) assert r.status_code == 204 - r = yield api_request(app, 'users', for_user, 'tokens', token_id, + r = await api_request(app, 'users', for_user, 'tokens', token_id, headers=headers, ) assert r.status_code == 404 -@mark.gen_test -def test_token_authenticator_noauth(app): +async def test_token_authenticator_noauth(app): """Create a token for a user relying on Authenticator.authenticate and no auth header""" name = 'user' data = { @@ -1356,7 +1226,7 @@ def test_token_authenticator_noauth(app): 'password': name, }, } - r = yield api_request(app, 'users', name, 'tokens', + r = await api_request(app, 'users', name, 'tokens', method='post', data=json.dumps(data) if data else None, noauth=True, @@ -1364,27 +1234,52 @@ def test_token_authenticator_noauth(app): assert r.status_code == 200 reply = r.json() assert 'token' in reply - r = yield api_request(app, 'authorizations', 'token', reply['token']) + r = await api_request(app, 'authorizations', 'token', reply['token']) + r.raise_for_status() + reply = r.json() + assert reply['name'] == name + + +async def test_token_authenticator_dict_noauth(app): + """Create a token for a user relying on Authenticator.authenticate and no auth header""" + app.authenticator.auth_state = { + 'who': 'cares', + } + name = 'user' + data = { + 'auth': { + 'username': name, + 'password': name, + }, + } + r = await api_request(app, 'users', name, 'tokens', + method='post', + data=json.dumps(data) if data else None, + noauth=True, + ) + assert r.status_code == 200 + reply = r.json() + assert 'token' in reply + r = await api_request(app, 'authorizations', 'token', reply['token']) r.raise_for_status() reply = r.json() assert reply['name'] == name -@mark.gen_test @mark.parametrize("as_user, for_user, status", [ ('admin', 'other', 200), ('admin', 'missing', 404), ('user', 'other', 403), ('user', 'user', 200), ]) -def test_token_list(app, as_user, for_user, status): +async def test_token_list(app, as_user, for_user, status): u = add_user(app.db, app, name=as_user) if for_user != 'missing': for_user_obj = add_user(app.db, app, name=for_user) headers = { 'Authorization': 'token %s' % u.new_api_token(), } - r = yield api_request(app, 'users', for_user, 'tokens', + r = await api_request(app, 'users', for_user, 'tokens', headers=headers, ) assert r.status_code == status @@ -1397,7 +1292,7 @@ def test_token_list(app, as_user, for_user, status): assert all(token['user'] == for_user for token in reply['oauth_tokens']) # validate individual token ids for token in reply['api_tokens'] + reply['oauth_tokens']: - r = yield api_request(app, 'users', for_user, 'tokens', token['id'], + r = await api_request(app, 'users', for_user, 'tokens', token['id'], headers=headers, ) r.raise_for_status() @@ -1411,9 +1306,8 @@ def test_token_list(app, as_user, for_user, status): @mark.group -@mark.gen_test -def test_groups_list(app): - r = yield api_request(app, 'groups') +async def test_groups_list(app): + r = await api_request(app, 'groups') r.raise_for_status() reply = r.json() assert reply == [] @@ -1423,7 +1317,7 @@ def test_groups_list(app): app.db.add(group) app.db.commit() - r = yield api_request(app, 'groups') + r = await api_request(app, 'groups') r.raise_for_status() reply = r.json() assert reply == [{ @@ -1434,11 +1328,10 @@ def test_groups_list(app): @mark.group -@mark.gen_test -def test_add_multi_group(app): +async def test_add_multi_group(app): db = app.db names = ['group1', 'group2'] - r = yield api_request(app, 'groups', method='post', + r = await api_request(app, 'groups', method='post', data=json.dumps({'groups': names}), ) assert r.status_code == 201 @@ -1447,24 +1340,23 @@ def test_add_multi_group(app): assert names == r_names # try to create the same groups again - r = yield api_request(app, 'groups', method='post', + r = await api_request(app, 'groups', method='post', data=json.dumps({'groups': names}), ) assert r.status_code == 409 @mark.group -@mark.gen_test -def test_group_get(app): +async def test_group_get(app): group = orm.Group.find(app.db, name='alphaflight') user = add_user(app.db, app=app, name='sasquatch') group.users.append(user) app.db.commit() - r = yield api_request(app, 'groups/runaways') + r = await api_request(app, 'groups/runaways') assert r.status_code == 404 - r = yield api_request(app, 'groups/alphaflight') + r = await api_request(app, 'groups/alphaflight') r.raise_for_status() reply = r.json() assert reply == { @@ -1475,19 +1367,18 @@ def test_group_get(app): @mark.group -@mark.gen_test -def test_group_create_delete(app): +async def test_group_create_delete(app): db = app.db - r = yield api_request(app, 'groups/runaways', method='delete') + r = await api_request(app, 'groups/runaways', method='delete') assert r.status_code == 404 - r = yield api_request(app, 'groups/new', method='post', + r = await api_request(app, 'groups/new', method='post', data=json.dumps({'users': ['doesntexist']}), ) assert r.status_code == 400 assert orm.Group.find(db, name='new') is None - r = yield api_request(app, 'groups/omegaflight', method='post', + r = await api_request(app, 'groups/omegaflight', method='post', data=json.dumps({'users': ['sasquatch']}), ) r.raise_for_status() @@ -1498,30 +1389,29 @@ def test_group_create_delete(app): assert sasquatch in omegaflight.users # create duplicate raises 400 - r = yield api_request(app, 'groups/omegaflight', method='post') + r = await api_request(app, 'groups/omegaflight', method='post') assert r.status_code == 409 - r = yield api_request(app, 'groups/omegaflight', method='delete') + r = await api_request(app, 'groups/omegaflight', method='delete') assert r.status_code == 204 assert omegaflight not in sasquatch.groups assert orm.Group.find(db, name='omegaflight') is None # delete nonexistent gives 404 - r = yield api_request(app, 'groups/omegaflight', method='delete') + r = await api_request(app, 'groups/omegaflight', method='delete') assert r.status_code == 404 @mark.group -@mark.gen_test -def test_group_add_users(app): +async def test_group_add_users(app): db = app.db # must specify users - r = yield api_request(app, 'groups/alphaflight/users', method='post', data='{}') + r = await api_request(app, 'groups/alphaflight/users', method='post', data='{}') assert r.status_code == 400 names = ['aurora', 'guardian', 'northstar', 'sasquatch', 'shaman', 'snowbird'] users = [ find_user(db, name=name) or add_user(db, app=app, name=name) for name in names ] - r = yield api_request(app, 'groups/alphaflight/users', method='post', data=json.dumps({ + r = await api_request(app, 'groups/alphaflight/users', method='post', data=json.dumps({ 'users': names, })) r.raise_for_status() @@ -1535,16 +1425,15 @@ def test_group_add_users(app): @mark.group -@mark.gen_test -def test_group_delete_users(app): +async def test_group_delete_users(app): db = app.db # must specify users - r = yield api_request(app, 'groups/alphaflight/users', method='delete', data='{}') + r = await api_request(app, 'groups/alphaflight/users', method='delete', data='{}') assert r.status_code == 400 names = ['aurora', 'guardian', 'northstar', 'sasquatch', 'shaman', 'snowbird'] users = [ find_user(db, name=name) for name in names ] - r = yield api_request(app, 'groups/alphaflight/users', method='delete', data=json.dumps({ + r = await api_request(app, 'groups/alphaflight/users', method='delete', data=json.dumps({ 'users': names[:2], })) r.raise_for_status() @@ -1564,11 +1453,10 @@ def test_group_delete_users(app): @mark.services -@mark.gen_test -def test_get_services(app, mockservice_url): +async def test_get_services(app, mockservice_url): mockservice = mockservice_url db = app.db - r = yield api_request(app, 'services') + r = await api_request(app, 'services') r.raise_for_status() assert r.status_code == 200 @@ -1585,18 +1473,17 @@ def test_get_services(app, mockservice_url): } } - r = yield api_request(app, 'services', + r = await api_request(app, 'services', headers=auth_header(db, 'user'), ) assert r.status_code == 403 @mark.services -@mark.gen_test -def test_get_service(app, mockservice_url): +async def test_get_service(app, mockservice_url): mockservice = mockservice_url db = app.db - r = yield api_request(app, 'services/%s' % mockservice.name) + r = await api_request(app, 'services/%s' % mockservice.name) r.raise_for_status() assert r.status_code == 200 @@ -1611,27 +1498,26 @@ def test_get_service(app, mockservice_url): 'info': {}, } - r = yield api_request(app, 'services/%s' % mockservice.name, + r = await api_request(app, 'services/%s' % mockservice.name, headers={ 'Authorization': 'token %s' % mockservice.api_token } ) r.raise_for_status() - r = yield api_request(app, 'services/%s' % mockservice.name, + r = await api_request(app, 'services/%s' % mockservice.name, headers=auth_header(db, 'user'), ) assert r.status_code == 403 -@mark.gen_test -def test_root_api(app): +async def test_root_api(app): base_url = app.hub.url url = ujoin(base_url, 'api') kwargs = {} if app.internal_ssl: kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key) kwargs["verify"] = app.internal_ssl_ca - r = yield async_requests.get(url, **kwargs) + r = await async_requests.get(url, **kwargs) r.raise_for_status() expected = { 'version': jupyterhub.__version__ @@ -1639,9 +1525,8 @@ def test_root_api(app): assert r.json() == expected -@mark.gen_test -def test_info(app): - r = yield api_request(app, 'info') +async def test_info(app): + r = await api_request(app, 'info') r.raise_for_status() data = r.json() assert data['version'] == jupyterhub.__version__ @@ -1664,21 +1549,104 @@ def test_info(app): } +# ------------------ +# Activity API tests +# ------------------ + + +async def test_update_activity_403(app, user, admin_user): + token = user.new_api_token() + r = await api_request( + app, + "users/{}/activity".format(admin_user.name), + headers={"Authorization": "token {}".format(token)}, + data="{}", + method="post", + ) + assert r.status_code == 403 + + +async def test_update_activity_admin(app, user, admin_user): + token = admin_user.new_api_token() + r = await api_request( + app, + "users/{}/activity".format(user.name), + headers={"Authorization": "token {}".format(token)}, + data=json.dumps({"last_activity": utcnow().isoformat()}), + method="post", + ) + r.raise_for_status() + + +@mark.parametrize( + "server_name, fresh", + [ + ("", True), + ("", False), + ("exists", True), + ("exists", False), + ("nope", True), + ("nope", False), + ], +) +async def test_update_server_activity(app, user, server_name, fresh): + token = user.new_api_token() + now = utcnow() + internal_now = now.replace(tzinfo=None) + # we use naive utc internally + # initialize last_activity for one named and the default server + for name in ("", "exists"): + user.spawners[name].orm_spawner.last_activity = now.replace(tzinfo=None) + app.db.commit() + + td = timedelta(minutes=1) + if fresh: + activity = now + td + else: + activity = now - td + + r = await api_request( + app, + "users/{}/activity".format(user.name), + headers={"Authorization": "token {}".format(token)}, + data=json.dumps( + {"servers": {server_name: {"last_activity": activity.isoformat()}}} + ), + method="post", + ) + if server_name == "nope": + assert r.status_code == 400 + reply = r.json() + assert server_name in reply["message"] + assert "No such server" in reply["message"] + assert user.name in reply["message"] + return + + r.raise_for_status() + + # check that last activity was updated + + if fresh: + expected = activity.replace(tzinfo=None) + else: + expected = now.replace(tzinfo=None) + + assert user.spawners[server_name].orm_spawner.last_activity == expected + + # ----------------- # General API tests # ----------------- -@mark.gen_test -def test_options(app): - r = yield api_request(app, 'users', method='options') +async def test_options(app): + r = await api_request(app, 'users', method='options') r.raise_for_status() assert 'Access-Control-Allow-Headers' in r.headers -@mark.gen_test -def test_bad_json_body(app): - r = yield api_request(app, 'users', method='post', data='notjson') +async def test_bad_json_body(app): + r = await api_request(app, 'users', method='post', data='notjson') assert r.status_code == 400 @@ -1693,9 +1661,8 @@ def test_shutdown(app): # have to do things a little funky since we are going to stop the loop, # which makes gen_test unhappy. So we run the loop ourselves. - @gen.coroutine - def shutdown(): - r = yield api_request(app, 'shutdown', method='post', + async def shutdown(): + r = await api_request(app, 'shutdown', method='post', data=json.dumps({'servers': True, 'proxy': True,}), ) return r diff --git a/jupyterhub/tests/test_app.py b/jupyterhub/tests/test_app.py index c758489d..f34d3c5a 100644 --- a/jupyterhub/tests/test_app.py +++ b/jupyterhub/tests/test_app.py @@ -63,8 +63,7 @@ def test_generate_config(): assert 'Authenticator.whitelist' in cfg_text -@pytest.mark.gen_test -def test_init_tokens(request): +async def test_init_tokens(request): with TemporaryDirectory() as td: db_file = os.path.join(td, 'jupyterhub.sqlite') tokens = { @@ -77,7 +76,7 @@ def test_init_tokens(request): if ssl_enabled: kwargs['internal_certs_location'] = td app = MockHub(**kwargs) - yield app.initialize([]) + await app.initialize([]) db = app.db for token, username in tokens.items(): api_token = orm.APIToken.find(db, token) @@ -87,7 +86,7 @@ def test_init_tokens(request): # simulate second startup, reloading same tokens: app = MockHub(**kwargs) - yield app.initialize([]) + await app.initialize([]) db = app.db for token, username in tokens.items(): api_token = orm.APIToken.find(db, token) @@ -99,7 +98,7 @@ def test_init_tokens(request): tokens['short'] = 'gman' app = MockHub(**kwargs) with pytest.raises(ValueError): - yield app.initialize([]) + await app.initialize([]) assert orm.User.find(app.db, 'gman') is None @@ -169,8 +168,7 @@ def test_cookie_secret_env(tmpdir, request): assert not os.path.exists(hub.cookie_secret_file) -@pytest.mark.gen_test -def test_load_groups(tmpdir, request): +async def test_load_groups(tmpdir, request): to_load = { 'blue': ['cyclops', 'rogue', 'wolverine'], 'gold': ['storm', 'jean-grey', 'colossus'], @@ -181,8 +179,8 @@ def test_load_groups(tmpdir, request): kwargs['internal_certs_location'] = str(tmpdir) hub = MockHub(**kwargs) hub.init_db() - yield hub.init_users() - yield hub.init_groups() + await hub.init_users() + await hub.init_groups() db = hub.db blue = orm.Group.find(db, name='blue') assert blue is not None @@ -192,16 +190,15 @@ def test_load_groups(tmpdir, request): assert sorted([ u.name for u in gold.users ]) == sorted(to_load['gold']) -@pytest.mark.gen_test -def test_resume_spawners(tmpdir, request): +async def test_resume_spawners(tmpdir, request): if not os.getenv('JUPYTERHUB_TEST_DB_URL'): p = patch.dict(os.environ, { 'JUPYTERHUB_TEST_DB_URL': 'sqlite:///%s' % tmpdir.join('jupyterhub.sqlite'), }) p.start() request.addfinalizer(p.stop) - @gen.coroutine - def new_hub(): + + async def new_hub(): kwargs = {} ssl_enabled = getattr(request.module, "ssl_enabled", False) if ssl_enabled: @@ -209,26 +206,27 @@ def test_resume_spawners(tmpdir, request): app = MockHub(test_clean_db=False, **kwargs) app.config.ConfigurableHTTPProxy.should_start = False app.config.ConfigurableHTTPProxy.auth_token = 'unused' - yield app.initialize([]) + await app.initialize([]) return app - app = yield new_hub() + + app = await new_hub() db = app.db # spawn a user's server name = 'kurt' user = add_user(db, app, name=name) - yield user.spawn() + await user.spawn() proc = user.spawner.proc assert proc is not None # stop the Hub without cleaning up servers app.cleanup_servers = False - yield app.stop() + app.stop() # proc is still running assert proc.poll() is None # resume Hub, should still be running - app = yield new_hub() + app = await new_hub() db = app.db user = app.users[name] assert user.running @@ -236,7 +234,7 @@ def test_resume_spawners(tmpdir, request): # stop the Hub without cleaning up servers app.cleanup_servers = False - yield app.stop() + app.stop() # stop the server while the Hub is down. BAMF! proc.terminate() @@ -244,7 +242,7 @@ def test_resume_spawners(tmpdir, request): assert proc.poll() is not None # resume Hub, should be stopped - app = yield new_hub() + app = await new_hub() db = app.db user = app.users[name] assert not user.running diff --git a/jupyterhub/tests/test_auth.py b/jupyterhub/tests/test_auth.py index ea4005e0..f5055af3 100644 --- a/jupyterhub/tests/test_auth.py +++ b/jupyterhub/tests/test_auth.py @@ -12,49 +12,47 @@ from requests import HTTPError from jupyterhub import auth, crypto, orm from .mocking import MockPAMAuthenticator, MockStructGroup, MockStructPasswd -from .test_api import add_user +from .utils import add_user -@pytest.mark.gen_test -def test_pam_auth(): + +async def test_pam_auth(): authenticator = MockPAMAuthenticator() - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'match', 'password': 'match', }) assert authorized['name'] == 'match' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'match', 'password': 'nomatch', }) assert authorized is None # Account check is on by default for increased security - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'notallowedmatch', 'password': 'notallowedmatch', }) assert authorized is None -@pytest.mark.gen_test -def test_pam_auth_account_check_disabled(): +async def test_pam_auth_account_check_disabled(): authenticator = MockPAMAuthenticator(check_account=False) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'allowedmatch', 'password': 'allowedmatch', }) assert authorized['name'] == 'allowedmatch' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'notallowedmatch', 'password': 'notallowedmatch', }) assert authorized['name'] == 'notallowedmatch' -@pytest.mark.gen_test -def test_pam_auth_admin_groups(): +async def test_pam_auth_admin_groups(): jh_users = MockStructGroup('jh_users', ['group_admin', 'also_group_admin', 'override_admin', 'non_admin'], 1234) jh_admins = MockStructGroup('jh_admins', ['group_admin'], 5678) wheel = MockStructGroup('wheel', ['also_group_admin'], 9999) @@ -67,10 +65,10 @@ def test_pam_auth_admin_groups(): system_users = [group_admin, also_group_admin, override_admin, non_admin] user_group_map = { - 'group_admin': [jh_users, jh_admins], - 'also_group_admin': [jh_users, wheel], - 'override_admin': [jh_users], - 'non_admin': [jh_users] + 'group_admin': [jh_users.gr_gid, jh_admins.gr_gid], + 'also_group_admin': [jh_users.gr_gid, wheel.gr_gid], + 'override_admin': [jh_users.gr_gid], + 'non_admin': [jh_users.gr_gid] } def getgrnam(name): @@ -86,103 +84,100 @@ def test_pam_auth_admin_groups(): admin_users={'override_admin'}) # Check admin_group applies as expected - with mock.patch.multiple(auth, - getgrnam=getgrnam, - getpwnam=getpwnam, - getgrouplist=getgrouplist): - authorized = yield authenticator.get_authenticated_user(None, { + with mock.patch.multiple(authenticator, + _getgrnam=getgrnam, + _getpwnam=getpwnam, + _getgrouplist=getgrouplist): + authorized = await authenticator.get_authenticated_user(None, { 'username': 'group_admin', 'password': 'group_admin' }) assert authorized['name'] == 'group_admin' - assert authorized['admin'] == True + assert authorized['admin'] is True # Check multiple groups work, just in case. - with mock.patch.multiple(auth, - getgrnam=getgrnam, - getpwnam=getpwnam, - getgrouplist=getgrouplist): - authorized = yield authenticator.get_authenticated_user(None, { + with mock.patch.multiple(authenticator, + _getgrnam=getgrnam, + _getpwnam=getpwnam, + _getgrouplist=getgrouplist): + authorized = await authenticator.get_authenticated_user(None, { 'username': 'also_group_admin', 'password': 'also_group_admin' }) assert authorized['name'] == 'also_group_admin' - assert authorized['admin'] == True + assert authorized['admin'] is True # Check admin_users still applies correctly - with mock.patch.multiple(auth, - getgrnam=getgrnam, - getpwnam=getpwnam, - getgrouplist=getgrouplist): - authorized = yield authenticator.get_authenticated_user(None, { + with mock.patch.multiple(authenticator, + _getgrnam=getgrnam, + _getpwnam=getpwnam, + _getgrouplist=getgrouplist): + authorized = await authenticator.get_authenticated_user(None, { 'username': 'override_admin', 'password': 'override_admin' }) assert authorized['name'] == 'override_admin' - assert authorized['admin'] == True + assert authorized['admin'] is True # Check it doesn't admin everyone - with mock.patch.multiple(auth, - getgrnam=getgrnam, - getpwnam=getpwnam, - getgrouplist=getgrouplist): - authorized = yield authenticator.get_authenticated_user(None, { + with mock.patch.multiple(authenticator, + _getgrnam=getgrnam, + _getpwnam=getpwnam, + _getgrouplist=getgrouplist): + authorized = await authenticator.get_authenticated_user(None, { 'username': 'non_admin', 'password': 'non_admin' }) assert authorized['name'] == 'non_admin' - assert authorized['admin'] == False + assert authorized['admin'] is False -@pytest.mark.gen_test -def test_pam_auth_whitelist(): +async def test_pam_auth_whitelist(): authenticator = MockPAMAuthenticator(whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) assert authorized['name'] == 'kaylee' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'nomatch', }) assert authorized is None - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'mal', 'password': 'mal', }) assert authorized is None -@pytest.mark.gen_test -def test_pam_auth_group_whitelist(): +async def test_pam_auth_group_whitelist(): def getgrnam(name): return MockStructGroup('grp', ['kaylee']) authenticator = MockPAMAuthenticator(group_whitelist={'group'}) - with mock.patch.object(auth, 'getgrnam', getgrnam): - authorized = yield authenticator.get_authenticated_user(None, { + with mock.patch.object(authenticator, '_getgrnam', getgrnam): + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) assert authorized['name'] == 'kaylee' - with mock.patch.object(auth, 'getgrnam', getgrnam): - authorized = yield authenticator.get_authenticated_user(None, { + with mock.patch.object(authenticator, '_getgrnam', getgrnam): + authorized = await authenticator.get_authenticated_user(None, { 'username': 'mal', 'password': 'mal', }) assert authorized is None -@pytest.mark.gen_test -def test_pam_auth_blacklist(): +async def test_pam_auth_blacklist(): # Null case compared to next case authenticator = MockPAMAuthenticator() - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'wash', }) @@ -190,7 +185,7 @@ def test_pam_auth_blacklist(): # Blacklist basics authenticator = MockPAMAuthenticator(blacklist={'wash'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'wash', }) @@ -198,7 +193,7 @@ def test_pam_auth_blacklist(): # User in both white and blacklists: default deny. Make error someday? authenticator = MockPAMAuthenticator(blacklist={'wash'}, whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'wash', }) @@ -206,7 +201,7 @@ def test_pam_auth_blacklist(): # User not in blacklist can log in authenticator = MockPAMAuthenticator(blacklist={'wash'}, whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) @@ -214,7 +209,7 @@ def test_pam_auth_blacklist(): # User in whitelist, blacklist irrelevent authenticator = MockPAMAuthenticator(blacklist={'mal'}, whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'wash', 'password': 'wash', }) @@ -222,7 +217,7 @@ def test_pam_auth_blacklist(): # User in neither list authenticator = MockPAMAuthenticator(blacklist={'mal'}, whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'simon', 'password': 'simon', }) @@ -230,15 +225,14 @@ def test_pam_auth_blacklist(): # blacklist == {} authenticator = MockPAMAuthenticator(blacklist=set(), whitelist={'wash', 'kaylee'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) assert authorized['name'] == 'kaylee' -@pytest.mark.gen_test -def test_deprecated_signatures(): +async def test_deprecated_signatures(): deprecated_authenticator = MockPAMAuthenticator() def deprecated_xlist(username): @@ -247,35 +241,32 @@ def test_deprecated_signatures(): with mock.patch.multiple(deprecated_authenticator, check_whitelist=deprecated_xlist, check_blacklist=deprecated_xlist): - authorized = yield deprecated_authenticator.get_authenticated_user(None, { + authorized = await deprecated_authenticator.get_authenticated_user(None, { 'username': 'test', 'password': 'test' }) assert authorized is not None - -@pytest.mark.gen_test -def test_pam_auth_no_such_group(): + +async def test_pam_auth_no_such_group(): authenticator = MockPAMAuthenticator(group_whitelist={'nosuchcrazygroup'}) - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'kaylee', 'password': 'kaylee', }) assert authorized is None -@pytest.mark.gen_test -def test_wont_add_system_user(): +async def test_wont_add_system_user(): user = orm.User(name='lioness4321') authenticator = auth.PAMAuthenticator(whitelist={'mal'}) authenticator.create_system_users = False with pytest.raises(KeyError): - yield authenticator.add_user(user) + await authenticator.add_user(user) -@pytest.mark.gen_test -def test_cant_add_system_user(): +async def test_cant_add_system_user(): user = orm.User(name='lioness4321') authenticator = auth.PAMAuthenticator(whitelist={'mal'}) authenticator.add_user_cmd = ['jupyterhub-fake-command'] @@ -297,12 +288,11 @@ def test_cant_add_system_user(): with mock.patch.object(auth, 'Popen', DummyPopen): with pytest.raises(RuntimeError) as exc: - yield authenticator.add_user(user) + await authenticator.add_user(user) assert str(exc.value) == 'Failed to create system user lioness4321: dummy error' -@pytest.mark.gen_test -def test_add_system_user(): +async def test_add_system_user(): user = orm.User(name='lioness4321') authenticator = auth.PAMAuthenticator(whitelist={'mal'}) authenticator.create_system_users = True @@ -318,17 +308,16 @@ def test_add_system_user(): return with mock.patch.object(auth, 'Popen', DummyPopen): - yield authenticator.add_user(user) + await authenticator.add_user(user) assert record['cmd'] == ['echo', '/home/lioness4321', 'lioness4321'] -@pytest.mark.gen_test -def test_delete_user(): +async def test_delete_user(): user = orm.User(name='zoe') a = MockPAMAuthenticator(whitelist={'mal'}) assert 'zoe' not in a.whitelist - yield a.add_user(user) + await a.add_user(user) assert 'zoe' in a.whitelist a.delete_user(user) assert 'zoe' not in a.whitelist @@ -348,47 +337,43 @@ def test_handlers(app): assert handlers[0][0] == '/login' -@pytest.mark.gen_test -def test_auth_state(app, auth_state_enabled): +async def test_auth_state(app, auth_state_enabled): """auth_state enabled and available""" name = 'kiwi' user = add_user(app.db, app, name=name) assert user.encrypted_auth_state is None - cookies = yield app.login_user(name) - auth_state = yield user.get_auth_state() + cookies = await app.login_user(name) + auth_state = await user.get_auth_state() assert auth_state == app.authenticator.auth_state -@pytest.mark.gen_test -def test_auth_admin_non_admin(app): +async def test_auth_admin_non_admin(app): """admin should be passed through for non-admin users""" name = 'kiwi' user = add_user(app.db, app, name=name, admin=False) assert user.admin is False - cookies = yield app.login_user(name) + cookies = await app.login_user(name) assert user.admin is False -@pytest.mark.gen_test -def test_auth_admin_is_admin(app): +async def test_auth_admin_is_admin(app): """admin should be passed through for admin users""" # Admin user defined in MockPAMAuthenticator. name = 'admin' user = add_user(app.db, app, name=name, admin=False) assert user.admin is False - cookies = yield app.login_user(name) + cookies = await app.login_user(name) assert user.admin is True -@pytest.mark.gen_test -def test_auth_admin_retained_if_unset(app): +async def test_auth_admin_retained_if_unset(app): """admin should be unchanged if authenticator doesn't return admin value""" name = 'kiwi' # Add user as admin. user = add_user(app.db, app, name=name, admin=True) assert user.admin is True # User should remain unchanged. - cookies = yield app.login_user(name) + cookies = await app.login_user(name) assert user.admin is True @@ -402,56 +387,53 @@ def auth_state_unavailable(auth_state_enabled): yield -@pytest.mark.gen_test -def test_auth_state_disabled(app, auth_state_unavailable): +async def test_auth_state_disabled(app, auth_state_unavailable): name = 'driebus' user = add_user(app.db, app, name=name) assert user.encrypted_auth_state is None with pytest.raises(HTTPError): - cookies = yield app.login_user(name) - auth_state = yield user.get_auth_state() + cookies = await app.login_user(name) + auth_state = await user.get_auth_state() assert auth_state is None -@pytest.mark.gen_test -def test_normalize_names(): +async def test_normalize_names(): a = MockPAMAuthenticator() - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'ZOE', 'password': 'ZOE', }) assert authorized['name'] == 'zoe' - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'Glenn', 'password': 'Glenn', }) assert authorized['name'] == 'glenn' - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'hExi', 'password': 'hExi', }) assert authorized['name'] == 'hexi' - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'Test', 'password': 'Test', }) assert authorized['name'] == 'test' -@pytest.mark.gen_test -def test_username_map(): +async def test_username_map(): a = MockPAMAuthenticator(username_map={'wash': 'alpha'}) - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'WASH', 'password': 'WASH', }) assert authorized['name'] == 'alpha' - authorized = yield a.get_authenticated_user(None, { + authorized = await a.get_authenticated_user(None, { 'username': 'Inara', 'password': 'Inara', }) diff --git a/jupyterhub/tests/test_auth_expiry.py b/jupyterhub/tests/test_auth_expiry.py new file mode 100644 index 00000000..278b9169 --- /dev/null +++ b/jupyterhub/tests/test_auth_expiry.py @@ -0,0 +1,179 @@ +""" +test authentication expiry + +authentication can expire in a number of ways: + +- needs refresh and can be refreshed +- doesn't need refresh +- needs refresh and cannot be refreshed without new login +""" + +import asyncio +from contextlib import contextmanager +from unittest import mock +from urllib.parse import parse_qs, urlparse + +import pytest + +from .utils import api_request, get_page + + +async def refresh_expired(authenticator, user): + return None + + +@pytest.fixture +def disable_refresh(app): + """Fixture disabling auth refresh""" + with mock.patch.object(app.authenticator, 'refresh_user', refresh_expired): + yield + + +@pytest.fixture +def refresh_pre_spawn(app): + """Fixture enabling auth refresh pre spawn""" + app.authenticator.refresh_pre_spawn = True + try: + yield + finally: + app.authenticator.refresh_pre_spawn = False + + +async def test_auth_refresh_at_login(app, user): + # auth_refreshed starts unset: + assert not user._auth_refreshed + # login sets auth_refreshed timestamp + await app.login_user(user.name) + assert user._auth_refreshed + user._auth_refreshed -= 10 + before = user._auth_refreshed + # login again updates auth_refreshed timestamp + # even when auth is fresh + await app.login_user(user.name) + assert user._auth_refreshed > before + + +async def test_auth_refresh_page(app, user): + cookies = await app.login_user(user.name) + assert user._auth_refreshed + user._auth_refreshed -= 10 + before = user._auth_refreshed + + # get a page with auth not expired + # doesn't trigger refresh + r = await get_page('home', app, cookies=cookies) + assert r.status_code == 200 + assert user._auth_refreshed == before + + # get a page with stale auth, refreshes auth + user._auth_refreshed -= app.authenticator.auth_refresh_age + r = await get_page('home', app, cookies=cookies) + assert r.status_code == 200 + assert user._auth_refreshed > before + + +async def test_auth_expired_page(app, user, disable_refresh): + cookies = await app.login_user(user.name) + assert user._auth_refreshed + user._auth_refreshed -= 10 + before = user._auth_refreshed + + # auth is fresh, doesn't trigger expiry + r = await get_page('home', app, cookies=cookies) + assert user._auth_refreshed == before + assert r.status_code == 200 + + # get a page with stale auth, triggers expiry + user._auth_refreshed -= app.authenticator.auth_refresh_age + before = user._auth_refreshed + r = await get_page('home', app, cookies=cookies, allow_redirects=False) + + # verify that we redirect to login with ?next=requested page + assert r.status_code == 302 + redirect_url = urlparse(r.headers['Location']) + assert redirect_url.path.endswith('/login') + query = parse_qs(redirect_url.query) + assert query['next'] + next_url = urlparse(query['next'][0]) + assert next_url.path == urlparse(r.url).path + + # make sure refresh didn't get updated + assert user._auth_refreshed == before + + +async def test_auth_expired_api(app, user, disable_refresh): + cookies = await app.login_user(user.name) + assert user._auth_refreshed + user._auth_refreshed -= 10 + before = user._auth_refreshed + + # auth is fresh, doesn't trigger expiry + r = await api_request(app, 'users/' + user.name, name=user.name) + assert user._auth_refreshed == before + assert r.status_code == 200 + + # get a page with stale auth, triggers expiry + user._auth_refreshed -= app.authenticator.auth_refresh_age + r = await api_request(app, 'users/' + user.name, name=user.name) + # api requests can't do login redirects + assert r.status_code == 403 + + +async def test_refresh_pre_spawn(app, user, refresh_pre_spawn): + cookies = await app.login_user(user.name) + assert user._auth_refreshed + user._auth_refreshed -= 10 + before = user._auth_refreshed + + # auth is fresh, but should be forced to refresh by spawn + r = await api_request( + app, 'users/{}/server'.format(user.name), method='post', name=user.name + ) + assert 200 <= r.status_code < 300 + assert user._auth_refreshed > before + + +async def test_refresh_pre_spawn_expired(app, user, refresh_pre_spawn, disable_refresh): + cookies = await app.login_user(user.name) + assert user._auth_refreshed + user._auth_refreshed -= 10 + before = user._auth_refreshed + + # auth is fresh, doesn't trigger expiry + r = await api_request( + app, 'users/{}/server'.format(user.name), method='post', name=user.name + ) + assert r.status_code == 403 + assert user._auth_refreshed == before + + +async def test_refresh_pre_spawn_admin_request( + app, user, admin_user, refresh_pre_spawn +): + await app.login_user(user.name) + await app.login_user(admin_user.name) + user._auth_refreshed -= 10 + before = user._auth_refreshed + + # admin request, auth is fresh. Should still refresh user auth. + r = await api_request( + app, 'users', user.name, 'server', method='post', name=admin_user.name + ) + assert 200 <= r.status_code < 300 + assert user._auth_refreshed > before + + +async def test_refresh_pre_spawn_expired_admin_request( + app, user, admin_user, refresh_pre_spawn, disable_refresh +): + await app.login_user(user.name) + await app.login_user(admin_user.name) + user._auth_refreshed -= 10 + + # auth needs refresh but can't without a new login; spawn should fail + user._auth_refreshed -= app.authenticator.auth_refresh_age + r = await api_request( + app, 'users', user.name, 'server', method='post', name=admin_user.name + ) + # api requests can't do login redirects + assert r.status_code == 403 diff --git a/jupyterhub/tests/test_crypto.py b/jupyterhub/tests/test_crypto.py index e133745f..db7799a0 100644 --- a/jupyterhub/tests/test_crypto.py +++ b/jupyterhub/tests/test_crypto.py @@ -11,6 +11,7 @@ keys = [('%i' % i).encode('ascii') * 32 for i in range(3)] hex_keys = [ b2a_hex(key).decode('ascii') for key in keys ] b64_keys = [ b2a_base64(key).decode('ascii').strip() for key in keys ] + @pytest.mark.parametrize("key_env, keys", [ (hex_keys[0], [keys[0]]), (';'.join([b64_keys[0], hex_keys[1]]), keys[:2]), @@ -52,30 +53,27 @@ def crypt_keeper(): ck.keys = save_keys -@pytest.mark.gen_test -def test_roundtrip(crypt_keeper): +async def test_roundtrip(crypt_keeper): data = {'key': 'value'} - encrypted = yield encrypt(data) - decrypted = yield decrypt(encrypted) + encrypted = await encrypt(data) + decrypted = await decrypt(encrypted) assert decrypted == data -@pytest.mark.gen_test -def test_missing_crypto(crypt_keeper): +async def test_missing_crypto(crypt_keeper): with patch.object(crypto, 'cryptography', None): with pytest.raises(crypto.CryptographyUnavailable): - yield encrypt({}) + await encrypt({}) with pytest.raises(crypto.CryptographyUnavailable): - yield decrypt(b'whatever') + await decrypt(b'whatever') -@pytest.mark.gen_test -def test_missing_keys(crypt_keeper): +async def test_missing_keys(crypt_keeper): crypt_keeper.keys = [] with pytest.raises(crypto.NoEncryptionKeys): - yield encrypt({}) + await encrypt({}) with pytest.raises(crypto.NoEncryptionKeys): - yield decrypt(b'whatever') + await decrypt(b'whatever') diff --git a/jupyterhub/tests/test_db.py b/jupyterhub/tests/test_db.py index cc166aad..18a8e952 100644 --- a/jupyterhub/tests/test_db.py +++ b/jupyterhub/tests/test_db.py @@ -40,8 +40,7 @@ def generate_old_db(env_dir, hub_version, db_url): '0.8.1', ], ) -@pytest.mark.gen_test -def test_upgrade(tmpdir, hub_version): +async def test_upgrade(tmpdir, hub_version): db_url = os.getenv('JUPYTERHUB_TEST_DB_URL') if db_url: db_url += '_upgrade_' + hub_version.replace('.', '') @@ -69,7 +68,7 @@ def test_upgrade(tmpdir, hub_version): assert len(sqlite_files) == 1 upgradeapp = UpgradeDB(config=cfg) - yield upgradeapp.initialize([]) + upgradeapp.initialize([]) upgradeapp.start() # check that backup was created: diff --git a/jupyterhub/tests/test_dummyauth.py b/jupyterhub/tests/test_dummyauth.py index 8e419ef2..84fb480e 100644 --- a/jupyterhub/tests/test_dummyauth.py +++ b/jupyterhub/tests/test_dummyauth.py @@ -7,48 +7,45 @@ import pytest from jupyterhub.auth import DummyAuthenticator -@pytest.mark.gen_test -def test_dummy_auth_without_global_password(): +async def test_dummy_auth_without_global_password(): authenticator = DummyAuthenticator() - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'test_user', 'password': 'test_pass', }) assert authorized['name'] == 'test_user' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'test_user', 'password': '', }) assert authorized['name'] == 'test_user' -@pytest.mark.gen_test -def test_dummy_auth_without_username(): +async def test_dummy_auth_without_username(): authenticator = DummyAuthenticator() - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': '', 'password': 'test_pass', }) assert authorized is None -@pytest.mark.gen_test -def test_dummy_auth_with_global_password(): +async def test_dummy_auth_with_global_password(): authenticator = DummyAuthenticator() authenticator.password = "test_password" - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'test_user', 'password': 'test_password', }) assert authorized['name'] == 'test_user' - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'test_user', 'password': 'qwerty', }) assert authorized is None - authorized = yield authenticator.get_authenticated_user(None, { + authorized = await authenticator.get_authenticated_user(None, { 'username': 'some_other_user', 'password': 'test_password', }) diff --git a/jupyterhub/tests/test_internal_ssl_connections.py b/jupyterhub/tests/test_internal_ssl_connections.py index 42b306aa..be5ddcca 100644 --- a/jupyterhub/tests/test_internal_ssl_connections.py +++ b/jupyterhub/tests/test_internal_ssl_connections.py @@ -39,39 +39,36 @@ def wait_for_spawner(spawner, timeout=10): yield wait() -@pytest.mark.gen_test -def test_connection_hub_wrong_certs(app): +async def test_connection_hub_wrong_certs(app): """Connecting to the internal hub url fails without correct certs""" with pytest.raises(SSLError): kwargs = {'verify': False} - r = yield async_requests.get(app.hub.url, **kwargs) + r = await async_requests.get(app.hub.url, **kwargs) r.raise_for_status() -@pytest.mark.gen_test -def test_connection_proxy_api_wrong_certs(app): +async def test_connection_proxy_api_wrong_certs(app): """Connecting to the proxy api fails without correct certs""" with pytest.raises(SSLError): kwargs = {'verify': False} - r = yield async_requests.get(app.proxy.api_url, **kwargs) + r = await async_requests.get(app.proxy.api_url, **kwargs) r.raise_for_status() -@pytest.mark.gen_test -def test_connection_notebook_wrong_certs(app): +async def test_connection_notebook_wrong_certs(app): """Connecting to a notebook fails without correct certs""" with mock.patch.dict( app.config.LocalProcessSpawner, {'cmd': [sys.executable, '-m', 'jupyterhub.tests.mocksu']} ): user = add_user(app.db, app, name='foo') - yield user.spawn() - yield wait_for_spawner(user.spawner) + await user.spawn() + await wait_for_spawner(user.spawner) spawner = user.spawner - status = yield spawner.poll() + status = await spawner.poll() assert status is None with pytest.raises(SSLError): kwargs = {'verify': False} - r = yield async_requests.get(spawner.server.url, **kwargs) + r = await async_requests.get(spawner.server.url, **kwargs) r.raise_for_status() diff --git a/jupyterhub/tests/test_named_servers.py b/jupyterhub/tests/test_named_servers.py index 8f808d33..7715092d 100644 --- a/jupyterhub/tests/test_named_servers.py +++ b/jupyterhub/tests/test_named_servers.py @@ -13,20 +13,19 @@ from .utils import async_requests @pytest.fixture def named_servers(app): with mock.patch.dict(app.tornado_settings, - {'allow_named_servers': True}): + {'allow_named_servers': True, 'named_server_limit_per_user': 2}): yield -@pytest.mark.gen_test -def test_default_server(app, named_servers): +async def test_default_server(app, named_servers): """Test the default /users/:user/server handler when named servers are enabled""" username = 'rosie' user = add_user(app.db, app, name=username) - r = yield api_request(app, 'users', username, 'server', method='post') + r = await api_request(app, 'users', username, 'server', method='post') assert r.status_code == 201 assert r.text == '' - r = yield api_request(app, 'users', username) + r = await api_request(app, 'users', username) r.raise_for_status() user_model = normalize_user(r.json()) @@ -51,11 +50,11 @@ def test_default_server(app, named_servers): # now stop the server - r = yield api_request(app, 'users', username, 'server', method='delete') + r = await api_request(app, 'users', username, 'server', method='delete') assert r.status_code == 204 assert r.text == '' - r = yield api_request(app, 'users', username) + r = await api_request(app, 'users', username) r.raise_for_status() user_model = normalize_user(r.json()) @@ -66,20 +65,19 @@ def test_default_server(app, named_servers): }) -@pytest.mark.gen_test -def test_create_named_server(app, named_servers): +async def test_create_named_server(app, named_servers): username = 'walnut' user = add_user(app.db, app, name=username) # assert user.allow_named_servers == True - cookies = yield app.login_user(username) + cookies = await app.login_user(username) servername = 'trevor' - r = yield api_request(app, 'users', username, 'servers', servername, method='post') + r = await api_request(app, 'users', username, 'servers', servername, method='post') r.raise_for_status() assert r.status_code == 201 assert r.text == '' url = url_path_join(public_url(app, user), servername, 'env') - r = yield async_requests.get(url, cookies=cookies) + r = await async_requests.get(url, cookies=cookies) r.raise_for_status() assert r.url == url env = r.json() @@ -87,7 +85,7 @@ def test_create_named_server(app, named_servers): assert prefix == user.spawners[servername].server.base_url assert prefix.endswith('/user/%s/%s/' % (username, servername)) - r = yield api_request(app, 'users', username) + r = await api_request(app, 'users', username) r.raise_for_status() user_model = normalize_user(r.json()) @@ -111,22 +109,21 @@ def test_create_named_server(app, named_servers): }) -@pytest.mark.gen_test -def test_delete_named_server(app, named_servers): +async def test_delete_named_server(app, named_servers): username = 'donaar' user = add_user(app.db, app, name=username) assert user.allow_named_servers cookies = app.login_user(username) servername = 'splugoth' - r = yield api_request(app, 'users', username, 'servers', servername, method='post') + r = await api_request(app, 'users', username, 'servers', servername, method='post') r.raise_for_status() assert r.status_code == 201 - r = yield api_request(app, 'users', username, 'servers', servername, method='delete') + r = await api_request(app, 'users', username, 'servers', servername, method='delete') r.raise_for_status() assert r.status_code == 204 - r = yield api_request(app, 'users', username) + r = await api_request(app, 'users', username) r.raise_for_status() user_model = normalize_user(r.json()) @@ -140,7 +137,7 @@ def test_delete_named_server(app, named_servers): # low-level record still exists assert servername in user.orm_spawners - r = yield api_request( + r = await api_request( app, 'users', username, 'servers', servername, method='delete', data=json.dumps({'remove': True}), @@ -151,11 +148,56 @@ def test_delete_named_server(app, named_servers): assert servername not in user.orm_spawners -@pytest.mark.gen_test -def test_named_server_disabled(app): +async def test_named_server_disabled(app): username = 'user' servername = 'okay' - r = yield api_request(app, 'users', username, 'servers', servername, method='post') + r = await api_request(app, 'users', username, 'servers', servername, method='post') assert r.status_code == 400 - r = yield api_request(app, 'users', username, 'servers', servername, method='delete') + r = await api_request(app, 'users', username, 'servers', servername, method='delete') assert r.status_code == 400 + + +async def test_named_server_limit(app, named_servers): + username = 'foo' + user = add_user(app.db, app, name=username) + cookies = await app.login_user(username) + + # Create 1st named server + servername1 = 'bar-1' + r = await api_request(app, 'users', username, 'servers', servername1, method='post') + r.raise_for_status() + assert r.status_code == 201 + assert r.text == '' + + # Create 2nd named server + servername2 = 'bar-2' + r = await api_request(app, 'users', username, 'servers', servername2, method='post') + r.raise_for_status() + assert r.status_code == 201 + assert r.text == '' + + # Create 3rd named server + servername3 = 'bar-3' + r = await api_request(app, 'users', username, 'servers', servername3, method='post') + assert r.status_code == 400 + assert r.json() == {"status": 400, "message": "User foo already has the maximum of 2 named servers. One must be deleted before a new server can be created"} + + # Create default server + r = await api_request(app, 'users', username, 'server', method='post') + assert r.status_code == 201 + assert r.text == '' + + # Delete 1st named server + r = await api_request( + app, 'users', username, 'servers', servername1, + method='delete', + data=json.dumps({'remove': True}), + ) + r.raise_for_status() + assert r.status_code == 204 + + # Create 3rd named server again + r = await api_request(app, 'users', username, 'servers', servername3, method='post') + r.raise_for_status() + assert r.status_code == 201 + assert r.text == '' diff --git a/jupyterhub/tests/test_orm.py b/jupyterhub/tests/test_orm.py index cb442f42..5c8eff93 100644 --- a/jupyterhub/tests/test_orm.py +++ b/jupyterhub/tests/test_orm.py @@ -205,8 +205,7 @@ def test_token_find(db): assert found is None -@pytest.mark.gen_test -def test_spawn_fails(db): +async def test_spawn_fails(db): orm_user = orm.User(name='aeofel') db.add(orm_user) db.commit() @@ -223,7 +222,7 @@ def test_spawn_fails(db): }) with pytest.raises(RuntimeError) as exc: - yield user.spawn() + await user.spawn() assert user.spawners[''].server is None assert not user.running @@ -246,8 +245,7 @@ def test_groups(db): assert group.users == [] -@pytest.mark.gen_test -def test_auth_state(db): +async def test_auth_state(db): orm_user = orm.User(name='eve') db.add(orm_user) db.commit() @@ -262,51 +260,51 @@ def test_auth_state(db): state = {'key': 'value'} ck.keys = [] with pytest.raises(crypto.EncryptionUnavailable): - yield user.save_auth_state(state) + await user.save_auth_state(state) assert user.encrypted_auth_state is None # saving/loading None doesn't require keys - yield user.save_auth_state(None) - current = yield user.get_auth_state() + await user.save_auth_state(None) + current = await user.get_auth_state() assert current is None first_key = os.urandom(32) second_key = os.urandom(32) ck.keys = [first_key] - yield user.save_auth_state(state) + await user.save_auth_state(state) assert user.encrypted_auth_state is not None - decrypted_state = yield user.get_auth_state() + decrypted_state = await user.get_auth_state() assert decrypted_state == state # can't read auth_state without keys ck.keys = [] - auth_state = yield user.get_auth_state() + auth_state = await user.get_auth_state() assert auth_state is None # key rotation works db.rollback() ck.keys = [second_key, first_key] - decrypted_state = yield user.get_auth_state() + decrypted_state = await user.get_auth_state() assert decrypted_state == state new_state = {'key': 'newvalue'} - yield user.save_auth_state(new_state) + await user.save_auth_state(new_state) db.commit() ck.keys = [first_key] db.rollback() # can't read anymore with new-key after encrypting with second-key - decrypted_state = yield user.get_auth_state() + decrypted_state = await user.get_auth_state() assert decrypted_state is None - yield user.save_auth_state(new_state) - decrypted_state = yield user.get_auth_state() + await user.save_auth_state(new_state) + decrypted_state = await user.get_auth_state() assert decrypted_state == new_state ck.keys = [] db.rollback() - decrypted_state = yield user.get_auth_state() + decrypted_state = await user.get_auth_state() assert decrypted_state is None diff --git a/jupyterhub/tests/test_pages.py b/jupyterhub/tests/test_pages.py index d62393eb..3ddd0210 100644 --- a/jupyterhub/tests/test_pages.py +++ b/jupyterhub/tests/test_pages.py @@ -1,6 +1,8 @@ """Tests for HTML pages""" +import asyncio import sys +from unittest import mock from urllib.parse import urlencode, urlparse from bs4 import BeautifulSoup @@ -12,108 +14,94 @@ from ..utils import url_path_join as ujoin from .. import orm from ..auth import Authenticator -import mock import pytest -from .mocking import FormSpawner, public_url, public_host -from .test_api import api_request, add_user -from .utils import async_requests +from .mocking import FormSpawner, FalsyCallableFormSpawner +from .utils import ( + async_requests, + api_request, + add_user, + get_page, + public_url, + public_host, +) -def get_page(path, app, hub=True, **kw): - if hub: - prefix = app.hub.base_url - else: - prefix = app.base_url - base_url = ujoin(public_host(app), prefix) - return async_requests.get(ujoin(base_url, path), **kw) - - -@pytest.mark.gen_test -def test_root_no_auth(app): +async def test_root_no_auth(app): url = ujoin(public_host(app), app.hub.base_url) - r = yield async_requests.get(url) + r = await async_requests.get(url) r.raise_for_status() assert r.url == ujoin(url, 'login') -@pytest.mark.gen_test -def test_root_auth(app): - cookies = yield app.login_user('river') - r = yield async_requests.get(public_url(app), cookies=cookies) +async def test_root_auth(app): + cookies = await app.login_user('river') + r = await async_requests.get(public_url(app), cookies=cookies) r.raise_for_status() assert r.url.startswith(public_url(app, app.users['river'])) -@pytest.mark.gen_test -def test_root_redirect(app): +async def test_root_redirect(app): name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) next_url = ujoin(app.base_url, 'user/other/test.ipynb') url = '/?' + urlencode({'next': next_url}) - r = yield get_page(url, app, cookies=cookies) + r = await get_page(url, app, cookies=cookies) r.raise_for_status() path = urlparse(r.url).path assert path == ujoin(app.base_url, 'user/%s/test.ipynb' % name) -@pytest.mark.gen_test -def test_root_default_url_noauth(app): +async def test_root_default_url_noauth(app): with mock.patch.dict(app.tornado_settings, {'default_url': '/foo/bar'}): - r = yield get_page('/', app, allow_redirects=False) + r = await get_page('/', app, allow_redirects=False) r.raise_for_status() url = r.headers.get('Location', '') path = urlparse(url).path assert path == '/foo/bar' -@pytest.mark.gen_test -def test_root_default_url_auth(app): +async def test_root_default_url_auth(app): name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) with mock.patch.dict(app.tornado_settings, {'default_url': '/foo/bar'}): - r = yield get_page('/', app, cookies=cookies, allow_redirects=False) + r = await get_page('/', app, cookies=cookies, allow_redirects=False) r.raise_for_status() url = r.headers.get('Location', '') path = urlparse(url).path assert path == '/foo/bar' -@pytest.mark.gen_test -def test_home_no_auth(app): - r = yield get_page('home', app, allow_redirects=False) +async def test_home_no_auth(app): + r = await get_page('home', app, allow_redirects=False) r.raise_for_status() assert r.status_code == 302 assert '/hub/login' in r.headers['Location'] -@pytest.mark.gen_test -def test_home_auth(app): - cookies = yield app.login_user('river') - r = yield get_page('home', app, cookies=cookies) +async def test_home_auth(app): + cookies = await app.login_user('river') + r = await get_page('home', app, cookies=cookies) r.raise_for_status() assert r.url.endswith('home') -@pytest.mark.gen_test -def test_admin_no_auth(app): - r = yield get_page('admin', app) +async def test_admin_no_auth(app): + r = await get_page('admin', app) assert r.status_code == 403 -@pytest.mark.gen_test -def test_admin_not_admin(app): - cookies = yield app.login_user('wash') - r = yield get_page('admin', app, cookies=cookies) +async def test_admin_not_admin(app): + cookies = await app.login_user('wash') + r = await get_page('admin', app, cookies=cookies) assert r.status_code == 403 -@pytest.mark.gen_test -def test_admin(app): - cookies = yield app.login_user('admin') - r = yield get_page('admin', app, cookies=cookies, allow_redirects=False) +async def test_admin(app): + cookies = await app.login_user('admin') + r = await get_page('admin', app, cookies=cookies, allow_redirects=False) r.raise_for_status() assert r.url.endswith('/admin') @@ -124,127 +112,127 @@ def test_admin(app): 'admin', 'name', ]) -@pytest.mark.gen_test -def test_admin_sort(app, sort): - cookies = yield app.login_user('admin') - r = yield get_page('admin?sort=%s' % sort, app, cookies=cookies) +async def test_admin_sort(app, sort): + cookies = await app.login_user('admin') + r = await get_page('admin?sort=%s' % sort, app, cookies=cookies) r.raise_for_status() assert r.status_code == 200 -@pytest.mark.gen_test -def test_spawn_redirect(app): +async def test_spawn_redirect(app): name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) u = app.users[orm.User.find(app.db, name)] - status = yield u.spawner.poll() + status = await u.spawner.poll() assert status is not None # test spawn page when no server is running - r = yield get_page('spawn', app, cookies=cookies) + r = await get_page('spawn', app, cookies=cookies) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, 'user/%s/' % name) # should have started server - status = yield u.spawner.poll() + status = await u.spawner.poll() assert status is None # test spawn page when server is already running (just redirect) - r = yield get_page('spawn', app, cookies=cookies) + r = await get_page('spawn', app, cookies=cookies) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/' % name) # stop server to ensure /user/name is handled by the Hub - r = yield api_request(app, 'users', name, 'server', method='delete', cookies=cookies) + r = await api_request(app, 'users', name, 'server', method='delete', cookies=cookies) r.raise_for_status() # test handing of trailing slash on `/user/name` - r = yield get_page('user/' + name, app, hub=False, cookies=cookies) + r = await get_page('user/' + name, app, hub=False, cookies=cookies) r.raise_for_status() path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/' % name) -@pytest.mark.gen_test -def test_spawn_handler_access(app): +async def test_spawn_handler_access(app): name = 'winston' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) u = app.users[orm.User.find(app.db, name)] - status = yield u.spawner.poll() + status = await u.spawner.poll() assert status is not None # spawn server via browser link with ?arg=value - r = yield get_page('spawn', app, cookies=cookies, params={'arg': 'value'}) + r = await get_page('spawn', app, cookies=cookies, params={'arg': 'value'}) r.raise_for_status() # verify that request params got passed down # implemented in MockSpawner - r = yield async_requests.get(ujoin(public_url(app, u), 'env')) + r = await async_requests.get(ujoin(public_url(app, u), 'env')) env = r.json() assert 'HANDLER_ARGS' in env assert env['HANDLER_ARGS'] == 'arg=value' # stop server - r = yield api_request(app, 'users', name, 'server', method='delete') + r = await api_request(app, 'users', name, 'server', method='delete') r.raise_for_status() -@pytest.mark.gen_test -def test_spawn_admin_access(app, admin_access): +async def test_spawn_admin_access(app, admin_access): """GET /user/:name as admin with admin-access spawns user's server""" - cookies = yield app.login_user('admin') + cookies = await app.login_user('admin') name = 'mariel' user = add_user(app.db, app=app, name=name) app.db.commit() - r = yield get_page('user/' + name, app, cookies=cookies) + r = await get_page('user/' + name, app, cookies=cookies) r.raise_for_status() assert (r.url.split('?')[0] + '/').startswith(public_url(app, user)) - r = yield get_page('user/{}/env'.format(name), app, hub=False, cookies=cookies) + r = await get_page('user/{}/env'.format(name), app, hub=False, cookies=cookies) r.raise_for_status() env = r.json() assert env['JUPYTERHUB_USER'] == name -@pytest.mark.gen_test -def test_spawn_page(app): +async def test_spawn_page(app): with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): - cookies = yield app.login_user('jones') - r = yield get_page('spawn', app, cookies=cookies) + cookies = await app.login_user('jones') + r = await get_page('spawn', app, cookies=cookies) assert r.url.endswith('/spawn') assert FormSpawner.options_form in r.text - r = yield get_page('spawn?next=foo', app, cookies=cookies) + r = await get_page('spawn?next=foo', app, cookies=cookies) assert r.url.endswith('/spawn?next=foo') assert FormSpawner.options_form in r.text -@pytest.mark.gen_test -def test_spawn_page_admin(app, admin_access): +async def test_spawn_page_falsy_callable(app): + with mock.patch.dict(app.users.settings, {'spawner_class': FalsyCallableFormSpawner}): + cookies = await app.login_user('erik') + r = await get_page('spawn', app, cookies=cookies) + assert 'user/erik' in r.url + + +async def test_spawn_page_admin(app, admin_access): with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): - cookies = yield app.login_user('admin') + cookies = await app.login_user('admin') u = add_user(app.db, app=app, name='melanie') - r = yield get_page('spawn/' + u.name, app, cookies=cookies) + r = await get_page('spawn/' + u.name, app, cookies=cookies) assert r.url.endswith('/spawn/' + u.name) assert FormSpawner.options_form in r.text assert "Spawning server for {}".format(u.name) in r.text -@pytest.mark.gen_test -def test_spawn_form(app): +async def test_spawn_form(app): with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): base_url = ujoin(public_host(app), app.hub.base_url) - cookies = yield app.login_user('jones') + cookies = await app.login_user('jones') orm_u = orm.User.find(app.db, 'jones') u = app.users[orm_u] - yield u.stop() + await u.stop() next_url = ujoin(app.base_url, 'user/jones/tree') - r = yield async_requests.post( + r = await async_requests.post( url_concat(ujoin(base_url, 'spawn'), {'next': next_url}), cookies=cookies, data={'bounds': ['-1', '1'], 'energy': '511keV'}, @@ -258,15 +246,14 @@ def test_spawn_form(app): } -@pytest.mark.gen_test -def test_spawn_form_admin_access(app, admin_access): +async def test_spawn_form_admin_access(app, admin_access): with mock.patch.dict(app.tornado_settings, {'spawner_class': FormSpawner}): base_url = ujoin(public_host(app), app.hub.base_url) - cookies = yield app.login_user('admin') + cookies = await app.login_user('admin') u = add_user(app.db, app=app, name='martha') next_url = ujoin(app.base_url, 'user', u.name, 'tree') - r = yield async_requests.post( + r = await async_requests.post( url_concat(ujoin(base_url, 'spawn', u.name), {'next': next_url}), cookies=cookies, data={'bounds': ['-3', '3'], 'energy': '938MeV'}, @@ -281,16 +268,15 @@ def test_spawn_form_admin_access(app, admin_access): } -@pytest.mark.gen_test -def test_spawn_form_with_file(app): +async def test_spawn_form_with_file(app): with mock.patch.dict(app.tornado_settings, {'spawner_class': FormSpawner}): base_url = ujoin(public_host(app), app.hub.base_url) - cookies = yield app.login_user('jones') + cookies = await app.login_user('jones') orm_u = orm.User.find(app.db, 'jones') u = app.users[orm_u] - yield u.stop() + await u.stop() - r = yield async_requests.post(ujoin(base_url, 'spawn'), + r = await async_requests.post(ujoin(base_url, 'spawn'), cookies=cookies, data={ 'bounds': ['-1', '1'], @@ -309,12 +295,11 @@ def test_spawn_form_with_file(app): } -@pytest.mark.gen_test -def test_user_redirect(app): +async def test_user_redirect(app): name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) - r = yield get_page('/user-redirect/tree/top/', app) + r = await get_page('/user-redirect/tree/top/', app) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path @@ -324,32 +309,31 @@ def test_user_redirect(app): 'next': ujoin(app.hub.base_url, '/user-redirect/tree/top/') }) - r = yield get_page('/user-redirect/notebooks/test.ipynb', app, cookies=cookies) + r = await get_page('/user-redirect/notebooks/test.ipynb', app, cookies=cookies) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/notebooks/test.ipynb' % name) -@pytest.mark.gen_test -def test_user_redirect_deprecated(app): +async def test_user_redirect_deprecated(app): """redirecting from /user/someonelse/ URLs (deprecated)""" name = 'wash' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) - r = yield get_page('/user/baduser', app, cookies=cookies, hub=False) + r = await get_page('/user/baduser', app, cookies=cookies, hub=False) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/' % name) - r = yield get_page('/user/baduser/test.ipynb', app, cookies=cookies, hub=False) + r = await get_page('/user/baduser/test.ipynb', app, cookies=cookies, hub=False) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path assert path == ujoin(app.base_url, '/user/%s/test.ipynb' % name) - r = yield get_page('/user/baduser/test.ipynb', app, hub=False) + r = await get_page('/user/baduser/test.ipynb', app, hub=False) r.raise_for_status() print(urlparse(r.url)) path = urlparse(r.url).path @@ -360,11 +344,10 @@ def test_user_redirect_deprecated(app): }) -@pytest.mark.gen_test -def test_login_fail(app): +async def test_login_fail(app): name = 'wash' base_url = public_url(app) - r = yield async_requests.post(base_url + 'hub/login', + r = await async_requests.post(base_url + 'hub/login', data={ 'username': name, 'password': 'wrong', @@ -374,8 +357,7 @@ def test_login_fail(app): assert not r.cookies -@pytest.mark.gen_test -def test_login_strip(app): +async def test_login_strip(app): """Test that login form doesn't strip whitespace from passwords""" form_data = { 'username': 'spiff', @@ -388,7 +370,7 @@ def test_login_strip(app): called_with.append(data) with mock.patch.object(app.authenticator, 'authenticate', mock_authenticate): - yield async_requests.post(base_url + 'hub/login', + await async_requests.post(base_url + 'hub/login', data=form_data, allow_redirects=False, ) @@ -414,9 +396,8 @@ def test_login_strip(app): (False, '//other.domain', ''), ] ) -@pytest.mark.gen_test -def test_login_redirect(app, running, next_url, location): - cookies = yield app.login_user('river') +async def test_login_redirect(app, running, next_url, location): + cookies = await app.login_user('river') user = app.users['river'] if location: location = ujoin(app.base_url, location) @@ -432,18 +413,17 @@ def test_login_redirect(app, running, next_url, location): if running and not user.active: # ensure running - yield user.spawn() + await user.spawn() elif user.active and not running: # ensure not running - yield user.stop() - r = yield get_page(url, app, cookies=cookies, allow_redirects=False) + await user.stop() + r = await get_page(url, app, cookies=cookies, allow_redirects=False) r.raise_for_status() assert r.status_code == 302 assert location == r.headers['Location'] -@pytest.mark.gen_test -def test_auto_login(app, request): +async def test_auto_login(app, request): class DummyLoginHandler(BaseHandler): def get(self): self.write('ok!') @@ -452,7 +432,7 @@ def test_auto_login(app, request): (ujoin(app.hub.base_url, 'dummy'), DummyLoginHandler), ]) # no auto_login: end up at /hub/login - r = yield async_requests.get(base_url) + r = await async_requests.get(base_url) assert r.url == public_url(app, path='hub/login') # enable auto_login: redirect from /hub/login to /hub/dummy authenticator = Authenticator(auto_login=True) @@ -461,78 +441,118 @@ def test_auto_login(app, request): with mock.patch.dict(app.tornado_settings, { 'authenticator': authenticator, }): - r = yield async_requests.get(base_url) + r = await async_requests.get(base_url) assert r.url == public_url(app, path='hub/dummy') -@pytest.mark.gen_test -def test_auto_login_logout(app): + +async def test_auto_login_logout(app): name = 'burnham' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) with mock.patch.dict(app.tornado_settings, { 'authenticator': Authenticator(auto_login=True), }): - r = yield async_requests.get(public_host(app) + app.tornado_settings['logout_url'], cookies=cookies) + r = await async_requests.get(public_host(app) + app.tornado_settings['logout_url'], cookies=cookies) r.raise_for_status() logout_url = public_host(app) + app.tornado_settings['logout_url'] assert r.url == logout_url assert r.cookies == {} -@pytest.mark.gen_test -def test_logout(app): + +async def test_logout(app): name = 'wash' - cookies = yield app.login_user(name) - r = yield async_requests.get(public_host(app) + app.tornado_settings['logout_url'], cookies=cookies) + cookies = await app.login_user(name) + r = await async_requests.get(public_host(app) + app.tornado_settings['logout_url'], cookies=cookies) r.raise_for_status() login_url = public_host(app) + app.tornado_settings['login_url'] assert r.url == login_url assert r.cookies == {} -@pytest.mark.gen_test -def test_login_no_whitelist_adds_user(app): +@pytest.mark.parametrize('shutdown_on_logout', [True, False]) +async def test_shutdown_on_logout(app, shutdown_on_logout): + name = 'shutitdown' + cookies = await app.login_user(name) + user = app.users[name] + + # start the user's server + await user.spawn() + spawner = user.spawner + + # wait for any pending state to resolve + for i in range(50): + if not spawner.pending: + break + await asyncio.sleep(0.1) + else: + assert False, "Spawner still pending" + assert spawner.active + + # logout + with mock.patch.dict(app.tornado_settings, { + 'shutdown_on_logout': shutdown_on_logout, + }): + r = await async_requests.get( + public_host(app) + app.tornado_settings['logout_url'], + cookies=cookies, + ) + r.raise_for_status() + + login_url = public_host(app) + app.tornado_settings['login_url'] + assert r.url == login_url + assert r.cookies == {} + + # wait for any pending state to resolve + for i in range(50): + if not spawner.pending: + break + await asyncio.sleep(0.1) + else: + assert False, "Spawner still pending" + + assert spawner.ready == (not shutdown_on_logout) + + +async def test_login_no_whitelist_adds_user(app): auth = app.authenticator mock_add_user = mock.Mock() with mock.patch.object(auth, 'add_user', mock_add_user): - cookies = yield app.login_user('jubal') + cookies = await app.login_user('jubal') user = app.users['jubal'] assert mock_add_user.mock_calls == [mock.call(user)] -@pytest.mark.gen_test -def test_static_files(app): +async def test_static_files(app): base_url = ujoin(public_host(app), app.hub.base_url) - r = yield async_requests.get(ujoin(base_url, 'logo')) + r = await async_requests.get(ujoin(base_url, 'logo')) r.raise_for_status() assert r.headers['content-type'] == 'image/png' - r = yield async_requests.get(ujoin(base_url, 'static', 'images', 'jupyter.png')) + r = await async_requests.get(ujoin(base_url, 'static', 'images', 'jupyter.png')) r.raise_for_status() assert r.headers['content-type'] == 'image/png' - r = yield async_requests.get(ujoin(base_url, 'static', 'css', 'style.min.css')) + r = await async_requests.get(ujoin(base_url, 'static', 'css', 'style.min.css')) r.raise_for_status() assert r.headers['content-type'] == 'text/css' -@pytest.mark.gen_test -def test_token_auth(app): - cookies = yield app.login_user('token') - r = yield get_page('token', app, cookies=cookies) +async def test_token_auth(app): + cookies = await app.login_user('token') + r = await get_page('token', app, cookies=cookies) r.raise_for_status() assert r.status_code == 200 -@pytest.mark.gen_test -def test_oauth_token_page(app): +async def test_oauth_token_page(app): name = 'token' - cookies = yield app.login_user(name) + cookies = await app.login_user(name) user = app.users[orm.User.find(app.db, name)] client = orm.OAuthClient(identifier='token') app.db.add(client) oauth_token = orm.OAuthAccessToken(client=client, user=user, grant_type=orm.GrantType.authorization_code) app.db.add(oauth_token) app.db.commit() - r = yield get_page('token', app, cookies=cookies) + r = await get_page('token', app, cookies=cookies) r.raise_for_status() assert r.status_code == 200 @@ -541,14 +561,11 @@ def test_oauth_token_page(app): 503, 404, ]) - -@pytest.mark.gen_test -def test_proxy_error(app, error_status): - r = yield get_page('/error/%i' % error_status, app) +async def test_proxy_error(app, error_status): + r = await get_page('/error/%i' % error_status, app) assert r.status_code == 200 -@pytest.mark.gen_test @pytest.mark.parametrize( "announcements", [ @@ -558,7 +575,7 @@ def test_proxy_error(app, error_status): "login,logout", ] ) -def test_announcements(app, announcements): +async def test_announcements(app, announcements): """Test announcements on various pages""" # Default announcement - same on all pages ann01 = "ANNOUNCE01" @@ -574,26 +591,26 @@ def test_announcements(app, announcements): else: assert ann01 in text - cookies = yield app.login_user("jones") + cookies = await app.login_user("jones") with mock.patch.dict( app.tornado_settings, {"template_vars": template_vars, "spawner_class": FormSpawner}, ): - r = yield get_page("login", app) + r = await get_page("login", app) r.raise_for_status() assert_announcement("login", r.text) - r = yield get_page("spawn", app, cookies=cookies) + r = await get_page("spawn", app, cookies=cookies) r.raise_for_status() assert_announcement("spawn", r.text) - r = yield get_page("home", app, cookies=cookies) # hub/home + r = await get_page("home", app, cookies=cookies) # hub/home r.raise_for_status() assert_announcement("home", r.text) # need auto_login=True to get logout page auto_login = app.authenticator.auto_login app.authenticator.auto_login = True try: - r = yield get_page("logout", app, cookies=cookies) + r = await get_page("logout", app, cookies=cookies) finally: app.authenticator.auto_login = auto_login r.raise_for_status() @@ -608,18 +625,16 @@ def test_announcements(app, announcements): "redirect_uri=ok&client_id=nosuchthing", ] ) -@pytest.mark.gen_test -def test_bad_oauth_get(app, params): - cookies = yield app.login_user("authorizer") - r = yield get_page("hub/api/oauth2/authorize?" + params, app, hub=False, cookies=cookies) +async def test_bad_oauth_get(app, params): + cookies = await app.login_user("authorizer") + r = await get_page("hub/api/oauth2/authorize?" + params, app, hub=False, cookies=cookies) assert r.status_code == 400 -@pytest.mark.gen_test -def test_token_page(app): +async def test_token_page(app): name = "cake" - cookies = yield app.login_user(name) - r = yield get_page("token", app, cookies=cookies) + cookies = await app.login_user(name) + r = await get_page("token", app, cookies=cookies) r.raise_for_status() assert urlparse(r.url).path.endswith('/hub/token') def extract_body(r): @@ -638,7 +653,7 @@ def test_token_page(app): token = user.new_api_token(expires_in=60, note="my-test-token") app.db.commit() - r = yield get_page("token", app, cookies=cookies) + r = await get_page("token", app, cookies=cookies) r.raise_for_status() body = extract_body(r) assert "API Tokens" in body, body @@ -649,10 +664,10 @@ def test_token_page(app): # spawn the user to trigger oauth, etc. # request an oauth token user.spawner.cmd = [sys.executable, '-m', 'jupyterhub.singleuser'] - r = yield get_page("spawn", app, cookies=cookies) + r = await get_page("spawn", app, cookies=cookies) r.raise_for_status() - r = yield get_page("token", app, cookies=cookies) + r = await get_page("token", app, cookies=cookies) r.raise_for_status() body = extract_body(r) assert "API Tokens" in body, body @@ -660,31 +675,27 @@ def test_token_page(app): assert "Authorized Applications" in body, body -@pytest.mark.gen_test -def test_server_not_running_api_request(app): - cookies = yield app.login_user("bees") - r = yield get_page("user/bees/api/status", app, hub=False, cookies=cookies) +async def test_server_not_running_api_request(app): + cookies = await app.login_user("bees") + r = await get_page("user/bees/api/status", app, hub=False, cookies=cookies) assert r.status_code == 404 assert r.headers["content-type"] == "application/json" assert r.json() == {"message": "bees is not running"} -@pytest.mark.gen_test -def test_metrics_no_auth(app): - r = yield get_page("metrics", app) +async def test_metrics_no_auth(app): + r = await get_page("metrics", app) assert r.status_code == 403 -@pytest.mark.gen_test -def test_metrics_auth(app): - cookies = yield app.login_user('river') +async def test_metrics_auth(app): + cookies = await app.login_user('river') metrics_url = ujoin(public_host(app), app.hub.base_url, 'metrics') - r = yield get_page("metrics", app, cookies=cookies) + r = await get_page("metrics", app, cookies=cookies) assert r.status_code == 200 assert r.url == metrics_url -@pytest.mark.gen_test -def test_health_check_request(app): - r = yield get_page('health', app) +async def test_health_check_request(app): + r = await get_page('health', app) assert r.status_code == 200 diff --git a/jupyterhub/tests/test_proxy.py b/jupyterhub/tests/test_proxy.py index 1e4816b7..b5255bdd 100644 --- a/jupyterhub/tests/test_proxy.py +++ b/jupyterhub/tests/test_proxy.py @@ -16,6 +16,7 @@ from .mocking import MockHub from .test_api import api_request, add_user from ..utils import wait_for_http_server, url_path_join as ujoin + @pytest.fixture def disable_check_routes(app): # disable periodic check_routes while we are testing @@ -25,9 +26,8 @@ def disable_check_routes(app): finally: app.last_activity_callback.start() -@pytest.mark.gen_test -def test_external_proxy(request): +async def test_external_proxy(request): auth_token = 'secret!' proxy_ip = '127.0.0.1' proxy_port = 54321 @@ -71,23 +71,24 @@ def test_external_proxy(request): def wait_for_proxy(): return wait_for_http_server('http://%s:%i' % (proxy_ip, proxy_port)) - yield wait_for_proxy() + await wait_for_proxy() - yield app.initialize([]) - yield app.start() + await app.initialize([]) + await app.start() assert app.proxy.proxy_process is None # test if api service has a root route '/' - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() assert list(routes.keys()) == [app.hub.routespec] - + # add user to the db and start a single user server name = 'river' add_user(app.db, app, name=name) - r = yield api_request(app, 'users', name, 'server', method='post') + r = await api_request(app, 'users', name, 'server', method='post', + bypass_proxy=True) r.raise_for_status() - - routes = yield app.proxy.get_all_routes() + + routes = await app.proxy.get_all_routes() # sets the desired path result user_path = ujoin(app.base_url, 'user/river') + '/' print(app.base_url, user_path) @@ -101,18 +102,18 @@ def test_external_proxy(request): proxy.terminate() proxy.wait(timeout=10) proxy = Popen(cmd, env=env) - yield wait_for_proxy() + await wait_for_proxy() - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() assert list(routes.keys()) == [] # poke the server to update the proxy - r = yield api_request(app, 'proxy', method='post') + r = await api_request(app, 'proxy', method='post', bypass_proxy=True) r.raise_for_status() # check that the routes are correct - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() assert sorted(routes.keys()) == [app.hub.routespec, user_spec] # teardown the proxy, and start a new one with different auth and port @@ -131,25 +132,30 @@ def test_external_proxy(request): if app.subdomain_host: cmd.append('--host-routing') proxy = Popen(cmd, env=env) - yield wait_for_proxy() + await wait_for_proxy() # tell the hub where the new proxy is new_api_url = 'http://{}:{}'.format(proxy_ip, proxy_port) - r = yield api_request(app, 'proxy', method='patch', data=json.dumps({ - 'api_url': new_api_url, - 'auth_token': new_auth_token, - })) + r = await api_request( + app, + 'proxy', + method='patch', + data=json.dumps({ + 'api_url': new_api_url, + 'auth_token': new_auth_token, + }), + bypass_proxy=True, + ) r.raise_for_status() assert app.proxy.api_url == new_api_url assert app.proxy.auth_token == new_auth_token # check that the routes are correct - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() assert sorted(routes.keys()) == [app.hub.routespec, user_spec] -@pytest.mark.gen_test @pytest.mark.parametrize("username", [ 'zoe', '50fia', @@ -157,27 +163,27 @@ def test_external_proxy(request): '~TestJH', 'has@', ]) -def test_check_routes(app, username, disable_check_routes): +async def test_check_routes(app, username, disable_check_routes): proxy = app.proxy test_user = add_user(app.db, app, name=username) - r = yield api_request(app, 'users/%s/server' % username, method='post') + r = await api_request(app, 'users/%s/server' % username, method='post') r.raise_for_status() # check a valid route exists for user - routes = yield app.proxy.get_all_routes() + routes = await app.proxy.get_all_routes() before = sorted(routes) assert test_user.proxy_spec in before # check if a route is removed when user deleted - yield app.proxy.check_routes(app.users, app._service_map) - yield proxy.delete_user(test_user) - routes = yield app.proxy.get_all_routes() + await app.proxy.check_routes(app.users, app._service_map) + await proxy.delete_user(test_user) + routes = await app.proxy.get_all_routes() during = sorted(routes) assert test_user.proxy_spec not in during # check if a route exists for user - yield app.proxy.check_routes(app.users, app._service_map) - routes = yield app.proxy.get_all_routes() + await app.proxy.check_routes(app.users, app._service_map) + routes = await app.proxy.get_all_routes() after = sorted(routes) assert test_user.proxy_spec in after @@ -185,7 +191,6 @@ def test_check_routes(app, username, disable_check_routes): assert before == after -@pytest.mark.gen_test @pytest.mark.parametrize("routespec", [ '/has%20space/foo/', '/missing-trailing/slash', @@ -194,11 +199,11 @@ def test_check_routes(app, username, disable_check_routes): 'host.name/path/', 'other.host/path/no/slash', ]) -def test_add_get_delete(app, routespec, disable_check_routes): +async def test_add_get_delete(app, routespec, disable_check_routes): arg = routespec if not routespec.endswith('/'): routespec = routespec + '/' - + # host-routes when not host-routing raises an error # and vice versa expect_value_error = bool(app.subdomain_host) ^ (not routespec.startswith('/')) @@ -213,26 +218,25 @@ def test_add_get_delete(app, routespec, disable_check_routes): proxy = app.proxy target = 'https://localhost:1234' with context(): - yield proxy.add_route(arg, target, {}) - routes = yield proxy.get_all_routes() + await proxy.add_route(arg, target, {}) + routes = await proxy.get_all_routes() if not expect_value_error: assert routespec in routes.keys() with context(): - route = yield proxy.get_route(arg) + route = await proxy.get_route(arg) assert route == { 'target': target, 'routespec': routespec, 'data': route.get('data'), } with context(): - yield proxy.delete_route(arg) + await proxy.delete_route(arg) with context(): - route = yield proxy.get_route(arg) + route = await proxy.get_route(arg) assert route is None -@pytest.mark.gen_test @pytest.mark.parametrize("test_data", [None, 'notjson', json.dumps([])]) -def test_proxy_patch_bad_request_data(app, test_data): - r = yield api_request(app, 'proxy', method='patch', data=test_data) +async def test_proxy_patch_bad_request_data(app, test_data): + r = await api_request(app, 'proxy', method='patch', data=test_data) assert r.status_code == 400 diff --git a/jupyterhub/tests/test_services.py b/jupyterhub/tests/test_services.py index 78069078..428e69ea 100644 --- a/jupyterhub/tests/test_services.py +++ b/jupyterhub/tests/test_services.py @@ -1,5 +1,6 @@ """Tests for services""" +import asyncio from binascii import hexlify from contextlib import contextmanager import os @@ -8,13 +9,14 @@ import sys from threading import Event import time +from async_generator import asynccontextmanager, async_generator, yield_ import pytest import requests from tornado import gen from tornado.ioloop import IOLoop from .mocking import public_url -from ..utils import url_path_join, wait_for_http_server, random_port +from ..utils import url_path_join, wait_for_http_server, random_port, maybe_future from .utils import async_requests mockservice_path = os.path.dirname(os.path.abspath(__file__)) @@ -22,8 +24,9 @@ mockservice_py = os.path.join(mockservice_path, 'mockservice.py') mockservice_cmd = [sys.executable, mockservice_py] -@contextmanager -def external_service(app, name='mockservice'): +@asynccontextmanager +@async_generator +async def external_service(app, name='mockservice'): env = { 'JUPYTERHUB_API_TOKEN': hexlify(os.urandom(5)), 'JUPYTERHUB_SERVICE_NAME': name, @@ -31,17 +34,14 @@ def external_service(app, name='mockservice'): 'JUPYTERHUB_SERVICE_URL': 'http://127.0.0.1:%i' % random_port(), } proc = Popen(mockservice_cmd, env=env) - IOLoop().run_sync( - lambda: wait_for_http_server(env['JUPYTERHUB_SERVICE_URL']) - ) try: - yield env + await wait_for_http_server(env['JUPYTERHUB_SERVICE_URL']) + await yield_(env) finally: proc.terminate() -@pytest.mark.gen_test -def test_managed_service(mockservice): +async def test_managed_service(mockservice): service = mockservice proc = service.proc assert isinstance(proc.pid, object) @@ -58,19 +58,18 @@ def test_managed_service(mockservice): if service.proc is not proc: break else: - yield gen.sleep(0.2) - + await asyncio.sleep(0.2) + assert service.proc.pid != first_pid assert service.proc.poll() is None -@pytest.mark.gen_test -def test_proxy_service(app, mockservice_url): +async def test_proxy_service(app, mockservice_url): service = mockservice_url name = service.name - yield app.proxy.get_all_routes() + await app.proxy.get_all_routes() url = public_url(app, service) + '/foo' - r = yield async_requests.get(url, allow_redirects=False) + r = await async_requests.get(url, allow_redirects=False) path = '/services/{}/foo'.format(name) r.raise_for_status() @@ -78,23 +77,22 @@ def test_proxy_service(app, mockservice_url): assert r.text.endswith(path) -@pytest.mark.gen_test -def test_external_service(app): +async def test_external_service(app): name = 'external' - with external_service(app, name=name) as env: + async with external_service(app, name=name) as env: app.services = [{ 'name': name, 'admin': True, 'url': env['JUPYTERHUB_SERVICE_URL'], 'api_token': env['JUPYTERHUB_API_TOKEN'], }] - yield app.init_services() - yield app.init_api_tokens() - yield app.proxy.add_all_services(app._service_map) + await maybe_future(app.init_services()) + await app.init_api_tokens() + await app.proxy.add_all_services(app._service_map) service = app._service_map[name] url = public_url(app, service) + '/api/users' - r = yield async_requests.get(url, allow_redirects=False) + r = await async_requests.get(url, allow_redirects=False) r.raise_for_status() assert r.status_code == 200 resp = r.json() diff --git a/jupyterhub/tests/test_services_auth.py b/jupyterhub/tests/test_services_auth.py index ce253e05..30fbec0d 100644 --- a/jupyterhub/tests/test_services_auth.py +++ b/jupyterhub/tests/test_services_auth.py @@ -170,6 +170,10 @@ def test_hub_authenticated(request): assert r.status_code == 302 assert auth.login_url in r.headers['Location'] + # clear the cache because we are going to request + # the same url again with a different result + auth.cache.clear() + # upstream 403 m.get(bad_url, status_code=403) r = requests.get('http://127.0.0.1:%i' % port, @@ -223,11 +227,10 @@ def test_hub_authenticated(request): assert r.status_code == 403 -@pytest.mark.gen_test -def test_hubauth_cookie(app, mockservice_url): +async def test_hubauth_cookie(app, mockservice_url): """Test HubAuthenticated service with user cookies""" - cookies = yield app.login_user('badger') - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/', cookies=cookies) + cookies = await app.login_user('badger') + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/', cookies=cookies) r.raise_for_status() print(r.text) reply = r.json() @@ -238,15 +241,14 @@ def test_hubauth_cookie(app, mockservice_url): } -@pytest.mark.gen_test -def test_hubauth_token(app, mockservice_url): +async def test_hubauth_token(app, mockservice_url): """Test HubAuthenticated service with user API tokens""" u = add_user(app.db, name='river') token = u.new_api_token() app.db.commit() # token in Authorization header - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/', + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/', headers={ 'Authorization': 'token %s' % token, }) @@ -258,7 +260,7 @@ def test_hubauth_token(app, mockservice_url): } # token in ?token parameter - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=%s' % token) + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=%s' % token) r.raise_for_status() reply = r.json() sub_reply = { key: reply.get(key, 'missing') for key in ['name', 'admin']} @@ -267,7 +269,7 @@ def test_hubauth_token(app, mockservice_url): 'admin': False, } - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=no-such-token', + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=no-such-token', allow_redirects=False, ) assert r.status_code == 302 @@ -277,17 +279,16 @@ def test_hubauth_token(app, mockservice_url): assert path.endswith('/hub/login') -@pytest.mark.gen_test -def test_hubauth_service_token(app, mockservice_url): +async def test_hubauth_service_token(app, mockservice_url): """Test HubAuthenticated service with service API tokens""" - + token = hexlify(os.urandom(5)).decode('utf8') name = 'test-api-service' app.service_tokens[token] = name - yield app.init_api_tokens() + await app.init_api_tokens() # token in Authorization header - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/', + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/', headers={ 'Authorization': 'token %s' % token, }) @@ -301,7 +302,7 @@ def test_hubauth_service_token(app, mockservice_url): assert not r.cookies # token in ?token parameter - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=%s' % token) + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=%s' % token) r.raise_for_status() reply = r.json() assert reply == { @@ -310,7 +311,7 @@ def test_hubauth_service_token(app, mockservice_url): 'admin': False, } - r = yield async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=no-such-token', + r = await async_requests.get(public_url(app, mockservice_url) + '/whoami/?token=no-such-token', allow_redirects=False, ) assert r.status_code == 302 @@ -320,16 +321,15 @@ def test_hubauth_service_token(app, mockservice_url): assert path.endswith('/hub/login') -@pytest.mark.gen_test -def test_oauth_service(app, mockservice_url): +async def test_oauth_service(app, mockservice_url): service = mockservice_url url = url_path_join(public_url(app, mockservice_url) + 'owhoami/?arg=x') # first request is only going to login and get us to the oauth form page s = AsyncSession() name = 'link' - s.cookies = yield app.login_user(name) + s.cookies = await app.login_user(name) - r = yield s.get(url) + r = await s.get(url) r.raise_for_status() # we should be looking at the oauth confirmation page assert urlparse(r.url).path == app.base_url + 'hub/api/oauth2/authorize' @@ -337,7 +337,7 @@ def test_oauth_service(app, mockservice_url): assert set(r.history[0].cookies.keys()) == {'service-%s-oauth-state' % service.name} # submit the oauth form to complete authorization - r = yield s.post(r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}) + r = await s.post(r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}) r.raise_for_status() assert r.url == url # verify oauth cookie is set @@ -346,7 +346,7 @@ def test_oauth_service(app, mockservice_url): assert 'service-%s-oauth-state' % service.name not in set(s.cookies.keys()) # second request should be authenticated, which means no redirects - r = yield s.get(url, allow_redirects=False) + r = await s.get(url, allow_redirects=False) r.raise_for_status() assert r.status_code == 200 reply = r.json() @@ -359,7 +359,7 @@ def test_oauth_service(app, mockservice_url): # token-authenticated request to HubOAuth token = app.users[name].new_api_token() # token in ?token parameter - r = yield async_requests.get(url_concat(url, {'token': token})) + r = await async_requests.get(url_concat(url, {'token': token})) r.raise_for_status() reply = r.json() assert reply['name'] == name @@ -367,7 +367,7 @@ def test_oauth_service(app, mockservice_url): # verify that ?token= requests set a cookie assert len(r.cookies) != 0 # ensure cookie works in future requests - r = yield async_requests.get( + r = await async_requests.get( url, cookies=r.cookies, allow_redirects=False, @@ -378,17 +378,16 @@ def test_oauth_service(app, mockservice_url): assert reply['name'] == name -@pytest.mark.gen_test -def test_oauth_cookie_collision(app, mockservice_url): +async def test_oauth_cookie_collision(app, mockservice_url): service = mockservice_url url = url_path_join(public_url(app, mockservice_url), 'owhoami/') print(url) s = AsyncSession() name = 'mypha' - s.cookies = yield app.login_user(name) + s.cookies = await app.login_user(name) state_cookie_name = 'service-%s-oauth-state' % service.name service_cookie_name = 'service-%s' % service.name - oauth_1 = yield s.get(url) + oauth_1 = await s.get(url) print(oauth_1.headers) print(oauth_1.cookies, oauth_1.url, url) assert state_cookie_name in s.cookies @@ -398,7 +397,7 @@ def test_oauth_cookie_collision(app, mockservice_url): state_1 = s.cookies[state_cookie_name] # start second oauth login before finishing the first - oauth_2 = yield s.get(url) + oauth_2 = await s.get(url) state_cookies = [ c for c in s.cookies.keys() if c.startswith(state_cookie_name) ] assert len(state_cookies) == 2 # get the random-suffix cookie name @@ -408,7 +407,7 @@ def test_oauth_cookie_collision(app, mockservice_url): # finish oauth 2 # submit the oauth form to complete authorization - r = yield s.post( + r = await s.post( oauth_2.url, data={'scopes': ['identify']}, headers={'Referer': oauth_2.url}, @@ -422,7 +421,7 @@ def test_oauth_cookie_collision(app, mockservice_url): service_cookie_2 = s.cookies[service_cookie_name] # finish oauth 1 - r = yield s.post( + r = await s.post( oauth_1.url, data={'scopes': ['identify']}, headers={'Referer': oauth_1.url}, @@ -441,8 +440,7 @@ def test_oauth_cookie_collision(app, mockservice_url): assert state_cookies == [] -@pytest.mark.gen_test -def test_oauth_logout(app, mockservice_url): +async def test_oauth_logout(app, mockservice_url): """Verify that logout via the Hub triggers logout for oauth services 1. clears session id cookie @@ -467,18 +465,18 @@ def test_oauth_logout(app, mockservice_url): # ensure we start empty assert auth_tokens() == [] - s.cookies = yield app.login_user(name) + s.cookies = await app.login_user(name) assert 'jupyterhub-session-id' in s.cookies - r = yield s.get(url) + r = await s.get(url) r.raise_for_status() assert urlparse(r.url).path.endswith('oauth2/authorize') # submit the oauth form to complete authorization - r = yield s.post(r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}) + r = await s.post(r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}) r.raise_for_status() assert r.url == url # second request should be authenticated - r = yield s.get(url, allow_redirects=False) + r = await s.get(url, allow_redirects=False) r.raise_for_status() assert r.status_code == 200 reply = r.json() @@ -497,13 +495,13 @@ def test_oauth_logout(app, mockservice_url): assert len(auth_tokens()) == 1 # hit hub logout URL - r = yield s.get(public_url(app, path='hub/logout')) + r = await s.get(public_url(app, path='hub/logout')) r.raise_for_status() # verify that all cookies other than the service cookie are cleared assert list(s.cookies.keys()) == [service_cookie_name] # verify that clearing session id invalidates service cookie # i.e. redirect back to login page - r = yield s.get(url) + r = await s.get(url) r.raise_for_status() assert r.url.split('?')[0] == public_url(app, path='hub/login') @@ -520,7 +518,7 @@ def test_oauth_logout(app, mockservice_url): # check that we got the old session id back assert session_id == s.cookies['jupyterhub-session-id'] - r = yield s.get(url, allow_redirects=False) + r = await s.get(url, allow_redirects=False) r.raise_for_status() assert r.status_code == 200 reply = r.json() diff --git a/jupyterhub/tests/test_singleuser.py b/jupyterhub/tests/test_singleuser.py index c9ab77d6..f098cbf7 100644 --- a/jupyterhub/tests/test_singleuser.py +++ b/jupyterhub/tests/test_singleuser.py @@ -13,42 +13,41 @@ from ..utils import url_path_join from .utils import async_requests, AsyncSession -@pytest.mark.gen_test -def test_singleuser_auth(app): +async def test_singleuser_auth(app): # use StubSingleUserSpawner to launch a single-user app in a thread app.spawner_class = StubSingleUserSpawner app.tornado_settings['spawner_class'] = StubSingleUserSpawner # login, start the server - cookies = yield app.login_user('nandy') + cookies = await app.login_user('nandy') user = app.users['nandy'] if not user.running: - yield user.spawn() + await user.spawn() url = public_url(app, user) # no cookies, redirects to login page - r = yield async_requests.get(url) + r = await async_requests.get(url) r.raise_for_status() assert '/hub/login' in r.url # with cookies, login successful - r = yield async_requests.get(url, cookies=cookies) + r = await async_requests.get(url, cookies=cookies) r.raise_for_status() assert urlparse(r.url).path.rstrip('/').endswith('/user/nandy/tree') assert r.status_code == 200 # logout - r = yield async_requests.get(url_path_join(url, 'logout'), cookies=cookies) + r = await async_requests.get(url_path_join(url, 'logout'), cookies=cookies) assert len(r.cookies) == 0 # accessing another user's server hits the oauth confirmation page - cookies = yield app.login_user('burgess') + cookies = await app.login_user('burgess') s = AsyncSession() s.cookies = cookies - r = yield s.get(url) + r = await s.get(url) assert urlparse(r.url).path.endswith('/oauth2/authorize') # submit the oauth form to complete authorization - r = yield s.post( + r = await s.post( r.url, data={'scopes': ['identify']}, headers={'Referer': r.url}, @@ -59,28 +58,27 @@ def test_singleuser_auth(app): assert 'burgess' in r.text -@pytest.mark.gen_test -def test_disable_user_config(app): +async def test_disable_user_config(app): # use StubSingleUserSpawner to launch a single-user app in a thread app.spawner_class = StubSingleUserSpawner app.tornado_settings['spawner_class'] = StubSingleUserSpawner # login, start the server - cookies = yield app.login_user('nandy') + cookies = await app.login_user('nandy') user = app.users['nandy'] # stop spawner, if running: if user.running: print("stopping") - yield user.stop() + await user.stop() # start with new config: user.spawner.debug = True user.spawner.disable_user_config = True - yield user.spawn() - yield app.proxy.add_user(user) - + await user.spawn() + await app.proxy.add_user(user) + url = public_url(app, user) - + # with cookies, login successful - r = yield async_requests.get(url, cookies=cookies) + r = await async_requests.get(url, cookies=cookies) r.raise_for_status() assert r.url.rstrip('/').endswith('/user/nandy/tree') assert r.status_code == 200 @@ -90,6 +88,7 @@ def test_help_output(): out = check_output([sys.executable, '-m', 'jupyterhub.singleuser', '--help-all']).decode('utf8', 'replace') assert 'JupyterHub' in out + def test_version(): out = check_output([sys.executable, '-m', 'jupyterhub.singleuser', '--version']).decode('utf8', 'replace') assert jupyterhub.__version__ in out diff --git a/jupyterhub/tests/test_spawner.py b/jupyterhub/tests/test_spawner.py index b6709acf..18b100ce 100644 --- a/jupyterhub/tests/test_spawner.py +++ b/jupyterhub/tests/test_spawner.py @@ -60,10 +60,9 @@ def new_spawner(db, **kwargs): return user._new_spawner('', spawner_class=LocalProcessSpawner, **kwargs) -@pytest.mark.gen_test -def test_spawner(db, request): +async def test_spawner(db, request): spawner = new_spawner(db) - ip, port = yield spawner.start() + ip, port = await spawner.start() assert ip == '127.0.0.1' assert isinstance(port, int) assert port > 0 @@ -72,115 +71,110 @@ def test_spawner(db, request): # wait for the process to get to the while True: loop time.sleep(1) - status = yield spawner.poll() + status = await spawner.poll() assert status is None - yield spawner.stop() - status = yield spawner.poll() + await spawner.stop() + status = await spawner.poll() assert status == 1 -@gen.coroutine -def wait_for_spawner(spawner, timeout=10): +async def wait_for_spawner(spawner, timeout=10): """Wait for an http server to show up - + polling at shorter intervals for early termination """ deadline = time.monotonic() + timeout def wait(): return spawner.server.wait_up(timeout=1, http=True) while time.monotonic() < deadline: - status = yield spawner.poll() + status = await spawner.poll() assert status is None try: - yield wait() + await wait() except TimeoutError: continue else: break - yield wait() + await wait() -@pytest.mark.gen_test -def test_single_user_spawner(app, request): +async def test_single_user_spawner(app, request): user = next(iter(app.users.values()), None) spawner = user.spawner spawner.cmd = ['jupyterhub-singleuser'] - yield user.spawn() + await user.spawn() assert spawner.server.ip == '127.0.0.1' assert spawner.server.port > 0 - yield wait_for_spawner(spawner) - status = yield spawner.poll() + await wait_for_spawner(spawner) + status = await spawner.poll() assert status is None - yield spawner.stop() - status = yield spawner.poll() + await spawner.stop() + status = await spawner.poll() assert status == 0 -@pytest.mark.gen_test -def test_stop_spawner_sigint_fails(db): +async def test_stop_spawner_sigint_fails(db): spawner = new_spawner(db, cmd=[sys.executable, '-c', _uninterruptible]) - yield spawner.start() + await spawner.start() # wait for the process to get to the while True: loop - yield gen.sleep(1) + await gen.sleep(1) - status = yield spawner.poll() + status = await spawner.poll() assert status is None - - yield spawner.stop() - status = yield spawner.poll() + + await spawner.stop() + status = await spawner.poll() assert status == -signal.SIGTERM -@pytest.mark.gen_test -def test_stop_spawner_stop_now(db): +async def test_stop_spawner_stop_now(db): spawner = new_spawner(db) - yield spawner.start() + await spawner.start() # wait for the process to get to the while True: loop - yield gen.sleep(1) + await gen.sleep(1) - status = yield spawner.poll() + status = await spawner.poll() assert status is None - - yield spawner.stop(now=True) - status = yield spawner.poll() + + await spawner.stop(now=True) + status = await spawner.poll() assert status == -signal.SIGTERM -@pytest.mark.gen_test -def test_spawner_poll(db): +async def test_spawner_poll(db): first_spawner = new_spawner(db) user = first_spawner.user - yield first_spawner.start() + await first_spawner.start() proc = first_spawner.proc - status = yield first_spawner.poll() + status = await first_spawner.poll() assert status is None if user.state is None: user.state = {} first_spawner.orm_spawner.state = first_spawner.get_state() assert 'pid' in first_spawner.orm_spawner.state - + # create a new Spawner, loading from state of previous spawner = new_spawner(db, user=first_spawner.user) spawner.start_polling() - + # wait for the process to get to the while True: loop - yield gen.sleep(1) - status = yield spawner.poll() + await gen.sleep(1) + status = await spawner.poll() assert status is None - + # kill the process proc.terminate() for i in range(10): if proc.poll() is None: - yield gen.sleep(1) + await gen.sleep(1) else: break assert proc.poll() is not None - yield gen.sleep(2) - status = yield spawner.poll() + await gen.sleep(2) + status = await spawner.poll() assert status is not None @@ -213,8 +207,7 @@ def test_string_formatting(db): assert s.format_string(s.default_url) == '/base/%s' % name -@pytest.mark.gen_test -def test_popen_kwargs(db): +async def test_popen_kwargs(db): mock_proc = mock.Mock(spec=Popen) def mock_popen(*args, **kwargs): mock_proc.args = args @@ -224,14 +217,13 @@ def test_popen_kwargs(db): s = new_spawner(db, popen_kwargs={'shell': True}, cmd='jupyterhub-singleuser') with mock.patch.object(spawnermod, 'Popen', mock_popen): - yield s.start() + await s.start() assert mock_proc.kwargs['shell'] == True assert mock_proc.args[0][:1] == (['jupyterhub-singleuser']) -@pytest.mark.gen_test -def test_shell_cmd(db, tmpdir, request): +async def test_shell_cmd(db, tmpdir, request): f = tmpdir.join('bashrc') f.write('export TESTVAR=foo\n') s = new_spawner(db, @@ -243,17 +235,17 @@ def test_shell_cmd(db, tmpdir, request): db.commit() s.server = Server.from_orm(server) db.commit() - (ip, port) = yield s.start() + (ip, port) = await s.start() request.addfinalizer(s.stop) s.server.ip = ip s.server.port = port db.commit() - yield wait_for_spawner(s) - r = yield async_requests.get('http://%s:%i/env' % (ip, port)) + await wait_for_spawner(s) + r = await async_requests.get('http://%s:%i/env' % (ip, port)) r.raise_for_status() env = r.json() assert env['TESTVAR'] == 'foo' - yield s.stop() + await s.stop() def test_inherit_overwrite(): @@ -277,8 +269,7 @@ def test_inherit_ok(): pass -@pytest.mark.gen_test -def test_spawner_reuse_api_token(db, app): +async def test_spawner_reuse_api_token(db, app): # setup: user with no tokens, whose spawner has set the .will_resume flag user = add_user(app.db, app, name='snoopy') spawner = user.spawner @@ -286,26 +277,25 @@ def test_spawner_reuse_api_token(db, app): # will_resume triggers reuse of tokens spawner.will_resume = True # first start: gets a new API token - yield user.spawn() + await user.spawn() api_token = spawner.api_token found = orm.APIToken.find(app.db, api_token) assert found assert found.user.name == user.name assert user.api_tokens == [found] - yield user.stop() + await user.stop() # stop now deletes unused spawners. # put back the mock spawner! user.spawners[''] = spawner # second start: should reuse the token - yield user.spawn() + await user.spawn() # verify re-use of API token assert spawner.api_token == api_token # verify that a new token was not created assert user.api_tokens == [found] -@pytest.mark.gen_test -def test_spawner_insert_api_token(app): +async def test_spawner_insert_api_token(app): """Token provided by spawner is not in the db Insert token into db as a user-provided token. @@ -322,17 +312,16 @@ def test_spawner_insert_api_token(app): # The spawner's provided API token would already be in the db # unless there is a bug somewhere else (in the Spawner), # but handle it anyway. - yield user.spawn() + await user.spawn() assert spawner.api_token == api_token found = orm.APIToken.find(app.db, api_token) assert found assert found.user.name == user.name assert user.api_tokens == [found] - yield user.stop() + await user.stop() -@pytest.mark.gen_test -def test_spawner_bad_api_token(app): +async def test_spawner_bad_api_token(app): """Tokens are revoked when a Spawner gets another user's token""" # we need two users for this one user = add_user(app.db, app, name='antimone') @@ -349,13 +338,12 @@ def test_spawner_bad_api_token(app): # starting a user's server with another user's token # should revoke it with pytest.raises(ValueError): - yield user.spawn() + await user.spawn() assert orm.APIToken.find(app.db, other_token) is None assert other_user.api_tokens == [] -@pytest.mark.gen_test -def test_spawner_delete_server(app): +async def test_spawner_delete_server(app): """Test deleting spawner.server This can occur during app startup if their server has been deleted. @@ -393,22 +381,21 @@ def test_spawner_delete_server(app): "has%40x", ] ) -@pytest.mark.gen_test -def test_spawner_routing(app, name): +async def test_spawner_routing(app, name): """Test routing of names with special characters""" db = app.db with mock.patch.dict(app.config.LocalProcessSpawner, {'cmd': [sys.executable, '-m', 'jupyterhub.tests.mocksu']}): user = add_user(app.db, app, name=name) - yield user.spawn() - yield wait_for_spawner(user.spawner) - yield app.proxy.add_user(user) + await user.spawn() + await wait_for_spawner(user.spawner) + await app.proxy.add_user(user) kwargs = {'allow_redirects': False} if app.internal_ssl: kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key) kwargs["verify"] = app.internal_ssl_ca url = url_path_join(public_url(app, user), "test/url") - r = yield async_requests.get(url, **kwargs) + r = await async_requests.get(url, **kwargs) r.raise_for_status() assert r.url == url assert r.text == urlparse(url).path - yield user.stop() + await user.stop() diff --git a/jupyterhub/tests/test_utils.py b/jupyterhub/tests/test_utils.py index 418f4795..7301b420 100644 --- a/jupyterhub/tests/test_utils.py +++ b/jupyterhub/tests/test_utils.py @@ -26,7 +26,6 @@ def schedule_future(io_loop, *, delay, result=None): return f -@pytest.mark.gen_test @pytest.mark.parametrize("deadline, n, delay, expected", [ (0, 3, 1, []), (0, 3, 0, [0, 1, 2]), @@ -43,7 +42,6 @@ async def test_iterate_until(io_loop, deadline, n, delay, expected): assert yielded == expected -@pytest.mark.gen_test async def test_iterate_until_ready_after_deadline(io_loop): f = schedule_future(io_loop, delay=0) diff --git a/jupyterhub/tests/utils.py b/jupyterhub/tests/utils.py index b5dbb162..ff45a396 100644 --- a/jupyterhub/tests/utils.py +++ b/jupyterhub/tests/utils.py @@ -1,19 +1,32 @@ +import asyncio from concurrent.futures import ThreadPoolExecutor -import requests from certipy import Certipy +import requests + +from jupyterhub import orm +from jupyterhub.objects import Server +from jupyterhub.utils import url_path_join as ujoin + class _AsyncRequests: """Wrapper around requests to return a Future from request methods A single thread is allocated to avoid blocking the IOLoop thread. """ + def __init__(self): self.executor = ThreadPoolExecutor(1) + real_submit = self.executor.submit + self.executor.submit = lambda *args, **kwargs: asyncio.wrap_future( + real_submit(*args, **kwargs) + ) def __getattr__(self, name): requests_method = getattr(requests, name) - return lambda *args, **kwargs: self.executor.submit(requests_method, *args, **kwargs) + return lambda *args, **kwargs: self.executor.submit( + requests_method, *args, **kwargs + ) # async_requests.get = requests.get returning a Future, etc. @@ -22,6 +35,7 @@ async_requests = _AsyncRequests() class AsyncSession(requests.Session): """requests.Session object that runs in the background thread""" + def request(self, *args, **kwargs): return async_requests.executor.submit(super().request, *args, **kwargs) @@ -30,9 +44,150 @@ def ssl_setup(cert_dir, authority_name): # Set up the external certs with the same authority as the internal # one so that certificate trust works regardless of chosen endpoint. certipy = Certipy(store_dir=cert_dir) - alt_names = ['DNS:localhost', 'IP:127.0.0.1'] + alt_names = ["DNS:localhost", "IP:127.0.0.1"] internal_authority = certipy.create_ca(authority_name, overwrite=True) - external_certs = certipy.create_signed_pair('external', authority_name, - overwrite=True, - alt_names=alt_names) + external_certs = certipy.create_signed_pair( + "external", authority_name, overwrite=True, alt_names=alt_names + ) return external_certs + + +def check_db_locks(func): + """Decorator that verifies no locks are held on database upon exit. + + This decorator for test functions verifies no locks are held on the + application's database upon exit by creating and dropping a dummy table. + + The decorator relies on an instance of JupyterHubApp being the first + argument to the decorated function. + + Example + ------- + + @check_db_locks + def api_request(app, *api_path, **kwargs): + + """ + def new_func(app, *args, **kwargs): + retval = func(app, *args, **kwargs) + + temp_session = app.session_factory() + temp_session.execute('CREATE TABLE dummy (foo INT)') + temp_session.execute('DROP TABLE dummy') + temp_session.close() + + return retval + + return new_func + + +def find_user(db, name, app=None): + """Find user in database.""" + orm_user = db.query(orm.User).filter(orm.User.name == name).first() + if app is None: + return orm_user + else: + return app.users[orm_user.id] + + +def add_user(db, app=None, **kwargs): + """Add a user to the database.""" + orm_user = find_user(db, name=kwargs.get('name')) + if orm_user is None: + orm_user = orm.User(**kwargs) + db.add(orm_user) + else: + for attr, value in kwargs.items(): + setattr(orm_user, attr, value) + db.commit() + if app: + return app.users[orm_user.id] + else: + return orm_user + + +def auth_header(db, name): + """Return header with user's API authorization token.""" + user = find_user(db, name) + if user is None: + user = add_user(db, name=name) + token = user.new_api_token() + return {'Authorization': 'token %s' % token} + + +@check_db_locks +async def api_request(app, *api_path, method='get', + noauth=False, bypass_proxy=False, + **kwargs): + """Make an API request""" + if bypass_proxy: + # make a direct request to the hub, + # skipping the proxy + base_url = app.hub.url + else: + base_url = public_url(app, path='hub') + headers = kwargs.setdefault('headers', {}) + + if ( + 'Authorization' not in headers + and not noauth + and 'cookies' not in kwargs + ): + # make a copy to avoid modifying arg in-place + kwargs['headers'] = h = {} + h.update(headers) + h.update(auth_header(app.db, kwargs.pop('name', 'admin'))) + + if 'cookies' in kwargs: + # for cookie-authenticated requests, + # set Referer so it looks like the request originated + # from a Hub-served page + headers.setdefault('Referer', ujoin(base_url, 'test')) + + url = ujoin(base_url, 'api', *api_path) + f = getattr(async_requests, method) + if app.internal_ssl: + kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key) + kwargs["verify"] = app.internal_ssl_ca + resp = await f(url, **kwargs) + assert "frame-ancestors 'self'" in resp.headers['Content-Security-Policy'] + assert ujoin(app.hub.base_url, "security/csp-report") in resp.headers['Content-Security-Policy'] + assert 'http' not in resp.headers['Content-Security-Policy'] + if not kwargs.get('stream', False) and resp.content: + assert resp.headers.get('content-type') == 'application/json' + return resp + + +def get_page(path, app, hub=True, **kw): + if hub: + prefix = app.hub.base_url + else: + prefix = app.base_url + base_url = ujoin(public_host(app), prefix) + return async_requests.get(ujoin(base_url, path), **kw) + + +def public_host(app): + """Return the public *host* (no URL prefix) of the given JupyterHub instance.""" + if app.subdomain_host: + return app.subdomain_host + else: + return Server.from_url(app.proxy.public_url).host + + +def public_url(app, user_or_service=None, path=''): + """Return the full, public base URL (including prefix) of the given JupyterHub instance.""" + if user_or_service: + if app.subdomain_host: + host = user_or_service.host + else: + host = public_host(app) + prefix = user_or_service.prefix + else: + host = public_host(app) + prefix = Server.from_url(app.proxy.public_url).base_url + if path: + return host + ujoin(prefix, path) + else: + return host + prefix + diff --git a/jupyterhub/user.py b/jupyterhub/user.py index 74003ed2..8248afaf 100644 --- a/jupyterhub/user.py +++ b/jupyterhub/user.py @@ -8,7 +8,9 @@ import warnings from sqlalchemy import inspect from tornado import gen +from tornado.httputil import urlencode from tornado.log import app_log +from tornado import web from .utils import maybe_future, url_path_join, make_ssl_context @@ -136,6 +138,7 @@ class User: orm_user = None log = app_log settings = None + _auth_refreshed = None def __init__(self, orm_user, settings=None, db=None): self.db = db or inspect(orm_user).session @@ -380,6 +383,59 @@ class User: url_parts.extend(['server/progress']) return url_path_join(*url_parts) + async def refresh_auth(self, handler): + """Refresh authentication if needed + + Checks authentication expiry and refresh it if needed. + See Spawner. + + If the auth is expired and cannot be refreshed + without forcing a new login, a few things can happen: + + 1. if this is a normal user spawn, + the user should be redirected to login + and back to spawn after login. + 2. if this is a spawn via API or other user, + spawn will fail until the user logs in again. + + Args: + handler (RequestHandler): + The handler for the request triggering the spawn. + May be None + """ + authenticator = self.authenticator + if authenticator is None or not authenticator.refresh_pre_spawn: + # nothing to do + return + + # refresh auth + auth_user = await handler.refresh_auth(self, force=True) + + if auth_user: + # auth refreshed, all done + return + + # if we got to here, auth is expired and couldn't be refreshed + self.log.error( + "Auth expired for %s; cannot spawn until they login again", + self.name, + ) + # auth expired, cannot spawn without a fresh login + # it's the current user *and* spawn via GET, trigger login redirect + if handler.request.method == 'GET' and handler.current_user is self: + self.log.info("Redirecting %s to login to refresh auth", self.name) + url = self.get_login_url() + next_url = self.request.uri + sep = '&' if '?' in url else '?' + url += sep + urlencode(dict(next=next_url)) + self.redirect(url) + raise web.Finish() + else: + # spawn via POST or on behalf of another user. + # nothing we can do here but fail + raise web.HTTPError(400, "{}'s authentication has expired".format(self.name)) + + async def spawn(self, server_name='', options=None, handler=None): """Start the user's spawner @@ -395,6 +451,9 @@ class User: """ db = self.db + if handler: + await self.refresh_auth(handler) + base_url = url_path_join(self.base_url, server_name) + '/' orm_server = orm.Server( @@ -436,7 +495,7 @@ class User: # trigger pre-spawn hook on authenticator authenticator = self.authenticator - if (authenticator): + if authenticator: await maybe_future(authenticator.pre_spawn_start(self, spawner)) spawner._start_pending = True @@ -531,7 +590,7 @@ class User: self.settings['statsd'].incr('spawner.failure.error') e.reason = 'error' try: - await self.stop() + await self.stop(spawner.name) except Exception: self.log.error("Failed to cleanup {user}'s server that failed to start".format( user=self.name, @@ -550,6 +609,15 @@ class User: spawner.orm_spawner.state = spawner.get_state() db.commit() spawner._waiting_for_response = True + await self._wait_up(spawner) + + async def _wait_up(self, spawner): + """Wait for a server to finish starting. + + Shuts the server down if it doesn't respond within + spawner.http_timeout. + """ + server = spawner.server key = self.settings.get('internal_ssl_key') cert = self.settings.get('internal_ssl_cert') ca = self.settings.get('internal_ssl_ca') @@ -578,7 +646,7 @@ class User: )) self.settings['statsd'].incr('spawner.failure.http_error') try: - await self.stop() + await self.stop(spawner.name) except Exception: self.log.error("Failed to cleanup {user}'s server that failed to start".format( user=self.name, @@ -594,7 +662,7 @@ class User: finally: spawner._waiting_for_response = False spawner._start_pending = False - return self + return spawner async def stop(self, server_name=''): """Stop the user's spawner @@ -606,6 +674,9 @@ class User: spawner._start_pending = False spawner.stop_polling() spawner._stop_pending = True + + self.log.debug("Stopping %s", spawner._log_name) + try: api_token = spawner.api_token status = await spawner.poll() @@ -637,6 +708,7 @@ class User: self.log.debug("Deleting oauth client %s", oauth_client.identifier) self.db.delete(oauth_client) self.db.commit() + self.log.debug("Finished stopping %s", spawner._log_name) finally: spawner.orm_spawner.started = None self.db.commit() diff --git a/jupyterhub/utils.py b/jupyterhub/utils.py index 2739d261..101f8f13 100644 --- a/jupyterhub/utils.py +++ b/jupyterhub/utils.py @@ -437,6 +437,9 @@ def print_stacks(file=sys.stderr): if ( last_frame[0].endswith('threading.py') and last_frame[-1] == 'waiter.acquire()' + ) or ( + last_frame[0].endswith('thread.py') + and last_frame[-1].endswith('work_queue.get(block=True)') ): # thread is waiting on a condition # call it idle rather than showing the uninteresting stack @@ -541,3 +544,9 @@ async def iterate_until(deadline_future, generator): else: # neither is done, this shouldn't happen continue + + +def utcnow(): + """Return timezone-aware utcnow""" + return datetime.now(timezone.utc) + diff --git a/package.json b/package.json index 7336aa1c..e1df9d12 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "prettier": "^1.14.2" }, "dependencies": { - "bootstrap": "^3.3.7", + "bootstrap": "^3.4.0", "font-awesome": "^4.7.0", "jquery": "^3.2.1", "moment": "^2.19.3", diff --git a/pytest.ini b/pytest.ini index 279b3e49..e3af6b27 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,8 @@ [pytest] -minversion = 3.3 +# pytest 3.10 has broken minversion checks, +# so we have to disable this until pytest 3.11 +# minversion = 3.3 + python_files = test_*.py markers = gen_test: marks an async tornado test diff --git a/requirements.txt b/requirements.txt index 12083646..894a0e8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ traitlets>=4.3.2 tornado>=5.0 jinja2 pamela -oauthlib>=2.0 +oauthlib>=2.0,<3 python-dateutil SQLAlchemy>=1.1 requests diff --git a/scripts/jupyterhub b/scripts/jupyterhub deleted file mode 100755 index 98d12891..00000000 --- a/scripts/jupyterhub +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python3 - -from jupyterhub.app import main -main() diff --git a/scripts/jupyterhub-singleuser b/scripts/jupyterhub-singleuser deleted file mode 100755 index e207396d..00000000 --- a/scripts/jupyterhub-singleuser +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python3 - -from jupyterhub.singleuser import main - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py index b085bce5..63e3b077 100755 --- a/setup.py +++ b/setup.py @@ -89,7 +89,6 @@ with open('README.md', encoding="utf8") as f: setup_args = dict( name = 'jupyterhub', - scripts = glob(pjoin('scripts', '*')), packages = packages, # dummy, so that install_data doesn't get skipped # this will be overridden when bower is run anyway @@ -119,7 +118,12 @@ setup_args = dict( 'jupyterhub.spawners': [ 'default = jupyterhub.spawner:LocalProcessSpawner', 'localprocess = jupyterhub.spawner:LocalProcessSpawner', + 'simple = jupyterhub.spawner:SimpleLocalProcessSpawner', ], + 'console_scripts': [ + 'jupyterhub = jupyterhub.app:main', + 'jupyterhub-singleuser = jupyterhub.singleuser:main', + ] }, classifiers = [ 'Intended Audience :: Developers', diff --git a/share/jupyterhub/static/js/home.js b/share/jupyterhub/static/js/home.js index a22f5b22..99f5901a 100644 --- a/share/jupyterhub/static/js/home.js +++ b/share/jupyterhub/static/js/home.js @@ -111,8 +111,9 @@ require(["jquery", "moment", "jhapi"], function($, moment, JHAPI) { }); }); - $("#new-server-btn").click(function() { - var serverName = $("#new-server-name").val(); + $(".new-server-btn").click(function() { + var row = getRow($(this)); + var serverName = row.find(".new-server-name").val(); api.start_named_server(user, serverName, { success: function(reply) { // reload after creating the server diff --git a/share/jupyterhub/templates/home.html b/share/jupyterhub/templates/home.html index ba347070..3cf44281 100644 --- a/share/jupyterhub/templates/home.html +++ b/share/jupyterhub/templates/home.html @@ -26,10 +26,12 @@

In addition to your default server, - you may have additional servers with names. + you may have additional {% if named_server_limit_per_user > 0 %}{{ named_server_limit_per_user }} {% endif %}server(s) with names. This allows you to have more than one server running at the same time.

+ {% set named_spawners = user.all_spawners(include_default=False)|list %} + @@ -42,13 +44,13 @@ - {% for spawner in user.all_spawners(include_default=False) %} + {% for spawner in named_spawners %} {# name #} diff --git a/share/jupyterhub/templates/page.html b/share/jupyterhub/templates/page.html index 7d973bc0..5677eb93 100644 --- a/share/jupyterhub/templates/page.html +++ b/share/jupyterhub/templates/page.html @@ -96,7 +96,11 @@
- - + + Add New Server
{{ spawner.name }}