From 9fa4106c04aef86fe0dc726fe163d1157316d5a4 Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Fri, 28 Sep 2018 07:05:24 -0700 Subject: [PATCH 01/81] bump bootstrap version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7336aa1c..42421e44 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "prettier": "^1.14.2" }, "dependencies": { - "bootstrap": "^3.3.7", + "bootstrap": "^4.1.3", "font-awesome": "^4.7.0", "jquery": "^3.2.1", "moment": "^2.19.3", From 80e241c86f6fa06f540059377dc94eb3ea18a525 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 5 Nov 2018 10:55:40 +0100 Subject: [PATCH 02/81] move simplespawner into jupyterhub.spawner --- jupyterhub/spawner.py | 41 +++++++++++++++++++++++++++++++++++++ spawners/__init__.py | 1 - spawners/simplespawner.py | 43 --------------------------------------- 3 files changed, 41 insertions(+), 44 deletions(-) delete mode 100644 spawners/__init__.py delete mode 100644 spawners/simplespawner.py diff --git a/jupyterhub/spawner.py b/jupyterhub/spawner.py index 3d8311f3..366fbbce 100644 --- a/jupyterhub/spawner.py +++ b/jupyterhub/spawner.py @@ -1383,3 +1383,44 @@ class LocalProcessSpawner(Spawner): if status is None: # it all failed, zombie process self.log.warning("Process %i never died", self.pid) + + +class SimpleLocalProcessSpawner(LocalProcessSpawner): + """ + A version of LocalProcessSpawner that doesn't require users to exist on + the system beforehand. + + Only use this for testing. + + Note: DO NOT USE THIS FOR PRODUCTION USE CASES! It is very insecure, and + provides absolutely no isolation between different users! + """ + + home_path_template = Unicode( + '/tmp/{username}', + config=True, + help='Template to expand to set the user home. {username} is expanded' + ) + + @property + def home_path(self): + return self.home_path_template.format( + username=self.user.name, + ) + + def make_preexec_fn(self, name): + home = self.home_path + def preexec(): + try: + os.makedirs(home, 0o755, exist_ok=True) + os.chdir(home) + except Exception as e: + + print(e) + return preexec + + def user_env(self, env): + env['USER'] = self.user.name + env['HOME'] = self.home_path + env['SHELL'] = '/bin/bash' + diff --git a/spawners/__init__.py b/spawners/__init__.py deleted file mode 100644 index ef919810..00000000 --- a/spawners/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .simplespawner import SimpleSpawner \ No newline at end of file diff --git a/spawners/simplespawner.py b/spawners/simplespawner.py deleted file mode 100644 index 8df323c9..00000000 --- a/spawners/simplespawner.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -from traitlets import Unicode - -from jupyterhub.spawner import LocalProcessSpawner - - -class SimpleLocalProcessSpawner(LocalProcessSpawner): - """ - A version of LocalProcessSpawner that doesn't require users to exist on - the system beforehand. - - Note: DO NOT USE THIS FOR PRODUCTION USE CASES! It is very insecure, and - provides absolutely no isolation between different users! - """ - - home_path_template = Unicode( - '/tmp/{userid}', - config=True, - help='Template to expand to set the user home. {userid} and {username} are expanded' - ) - - @property - def home_path(self): - return self.home_path_template.format( - userid=self.user.id, - username=self.user.name - ) - - def make_preexec_fn(self, name): - home = self.home_path - def preexec(): - try: - os.makedirs(home, 0o755, exist_ok=True) - os.chdir(home) - except e: - print(e) - return preexec - - def user_env(self, env): - env['USER'] = self.user.name - env['HOME'] = self.home_path - env['SHELL'] = '/bin/bash' - return env \ No newline at end of file From 52c468d89c53241f4d9b9de9c46eb134e52753ca Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 5 Nov 2018 10:57:02 +0100 Subject: [PATCH 03/81] make home_dir a traitlet so the property is only evaluated once and overrideable via hooks --- jupyterhub/spawner.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/jupyterhub/spawner.py b/jupyterhub/spawner.py index 366fbbce..b221a71a 100644 --- a/jupyterhub/spawner.py +++ b/jupyterhub/spawner.py @@ -1396,31 +1396,34 @@ class SimpleLocalProcessSpawner(LocalProcessSpawner): provides absolutely no isolation between different users! """ - home_path_template = Unicode( + home_dir_template = Unicode( '/tmp/{username}', config=True, - help='Template to expand to set the user home. {username} is expanded' + help=""" + Template to expand to set the user home. + {username} is expanded to the jupyterhub username. + """ ) - @property - def home_path(self): - return self.home_path_template.format( + home_dir = Unicode(help="The home directory for the user") + @default('home_dir') + def _default_home_dir(self): + return self.home_dir_template.format( username=self.user.name, ) def make_preexec_fn(self, name): - home = self.home_path + home = self.home_dir def preexec(): try: os.makedirs(home, 0o755, exist_ok=True) os.chdir(home) except Exception as e: - - print(e) + self.log.exception("Error in preexec for %s", name) return preexec def user_env(self, env): env['USER'] = self.user.name - env['HOME'] = self.home_path + env['HOME'] = self.home_dir env['SHELL'] = '/bin/bash' From 575af23e236c3904b151a4aa0005dac0ce4ccf28 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 5 Nov 2018 10:57:10 +0100 Subject: [PATCH 04/81] register simplespawner in setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index b085bce5..5b14fc37 100755 --- a/setup.py +++ b/setup.py @@ -119,6 +119,7 @@ setup_args = dict( 'jupyterhub.spawners': [ 'default = jupyterhub.spawner:LocalProcessSpawner', 'localprocess = jupyterhub.spawner:LocalProcessSpawner', + 'simple = jupyterhub.spawner:SimpleLocalProcessSpawner', ], }, classifiers = [ From 4fb158933ed4e329f8e803aedc66ad4ebc502b19 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 5 Nov 2018 11:01:06 +0100 Subject: [PATCH 05/81] no-op move_certs in simplespawner --- jupyterhub/spawner.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jupyterhub/spawner.py b/jupyterhub/spawner.py index b221a71a..6d0d2b9b 100644 --- a/jupyterhub/spawner.py +++ b/jupyterhub/spawner.py @@ -1426,4 +1426,9 @@ class SimpleLocalProcessSpawner(LocalProcessSpawner): env['USER'] = self.user.name env['HOME'] = self.home_dir env['SHELL'] = '/bin/bash' + return env + + def move_certs(self, paths): + """No-op for installing certs""" + return paths From e512847652bef695310af13c268d3a8b0e8bb7f2 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 5 Nov 2018 11:01:21 +0100 Subject: [PATCH 06/81] use simplespawner as base for testing --- jupyterhub/tests/mocking.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/jupyterhub/tests/mocking.py b/jupyterhub/tests/mocking.py index 2c553e34..e6540e4d 100644 --- a/jupyterhub/tests/mocking.py +++ b/jupyterhub/tests/mocking.py @@ -46,7 +46,7 @@ from ..app import JupyterHub from ..auth import PAMAuthenticator from .. import orm from ..objects import Server -from ..spawner import LocalProcessSpawner +from ..spawner import LocalProcessSpawner, SimpleLocalProcessSpawner from ..singleuser import SingleUserNotebookApp from ..utils import random_port, url_path_join from .utils import async_requests, ssl_setup @@ -73,20 +73,14 @@ def mock_open_session(username, service, encoding): pass -class MockSpawner(LocalProcessSpawner): +class MockSpawner(SimpleLocalProcessSpawner): """Base mock spawner - disables user-switching that we need root permissions to do - spawns `jupyterhub.tests.mocksu` instead of a full single-user server """ - def make_preexec_fn(self, *a, **kw): - # skip the setuid stuff - return - - def _set_user_changed(self, name, old, new): - pass - def user_env(self, env): + env = super().user_env(env) if self.handler: env['HANDLER_ARGS'] = self.handler.request.query return env @@ -95,10 +89,6 @@ class MockSpawner(LocalProcessSpawner): def _cmd_default(self): return [sys.executable, '-m', 'jupyterhub.tests.mocksu'] - def move_certs(self, paths): - """Return the paths unmodified""" - return paths - use_this_api_token = None def start(self): if self.use_this_api_token: From 26866153040afae5dd7d9ccfa67362f34d2b372a Mon Sep 17 00:00:00 2001 From: Kristiyan Date: Mon, 5 Nov 2018 11:47:46 +0100 Subject: [PATCH 07/81] add configuration for shutting down all user spawners on logout --- jupyterhub/app.py | 6 ++++++ jupyterhub/handlers/login.py | 13 ++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 036fb6de..cd3e1374 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -1020,6 +1020,11 @@ class JupyterHub(Application): statsd = Any(allow_none=False, help="The statsd client, if any. A mock will be used if we aren't using statsd") + shutdown_on_logout = Bool( + False, + help="""Shuts down all user servers on logout""" + ).tag(config=True) + @default('statsd') def _statsd(self): if self.statsd_host: @@ -1849,6 +1854,7 @@ class JupyterHub(Application): internal_ssl_cert=self.internal_ssl_cert, internal_ssl_ca=self.internal_ssl_ca, trusted_alt_names=self.trusted_alt_names, + shutdown_on_logout=self.shutdown_on_logout ) # allow configured settings to have priority settings.update(self.tornado_settings) diff --git a/jupyterhub/handlers/login.py b/jupyterhub/handlers/login.py index 666a5a00..a0948e83 100644 --- a/jupyterhub/handlers/login.py +++ b/jupyterhub/handlers/login.py @@ -9,13 +9,24 @@ from tornado.httputil import url_concat from tornado import web from .base import BaseHandler +from ..utils import maybe_future + class LogoutHandler(BaseHandler): + + @property + def shutdown_on_logout(self): + return self.settings.get('shutdown_on_logout', False) + """Log a user out by clearing their login cookie.""" - def get(self): + async def get(self): user = self.current_user if user: + if self.shutdown_on_logout: + self.log.info("Shutting down all %s's servers", user.name) + for name, spawner in user.spawners.items(): + await maybe_future(spawner.stop()) self.log.info("User logged out: %s", user.name) self.clear_login_cookie() self.statsd.incr('logout') From e5e6876cefa578d90c83040dcdd8cb894bf6b993 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 6 Nov 2018 14:30:26 +0100 Subject: [PATCH 08/81] test shutdown_on_logout --- jupyterhub/tests/test_pages.py | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/jupyterhub/tests/test_pages.py b/jupyterhub/tests/test_pages.py index d62393eb..6c65a63e 100644 --- a/jupyterhub/tests/test_pages.py +++ b/jupyterhub/tests/test_pages.py @@ -1,5 +1,6 @@ """Tests for HTML pages""" +import asyncio import sys from urllib.parse import urlencode, urlparse @@ -489,6 +490,51 @@ def test_logout(app): assert r.cookies == {} +@pytest.mark.parametrize('shutdown_on_logout', [True, False]) +@pytest.mark.gen_test +async def test_shutdown_on_logout(app, shutdown_on_logout): + name = 'shutitdown' + cookies = await app.login_user(name) + user = app.users[name] + + # start the user's server + await user.spawn() + spawner = user.spawner + + # wait for any pending state to resolve + for i in range(50): + if not spawner.pending: + break + await asyncio.sleep(0.1) + else: + assert False, "Spawner still pending" + assert spawner.active + + # logout + with mock.patch.dict(app.tornado_settings, { + 'shutdown_on_logout': shutdown_on_logout, + }): + r = await async_requests.get( + public_host(app) + app.tornado_settings['logout_url'], + cookies=cookies, + ) + r.raise_for_status() + + login_url = public_host(app) + app.tornado_settings['login_url'] + assert r.url == login_url + assert r.cookies == {} + + # wait for any pending state to resolve + for i in range(50): + if not spawner.pending: + break + await asyncio.sleep(0.1) + else: + assert False, "Spawner still pending" + + assert spawner.ready == (not shutdown_on_logout) + + @pytest.mark.gen_test def test_login_no_whitelist_adds_user(app): auth = app.authenticator From 1f7838ba5f50ee4680e32a24b9ea96dad423949b Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 6 Nov 2018 14:30:51 +0100 Subject: [PATCH 09/81] ensure async-requests is awaitable so we can use await, not just yield --- jupyterhub/tests/utils.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/jupyterhub/tests/utils.py b/jupyterhub/tests/utils.py index b5dbb162..e2b4c1b1 100644 --- a/jupyterhub/tests/utils.py +++ b/jupyterhub/tests/utils.py @@ -1,19 +1,24 @@ +import asyncio from concurrent.futures import ThreadPoolExecutor -import requests from certipy import Certipy +import requests + class _AsyncRequests: """Wrapper around requests to return a Future from request methods A single thread is allocated to avoid blocking the IOLoop thread. """ + def __init__(self): self.executor = ThreadPoolExecutor(1) def __getattr__(self, name): requests_method = getattr(requests, name) - return lambda *args, **kwargs: self.executor.submit(requests_method, *args, **kwargs) + return lambda *args, **kwargs: asyncio.wrap_future( + self.executor.submit(requests_method, *args, **kwargs) + ) # async_requests.get = requests.get returning a Future, etc. @@ -22,6 +27,7 @@ async_requests = _AsyncRequests() class AsyncSession(requests.Session): """requests.Session object that runs in the background thread""" + def request(self, *args, **kwargs): return async_requests.executor.submit(super().request, *args, **kwargs) @@ -30,9 +36,9 @@ def ssl_setup(cert_dir, authority_name): # Set up the external certs with the same authority as the internal # one so that certificate trust works regardless of chosen endpoint. certipy = Certipy(store_dir=cert_dir) - alt_names = ['DNS:localhost', 'IP:127.0.0.1'] + alt_names = ["DNS:localhost", "IP:127.0.0.1"] internal_authority = certipy.create_ca(authority_name, overwrite=True) - external_certs = certipy.create_signed_pair('external', authority_name, - overwrite=True, - alt_names=alt_names) + external_certs = certipy.create_signed_pair( + "external", authority_name, overwrite=True, alt_names=alt_names + ) return external_certs From 006b89746ad12abd8e404eb02fe5fb646a0045b1 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 6 Nov 2018 14:31:42 +0100 Subject: [PATCH 10/81] use stop_single_user wrapper to stop user servers rather than lower-level spawner.stop --- jupyterhub/handlers/login.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/jupyterhub/handlers/login.py b/jupyterhub/handlers/login.py index a0948e83..f806eae0 100644 --- a/jupyterhub/handlers/login.py +++ b/jupyterhub/handlers/login.py @@ -3,8 +3,9 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +import asyncio + from tornado.escape import url_escape -from tornado import gen from tornado.httputil import url_concat from tornado import web @@ -12,21 +13,31 @@ from .base import BaseHandler from ..utils import maybe_future - class LogoutHandler(BaseHandler): + """Log a user out by clearing their login cookie.""" @property def shutdown_on_logout(self): return self.settings.get('shutdown_on_logout', False) - """Log a user out by clearing their login cookie.""" async def get(self): user = self.current_user if user: if self.shutdown_on_logout: - self.log.info("Shutting down all %s's servers", user.name) - for name, spawner in user.spawners.items(): - await maybe_future(spawner.stop()) + active_servers = [ + name + for (name, spawner) in user.spawners.items() + if spawner.active and not spawner.pending + ] + if active_servers: + self.log.info("Shutting down %s's servers", user.name) + futures = [] + for server_name in active_servers: + futures.append( + maybe_future(self.stop_single_user(user, server_name)) + ) + await asyncio.gather(*futures) + self.log.info("User logged out: %s", user.name) self.clear_login_cookie() self.statsd.incr('logout') From 0cebb4c9d746976ee04ab1ffefe2fe37cded29d4 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 8 Nov 2018 14:36:00 +0100 Subject: [PATCH 11/81] disable pytest minversion check because it doesn't work with current pytest --- pytest.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 279b3e49..e3af6b27 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,8 @@ [pytest] -minversion = 3.3 +# pytest 3.10 has broken minversion checks, +# so we have to disable this until pytest 3.11 +# minversion = 3.3 + python_files = test_*.py markers = gen_test: marks an async tornado test From 87ce4998409e48b62b16d9479fcc04782d582f88 Mon Sep 17 00:00:00 2001 From: Phil Elson Date: Fri, 9 Nov 2018 06:36:17 +0000 Subject: [PATCH 12/81] Add a logo block, and update the docs regarding base.html. --- docs/source/reference/templates.md | 8 ++++---- share/jupyterhub/templates/page.html | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/source/reference/templates.md b/docs/source/reference/templates.md index fefca625..61784403 100644 --- a/docs/source/reference/templates.md +++ b/docs/source/reference/templates.md @@ -25,19 +25,19 @@ supplement the material in the block. The make extensive use of blocks, which allows you to customize parts of the interface easily. -In general, a child template can extend a base template, `base.html`, by beginning with: +In general, a child template can extend a base template, `page.html`, by beginning with: ```html -{% extends "base.html" %} +{% extends "page.html" %} ``` This works, unless you are trying to extend the default template for the same file name. Starting in version 0.9, you may refer to the base file with a -`templates/` prefix. Thus, if you are writing a custom `base.html`, start the +`templates/` prefix. Thus, if you are writing a custom `page.html`, start the file with this block: ```html -{% extends "templates/base.html" %} +{% extends "templates/page.html" %} ``` By defining `block`s with same name as in the base template, child templates diff --git a/share/jupyterhub/templates/page.html b/share/jupyterhub/templates/page.html index 7d973bc0..5677eb93 100644 --- a/share/jupyterhub/templates/page.html +++ b/share/jupyterhub/templates/page.html @@ -96,7 +96,11 @@