From 5eb7a14a3398f57e98d68aad390c14a0cf5fb1e4 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 4 Jun 2019 13:30:28 +0200 Subject: [PATCH 001/156] [WIP] Add support for Jupyter Server --- .travis.yml | 7 +++++++ jupyterhub/singleuser.py | 27 ++++++++++++++++++--------- jupyterhub/spawner.py | 2 +- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 788bfa4f..3980dd87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,6 +61,13 @@ script: make html popd fi + - | + if [[ "$TEST" == "jupyter_server" ]]; then + pip uninstall notebook + pip install jupyter_server + export USE_JUPYTER_SERVER=True + pytest -v --maxfail=2 --cov=jupyterhub jupyterhub/tests + fi after_success: - codecov after_failure: diff --git a/jupyterhub/singleuser.py b/jupyterhub/singleuser.py index 6fa4f350..0ab557a5 100755 --- a/jupyterhub/singleuser.py +++ b/jupyterhub/singleuser.py @@ -6,6 +6,7 @@ import asyncio import json import os import random +import importlib from datetime import datetime from datetime import timezone from textwrap import dedent @@ -20,10 +21,13 @@ from tornado.httpclient import HTTPRequest from tornado.web import HTTPError from tornado.web import RequestHandler +use_serverapp = os.environ.get('USE_JUPYTER_SERVER', 'False') == 'True' + +required_package = 'jupyter_server' if use_serverapp else 'notebook' try: - import notebook + parent_module = importlib.import_module(required_package) except ImportError: - raise ImportError("JupyterHub single-user server requires notebook >= 4.0") + raise ImportError("JupyterHub single-user server requires {}".format(required_package)) from traitlets import ( Any, @@ -38,14 +42,19 @@ from traitlets import ( TraitError, ) -from notebook.notebookapp import ( - NotebookApp, - aliases as notebook_aliases, - flags as notebook_flags, +app_name = 'jupyter_server.serverapp' if use_serverapp else 'notebook.notebookapp' +app_module = importlib.import_module(app_name) + +NotebookApp = getattr(app_module, 'ServerApp' if use_serverapp else 'NotebookApp') +notebook_aliases = app_module.aliases +notebook_flags = app_module.flags + +LoginHandler = getattr(importlib.import_module(required_package + '.auth.login'), 'LoginHandler') +LogoutHandler = getattr(importlib.import_module(required_package + '.auth.logout'), 'LogoutHandler') +IPythonHandler = getattr( + importlib.import_module(required_package + '.base.handlers'), + 'JupyterHandler' if use_serverapp else 'IPythonHandler' ) -from notebook.auth.login import LoginHandler -from notebook.auth.logout import LogoutHandler -from notebook.base.handlers import IPythonHandler from ._version import __version__, _check_version from .log import log_request diff --git a/jupyterhub/spawner.py b/jupyterhub/spawner.py index af84c122..348eb8d5 100644 --- a/jupyterhub/spawner.py +++ b/jupyterhub/spawner.py @@ -931,7 +931,7 @@ class Spawner(LoggingConfigurable): args.append('--notebook-dir=%s' % _quote_safe(notebook_dir)) if self.default_url: default_url = self.format_string(self.default_url) - args.append('--NotebookApp.default_url=%s' % _quote_safe(default_url)) + args.append('--SingleUserNotebookApp.default_url=%s' % _quote_safe(default_url)) if self.debug: args.append('--debug') From cab1bca6fbe7c5b87e38072499c41f26619c9f52 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 4 Jun 2019 13:42:52 +0200 Subject: [PATCH 002/156] Use jupyter_server if notebook package isn't available --- jupyterhub/singleuser.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/jupyterhub/singleuser.py b/jupyterhub/singleuser.py index 0ab557a5..deff5b01 100755 --- a/jupyterhub/singleuser.py +++ b/jupyterhub/singleuser.py @@ -21,13 +21,19 @@ from tornado.httpclient import HTTPRequest from tornado.web import HTTPError from tornado.web import RequestHandler -use_serverapp = os.environ.get('USE_JUPYTER_SERVER', 'False') == 'True' - -required_package = 'jupyter_server' if use_serverapp else 'notebook' try: - parent_module = importlib.import_module(required_package) + import notebook + use_serverapp = False + server_package = 'notebook' + app_name = 'notebook.notebookapp' except ImportError: - raise ImportError("JupyterHub single-user server requires {}".format(required_package)) + try: + import jupyter_server + use_serverapp = True + server_package = 'jupyter_server' + app_name = 'jupyter_server.serverapp' + except ImportError: + raise ImportError("JupyterHub single-user server requires notebook or jupyter_server packages") from traitlets import ( Any, @@ -42,17 +48,15 @@ from traitlets import ( TraitError, ) -app_name = 'jupyter_server.serverapp' if use_serverapp else 'notebook.notebookapp' app_module = importlib.import_module(app_name) - NotebookApp = getattr(app_module, 'ServerApp' if use_serverapp else 'NotebookApp') notebook_aliases = app_module.aliases notebook_flags = app_module.flags -LoginHandler = getattr(importlib.import_module(required_package + '.auth.login'), 'LoginHandler') -LogoutHandler = getattr(importlib.import_module(required_package + '.auth.logout'), 'LogoutHandler') +LoginHandler = getattr(importlib.import_module(server_package + '.auth.login'), 'LoginHandler') +LogoutHandler = getattr(importlib.import_module(server_package + '.auth.logout'), 'LogoutHandler') IPythonHandler = getattr( - importlib.import_module(required_package + '.base.handlers'), + importlib.import_module(server_package + '.base.handlers'), 'JupyterHandler' if use_serverapp else 'IPythonHandler' ) From f830b2a41798b1936639f5b98ce5434dc4c7a135 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 4 Jun 2019 13:45:57 +0200 Subject: [PATCH 003/156] Try to test notebook package is still uninstalled --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3980dd87..c2d83296 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,8 +65,10 @@ script: if [[ "$TEST" == "jupyter_server" ]]; then pip uninstall notebook pip install jupyter_server - export USE_JUPYTER_SERVER=True pytest -v --maxfail=2 --cov=jupyterhub jupyterhub/tests + # Make sure that the notebook package is still not installed + pip list | grep notebook + test $? -eq 1 fi after_success: - codecov From 399def182b639171d1fbf526fb0af43ab47fdff2 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 4 Jun 2019 13:57:26 +0200 Subject: [PATCH 004/156] Actually run jupyter_server test on Travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index c2d83296..d279947d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,6 +90,8 @@ matrix: include: - python: 3.6 env: TEST=lint + - python: 3.6 + env: TEST=jupyter_server - python: 3.6 env: TEST=docs - python: 3.6 From 8f7e25f9a1897d1db1370338380df89b27e06c09 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 4 Jun 2019 14:24:30 +0200 Subject: [PATCH 005/156] Don't make pip uninstall wait for human input --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d279947d..02c0645c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,7 +63,7 @@ script: fi - | if [[ "$TEST" == "jupyter_server" ]]; then - pip uninstall notebook + pip uninstall notebook --yes pip install jupyter_server pytest -v --maxfail=2 --cov=jupyterhub jupyterhub/tests # Make sure that the notebook package is still not installed From e9bc25cce0616b6cf1d14c4aab61b87b13617711 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 4 Jun 2019 14:42:49 +0200 Subject: [PATCH 006/156] Run all tests for jupyter_server regardless of failure --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 02c0645c..19f05529 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,7 +65,8 @@ script: if [[ "$TEST" == "jupyter_server" ]]; then pip uninstall notebook --yes pip install jupyter_server - pytest -v --maxfail=2 --cov=jupyterhub jupyterhub/tests + # Run all tests regardless of failure, so we know what's up + pytest -v --cov=jupyterhub jupyterhub/tests # Make sure that the notebook package is still not installed pip list | grep notebook test $? -eq 1 From 3d0da4f25adfb69ba36ef0af94d751c0f59b34a9 Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Thu, 13 Feb 2020 18:35:17 +0100 Subject: [PATCH 007/156] Adding python-paginate package and using it to paginate admin panel --- jupyterhub/handlers/pages.py | 22 ++++++++++++++++++++-- requirements.txt | 1 + share/jupyterhub/templates/admin.html | 12 +++++++++++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index 3ca03653..c9a15d6a 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -12,7 +12,8 @@ from http.client import responses from jinja2 import TemplateNotFound from tornado import gen from tornado import web -from tornado.httputil import url_concat +from tornado.httputil import url_concat, urlparse +from python_paginate.web.tornado_paginate import Pagination from .. import __version__ from .. import orm @@ -402,6 +403,15 @@ class AdminHandler(BaseHandler): @web.authenticated @admin_only async def get(self): + DEFAULT_PER_PAGE = 100 + + page, per_page, offset = Pagination.get_page_args(self) + _per_page = self.get_arguments("per_page") + # No arg called per_page in the URL, + # avoiding default value from the python_paginate lib + if per_page==10 and len(_per_page)==0: + per_page = DEFAULT_PER_PAGE + available = {'name', 'admin', 'running', 'last_activity'} default_sort = ['admin', 'name'] mapping = {'running': orm.Spawner.server_id} @@ -442,7 +452,7 @@ class AdminHandler(BaseHandler): # get User.col.desc() order objects ordered = [getattr(c, o)() for c, o in zip(cols, orders)] - users = self.db.query(orm.User).outerjoin(orm.Spawner).order_by(*ordered) + users = self.db.query(orm.User).outerjoin(orm.Spawner).order_by(*ordered).limit(per_page).offset(offset) users = [self._user_from_orm(u) for u in users] from itertools import chain @@ -450,9 +460,16 @@ class AdminHandler(BaseHandler): for u in users: running.extend(s for s in u.spawners.values() if s.active) + total = self.db.query(orm.User.id).count() + pagination = Pagination(url=self.request.uri, total=total, record_name='users', + display_msg = 'Displaying {record_name} {start} - {end}. Total {record_name}: {total}', + page=page, per_page=per_page) + + auth_state = await self.current_user.get_auth_state() html = self.render_template( 'admin.html', + request=self.request, current_user=self.current_user, auth_state=auth_state, admin_access=self.settings.get('admin_access', False), @@ -462,6 +479,7 @@ class AdminHandler(BaseHandler): allow_named_servers=self.allow_named_servers, named_server_limit_per_user=self.named_server_limit_per_user, server_version='{} {}'.format(__version__, self.version_hash), + pagination=pagination ) self.finish(html) diff --git a/requirements.txt b/requirements.txt index 825fd409..a6cdb504 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ requests SQLAlchemy>=1.1 tornado>=5.0 traitlets>=4.3.2 +python-paginate \ No newline at end of file diff --git a/share/jupyterhub/templates/admin.html b/share/jupyterhub/templates/admin.html index a0c6fb87..0835d758 100644 --- a/share/jupyterhub/templates/admin.html +++ b/share/jupyterhub/templates/admin.html @@ -18,11 +18,12 @@ {% block main %}
+ {{ pagination.info|safe }} {% block thead %} - {{ th("User (%i)" % users|length, 'name') }} + {{ th("User", 'name') }} {{ th("Admin", 'admin') }} {{ th("Last Activity", 'last_activity') }} {{ th("Running (%i)" % running|length, 'running', colspan=2) }} @@ -102,6 +103,15 @@ {% endfor %}
+ {% if pagination.links %} + + + + + + + + {% endif %}
From 95c5ebb0903dafb678c4fa4da2f396efba960f9c Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Fri, 14 Feb 2020 11:04:11 +0100 Subject: [PATCH 008/156] Fixing pre-commit errors --- jupyterhub/handlers/pages.py | 31 +++++++++++++++++++++---------- requirements.txt | 2 +- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index c9a15d6a..c079b3b9 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -10,10 +10,11 @@ from datetime import datetime from http.client import responses from jinja2 import TemplateNotFound +from python_paginate.web.tornado_paginate import Pagination from tornado import gen from tornado import web -from tornado.httputil import url_concat, urlparse -from python_paginate.web.tornado_paginate import Pagination +from tornado.httputil import url_concat +from tornado.httputil import urlparse from .. import __version__ from .. import orm @@ -407,9 +408,9 @@ class AdminHandler(BaseHandler): page, per_page, offset = Pagination.get_page_args(self) _per_page = self.get_arguments("per_page") - # No arg called per_page in the URL, + # No arg called per_page in the URL, # avoiding default value from the python_paginate lib - if per_page==10 and len(_per_page)==0: + if per_page == 10 and len(_per_page) == 0: per_page = DEFAULT_PER_PAGE available = {'name', 'admin', 'running', 'last_activity'} @@ -452,7 +453,13 @@ class AdminHandler(BaseHandler): # get User.col.desc() order objects ordered = [getattr(c, o)() for c, o in zip(cols, orders)] - users = self.db.query(orm.User).outerjoin(orm.Spawner).order_by(*ordered).limit(per_page).offset(offset) + users = ( + self.db.query(orm.User) + .outerjoin(orm.Spawner) + .order_by(*ordered) + .limit(per_page) + .offset(offset) + ) users = [self._user_from_orm(u) for u in users] from itertools import chain @@ -461,10 +468,14 @@ class AdminHandler(BaseHandler): running.extend(s for s in u.spawners.values() if s.active) total = self.db.query(orm.User.id).count() - pagination = Pagination(url=self.request.uri, total=total, record_name='users', - display_msg = 'Displaying {record_name} {start} - {end}. Total {record_name}: {total}', - page=page, per_page=per_page) - + pagination = Pagination( + url=self.request.uri, + total=total, + record_name='users', + display_msg='Displaying {record_name} {start} - {end}. Total {record_name}: {total}', + page=page, + per_page=per_page, + ) auth_state = await self.current_user.get_auth_state() html = self.render_template( @@ -479,7 +490,7 @@ class AdminHandler(BaseHandler): allow_named_servers=self.allow_named_servers, named_server_limit_per_user=self.named_server_limit_per_user, server_version='{} {}'.format(__version__, self.version_hash), - pagination=pagination + pagination=pagination, ) self.finish(html) diff --git a/requirements.txt b/requirements.txt index a6cdb504..74215f5a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,8 +9,8 @@ pamela prometheus_client>=0.0.21 psutil>=5.6.5; sys_platform == 'win32' python-dateutil +python-paginate requests SQLAlchemy>=1.1 tornado>=5.0 traitlets>=4.3.2 -python-paginate \ No newline at end of file From b2b85eb548869577fbd16935cea49dc02b37b175 Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Fri, 14 Feb 2020 11:47:43 +0100 Subject: [PATCH 009/156] Improving comments --- jupyterhub/handlers/pages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index c079b3b9..2adbef45 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -409,7 +409,8 @@ class AdminHandler(BaseHandler): page, per_page, offset = Pagination.get_page_args(self) _per_page = self.get_arguments("per_page") # No arg called per_page in the URL, - # avoiding default value from the python_paginate lib + # avoiding default value from the python-paginate lib + # https://github.com/lixxu/python-paginate/blob/master/python_paginate/web/tornado_paginate.py#L23 if per_page == 10 and len(_per_page) == 0: per_page = DEFAULT_PER_PAGE From facf52f117c709aa5f21b39139b65cd1a2cd23ff Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Tue, 25 Feb 2020 17:03:01 +0100 Subject: [PATCH 010/156] Removing unneeded pass of request to the template --- jupyterhub/handlers/pages.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index 2adbef45..0bcca7a6 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -481,7 +481,6 @@ class AdminHandler(BaseHandler): auth_state = await self.current_user.get_auth_state() html = self.render_template( 'admin.html', - request=self.request, current_user=self.current_user, auth_state=auth_state, admin_access=self.settings.get('admin_access', False), From 7b6ac158ccf4cd3bc4ffafd4ba3681b0a8ddb3df Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Tue, 25 Feb 2020 19:11:09 +0100 Subject: [PATCH 011/156] Removing python-paginate package and adding minimal Pagination class to enable a pagination API for AdminHandler --- jupyterhub/handlers/pages.py | 68 ++++++++++++++++++++++++++++++------ requirements.txt | 1 - 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index 0bcca7a6..20052b35 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -10,7 +10,6 @@ from datetime import datetime from http.client import responses from jinja2 import TemplateNotFound -from python_paginate.web.tornado_paginate import Pagination from tornado import gen from tornado import web from tornado.httputil import url_concat @@ -404,15 +403,7 @@ class AdminHandler(BaseHandler): @web.authenticated @admin_only async def get(self): - DEFAULT_PER_PAGE = 100 - page, per_page, offset = Pagination.get_page_args(self) - _per_page = self.get_arguments("per_page") - # No arg called per_page in the URL, - # avoiding default value from the python-paginate lib - # https://github.com/lixxu/python-paginate/blob/master/python_paginate/web/tornado_paginate.py#L23 - if per_page == 10 and len(_per_page) == 0: - per_page = DEFAULT_PER_PAGE available = {'name', 'admin', 'running', 'last_activity'} default_sort = ['admin', 'name'] @@ -472,8 +463,6 @@ class AdminHandler(BaseHandler): pagination = Pagination( url=self.request.uri, total=total, - record_name='users', - display_msg='Displaying {record_name} {start} - {end}. Total {record_name}: {total}', page=page, per_page=per_page, ) @@ -619,6 +608,63 @@ class HealthCheckHandler(BaseHandler): def get(self, *args): self.finish() +class Pagination(BaseHandler): + + _page_name = 'page' + _per_page_name = 'per_page' + _default_page = 1 + _default_per_page = 100 + _max_per_page = 250 + _record_name='users' + _display_msg='Displaying {record_name} {start} - {end}. Total {record_name}: {total}' + + def __init__(self, *args, **kwargs): + """Detail parameters remark. + **url**: current request url + **page**: current page + **per_page**: how many records displayed on one page. By default 100 + **total**: total records for pagination + **display_msg**: text for pagation information + **record_name**: record name showed in pagination information + """ + self.page = kwargs.get(self._page_name, 1) + + if self.per_page > self._max_per_page: + self.per_page = self._max_per_page + + self.total = int(kwargs.get('total', 0)) + self.display_msg = kwargs.get('display_msg', self._display_msg) + + self.record_name = kwargs.get('record_name', self._record_name) + self.url = kwargs.get('url') or self.get_url() + self.init_values() + + def init_values(self): + self._cached = {} + self.skip = (self.page - 1) * self.per_page + pages = divmod(self.total, self.per_page) + self.total_pages = pages[0] + 1 if pages[1] else pages[0] + + self.has_prev = self.page > 1 + self.has_next = self.page < self.total_pages + + @classmethod + def get_page_args(self, handler): + self.page = handler.get_argument(self._page_name, self._default_page) + self.per_page = handler.get_argument(self._per_page_name, self._default_per_page) + try: + self.per_page = int(self.per_page) + if self.per_page > self._max_per_page: + self.per_page = self._max_per_page + except: + self.per_page = self._default_per_page + + try: + self.page = int(self.page) + except: + self.page = self._default_page + + return self.page, self.per_page, self.per_page * (self.page - 1) default_handlers = [ (r'/', RootHandler), diff --git a/requirements.txt b/requirements.txt index 74215f5a..825fd409 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,6 @@ pamela prometheus_client>=0.0.21 psutil>=5.6.5; sys_platform == 'win32' python-dateutil -python-paginate requests SQLAlchemy>=1.1 tornado>=5.0 From 18205fbf4a14cdfac6e4e148b1feff09d18ac751 Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Wed, 26 Feb 2020 10:36:36 +0100 Subject: [PATCH 012/156] Fixing black formatting issues --- jupyterhub/handlers/pages.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index 20052b35..d2ebf235 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -461,10 +461,7 @@ class AdminHandler(BaseHandler): total = self.db.query(orm.User.id).count() pagination = Pagination( - url=self.request.uri, - total=total, - page=page, - per_page=per_page, + url=self.request.uri, total=total, page=page, per_page=per_page, ) auth_state = await self.current_user.get_auth_state() @@ -608,6 +605,7 @@ class HealthCheckHandler(BaseHandler): def get(self, *args): self.finish() + class Pagination(BaseHandler): _page_name = 'page' @@ -615,8 +613,10 @@ class Pagination(BaseHandler): _default_page = 1 _default_per_page = 100 _max_per_page = 250 - _record_name='users' - _display_msg='Displaying {record_name} {start} - {end}. Total {record_name}: {total}' + _record_name = 'users' + _display_msg = ( + 'Displaying {record_name} {start} - {end}. Total {record_name}: {total}' + ) def __init__(self, *args, **kwargs): """Detail parameters remark. @@ -634,7 +634,7 @@ class Pagination(BaseHandler): self.total = int(kwargs.get('total', 0)) self.display_msg = kwargs.get('display_msg', self._display_msg) - + self.record_name = kwargs.get('record_name', self._record_name) self.url = kwargs.get('url') or self.get_url() self.init_values() @@ -651,7 +651,9 @@ class Pagination(BaseHandler): @classmethod def get_page_args(self, handler): self.page = handler.get_argument(self._page_name, self._default_page) - self.per_page = handler.get_argument(self._per_page_name, self._default_per_page) + self.per_page = handler.get_argument( + self._per_page_name, self._default_per_page + ) try: self.per_page = int(self.per_page) if self.per_page > self._max_per_page: @@ -666,6 +668,7 @@ class Pagination(BaseHandler): return self.page, self.per_page, self.per_page * (self.page - 1) + default_handlers = [ (r'/', RootHandler), (r'/home', HomeHandler), From f49cc1fcf0b04f0b7417f8bc32dd100e0fc9fe6b Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Wed, 26 Feb 2020 10:40:44 +0100 Subject: [PATCH 013/156] Improving description of potential parameters --- jupyterhub/handlers/pages.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index d2ebf235..3bb7b63a 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -619,13 +619,13 @@ class Pagination(BaseHandler): ) def __init__(self, *args, **kwargs): - """Detail parameters remark. - **url**: current request url - **page**: current page - **per_page**: how many records displayed on one page. By default 100 - **total**: total records for pagination - **display_msg**: text for pagation information - **record_name**: record name showed in pagination information + """Potential parameters. + **url**: URL in request + **page**: current page in use + **per_page**: number of records to display in the page. By default 100 + **total**: total records considered while paginating + **display_msg**: informative text for pagination + **record_name**: name of the record, showed in pagination info """ self.page = kwargs.get(self._page_name, 1) From a2e2b1d5122a5ca613c30a05d5b612574535bf6c Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Fri, 28 Feb 2020 12:01:56 +0100 Subject: [PATCH 014/156] As pointed out in the PR, Pagination isn't a Handler --- jupyterhub/handlers/pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index 3bb7b63a..8a7e2033 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -606,7 +606,7 @@ class HealthCheckHandler(BaseHandler): self.finish() -class Pagination(BaseHandler): +class Pagination(): _page_name = 'page' _per_page_name = 'per_page' From ede71db11af9dd2680fbce9d84ab670c869bde8a Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Fri, 28 Feb 2020 12:04:53 +0100 Subject: [PATCH 015/156] Moving Pagination class to its own file --- jupyterhub/handlers/pages.py | 65 +----------------------------------- jupyterhub/pagination.py | 65 ++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 64 deletions(-) create mode 100644 jupyterhub/pagination.py diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index 8a7e2033..60b6ae50 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -23,6 +23,7 @@ from ..utils import admin_only from ..utils import maybe_future from ..utils import url_path_join from .base import BaseHandler +from ..pagination import Pagination class RootHandler(BaseHandler): @@ -605,70 +606,6 @@ class HealthCheckHandler(BaseHandler): def get(self, *args): self.finish() - -class Pagination(): - - _page_name = 'page' - _per_page_name = 'per_page' - _default_page = 1 - _default_per_page = 100 - _max_per_page = 250 - _record_name = 'users' - _display_msg = ( - 'Displaying {record_name} {start} - {end}. Total {record_name}: {total}' - ) - - def __init__(self, *args, **kwargs): - """Potential parameters. - **url**: URL in request - **page**: current page in use - **per_page**: number of records to display in the page. By default 100 - **total**: total records considered while paginating - **display_msg**: informative text for pagination - **record_name**: name of the record, showed in pagination info - """ - self.page = kwargs.get(self._page_name, 1) - - if self.per_page > self._max_per_page: - self.per_page = self._max_per_page - - self.total = int(kwargs.get('total', 0)) - self.display_msg = kwargs.get('display_msg', self._display_msg) - - self.record_name = kwargs.get('record_name', self._record_name) - self.url = kwargs.get('url') or self.get_url() - self.init_values() - - def init_values(self): - self._cached = {} - self.skip = (self.page - 1) * self.per_page - pages = divmod(self.total, self.per_page) - self.total_pages = pages[0] + 1 if pages[1] else pages[0] - - self.has_prev = self.page > 1 - self.has_next = self.page < self.total_pages - - @classmethod - def get_page_args(self, handler): - self.page = handler.get_argument(self._page_name, self._default_page) - self.per_page = handler.get_argument( - self._per_page_name, self._default_per_page - ) - try: - self.per_page = int(self.per_page) - if self.per_page > self._max_per_page: - self.per_page = self._max_per_page - except: - self.per_page = self._default_per_page - - try: - self.page = int(self.page) - except: - self.page = self._default_page - - return self.page, self.per_page, self.per_page * (self.page - 1) - - default_handlers = [ (r'/', RootHandler), (r'/home', HomeHandler), diff --git a/jupyterhub/pagination.py b/jupyterhub/pagination.py new file mode 100644 index 00000000..b1396cd8 --- /dev/null +++ b/jupyterhub/pagination.py @@ -0,0 +1,65 @@ +"""Basic class to manage pagination utils.""" +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +class Pagination(): + + _page_name = 'page' + _per_page_name = 'per_page' + _default_page = 1 + _default_per_page = 100 + _max_per_page = 250 + _record_name = 'users' + _display_msg = ( + 'Displaying {record_name} {start} - {end}. Total {record_name}: {total}' + ) + + def __init__(self, *args, **kwargs): + """Potential parameters. + **url**: URL in request + **page**: current page in use + **per_page**: number of records to display in the page. By default 100 + **total**: total records considered while paginating + **display_msg**: informative text for pagination + **record_name**: name of the record, showed in pagination info + """ + self.page = kwargs.get(self._page_name, 1) + + if self.per_page > self._max_per_page: + self.per_page = self._max_per_page + + self.total = int(kwargs.get('total', 0)) + self.display_msg = kwargs.get('display_msg', self._display_msg) + + self.record_name = kwargs.get('record_name', self._record_name) + self.url = kwargs.get('url') or self.get_url() + self.init_values() + + def init_values(self): + self._cached = {} + self.skip = (self.page - 1) * self.per_page + pages = divmod(self.total, self.per_page) + self.total_pages = pages[0] + 1 if pages[1] else pages[0] + + self.has_prev = self.page > 1 + self.has_next = self.page < self.total_pages + + @classmethod + def get_page_args(self, handler): + self.page = handler.get_argument(self._page_name, self._default_page) + self.per_page = handler.get_argument( + self._per_page_name, self._default_per_page + ) + try: + self.per_page = int(self.per_page) + if self.per_page > self._max_per_page: + self.per_page = self._max_per_page + except: + self.per_page = self._default_per_page + + try: + self.page = int(self.page) + except: + self.page = self._default_page + + return self.page, self.per_page, self.per_page * (self.page - 1) From 53927f04901f8d076ce3b49730cdbc0a44b109a8 Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Fri, 28 Feb 2020 12:05:50 +0100 Subject: [PATCH 016/156] Pre-commit fixes --- jupyterhub/handlers/pages.py | 3 ++- jupyterhub/pagination.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index 60b6ae50..676bdf54 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -19,11 +19,11 @@ from .. import __version__ from .. import orm from ..metrics import SERVER_POLL_DURATION_SECONDS from ..metrics import ServerPollStatus +from ..pagination import Pagination from ..utils import admin_only from ..utils import maybe_future from ..utils import url_path_join from .base import BaseHandler -from ..pagination import Pagination class RootHandler(BaseHandler): @@ -606,6 +606,7 @@ class HealthCheckHandler(BaseHandler): def get(self, *args): self.finish() + default_handlers = [ (r'/', RootHandler), (r'/home', HomeHandler), diff --git a/jupyterhub/pagination.py b/jupyterhub/pagination.py index b1396cd8..8f16bbe0 100644 --- a/jupyterhub/pagination.py +++ b/jupyterhub/pagination.py @@ -2,7 +2,8 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -class Pagination(): + +class Pagination: _page_name = 'page' _per_page_name = 'per_page' From 392525571f2b0f5858e381b14c3f7c12aae509e0 Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Fri, 28 Feb 2020 12:14:59 +0100 Subject: [PATCH 017/156] Documenting get_page_args method --- jupyterhub/pagination.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jupyterhub/pagination.py b/jupyterhub/pagination.py index 8f16bbe0..d961f478 100644 --- a/jupyterhub/pagination.py +++ b/jupyterhub/pagination.py @@ -47,6 +47,15 @@ class Pagination: @classmethod def get_page_args(self, handler): + """ + This method gets the arguments used in the webpage to configurate the pagination + In case of no arguments, it uses the default values from this class + + It returns: + - self.page: The page requested for paginating or the default value (1) + - self.per_page: The number of items to return in this page. By default 100 and no more than 250 + - self.per_page * (self.page - 1): The offset to consider when managing pagination via the ORM + """ self.page = handler.get_argument(self._page_name, self._default_page) self.per_page = handler.get_argument( self._per_page_name, self._default_per_page From e82c06cf93c5de13eedac7a5715d4b8f5ecf6b52 Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Fri, 28 Feb 2020 12:31:53 +0100 Subject: [PATCH 018/156] Removing display_msg and record name since it can be coded directly as they're needed in the templates --- jupyterhub/pagination.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/jupyterhub/pagination.py b/jupyterhub/pagination.py index d961f478..76d10342 100644 --- a/jupyterhub/pagination.py +++ b/jupyterhub/pagination.py @@ -10,10 +10,6 @@ class Pagination: _default_page = 1 _default_per_page = 100 _max_per_page = 250 - _record_name = 'users' - _display_msg = ( - 'Displaying {record_name} {start} - {end}. Total {record_name}: {total}' - ) def __init__(self, *args, **kwargs): """Potential parameters. @@ -21,8 +17,6 @@ class Pagination: **page**: current page in use **per_page**: number of records to display in the page. By default 100 **total**: total records considered while paginating - **display_msg**: informative text for pagination - **record_name**: name of the record, showed in pagination info """ self.page = kwargs.get(self._page_name, 1) @@ -30,9 +24,6 @@ class Pagination: self.per_page = self._max_per_page self.total = int(kwargs.get('total', 0)) - self.display_msg = kwargs.get('display_msg', self._display_msg) - - self.record_name = kwargs.get('record_name', self._record_name) self.url = kwargs.get('url') or self.get_url() self.init_values() From 8f46d89ac05c457a05f50696937e4416e06117f9 Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Fri, 28 Feb 2020 13:19:53 +0100 Subject: [PATCH 019/156] Adding info method to pagination and related items in admin template --- jupyterhub/pagination.py | 13 +++++++++++++ share/jupyterhub/templates/admin.html | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/jupyterhub/pagination.py b/jupyterhub/pagination.py index 76d10342..440e4e26 100644 --- a/jupyterhub/pagination.py +++ b/jupyterhub/pagination.py @@ -64,3 +64,16 @@ class Pagination: self.page = self._default_page return self.page, self.per_page, self.per_page * (self.page - 1) + + @property + def info(self): + """Get the pagination information.""" + start = 1 + (self.page - 1) * self.per_page + end = start + self.per_page - 1 + if end > self.total: + end = self.total + + if start > self.total: + start = self.total + + return {'total': self.total, 'start': start, 'end': end} diff --git a/share/jupyterhub/templates/admin.html b/share/jupyterhub/templates/admin.html index 0835d758..b98108de 100644 --- a/share/jupyterhub/templates/admin.html +++ b/share/jupyterhub/templates/admin.html @@ -18,7 +18,9 @@ {% block main %}
- {{ pagination.info|safe }} +
+ Displaying users {{ pagination.info.start|safe }} - {{ pagination.info.end|safe }}. Total users {{ pagination.info.total|safe }} +
From 534e251f97ffc39ea21d63ecf75cac4f35800d74 Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Fri, 28 Feb 2020 17:15:19 +0100 Subject: [PATCH 020/156] Adding links generation inside the Pagination class --- jupyterhub/pagination.py | 86 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/jupyterhub/pagination.py b/jupyterhub/pagination.py index 440e4e26..95e5293b 100644 --- a/jupyterhub/pagination.py +++ b/jupyterhub/pagination.py @@ -77,3 +77,89 @@ class Pagination: start = self.total return {'total': self.total, 'start': start, 'end': end} + + def calculate_pages_window(self): + """Calculates the set of pages to render later in links() method. + It returns the list of pages to render via links for the pagination + By default, as we've observed in other applications, we're going to render + only a finite and predefined number of pages, avoiding visual fatigue related + to a long list of pages. By default, we render 7 pages plus some inactive links with the characters '...' + to point out that there are other pages that aren't explicitly rendered. + The primary way of work is to provide current webpage and 5 next pages, the last 2 ones + (in case the current page + 5 does not overflow the total lenght of pages) and the first one for reference. + """ + + self.separator_character = '...' + default_pages_to_render = 7 + after_page = 5 + before_end = 2 + + pages = [] + + if self.total_pages > default_pages_to_render: + if self.page > 1: + pages.extend([1, '...']) + + if self.total_pages < self.page + after_page: + pages.extend(list(range(self.page, self.total_pages))) + else: + if self.total_pages > self.page + after_page + before_end: + pages.extend(list(range(self.page, self.page + after_page))) + pages.append('...') + pages.extend( + list(range(self.total_pages - before_end, self.total_pages)) + ) + else: + pages.extend(list(range(self.page, self.page + after_page))) + + return pages + + else: + return list(range(1, self.total_pages)) + + @property + def links(self): + """Sets the links for the pagination. + Getting the input from calculate_pages_window(), generates the HTML code + for the pages to render, plus the arrows to go onwards and backwards (if needed). + """ + + pages_to_render = self.calculate_pages_window() + print(f"pages_to_render {pages_to_render} ") + + links = ['') + + return ''.join(links) From 492c5072b714c50a9e34460cd2ca8fc1a91d6e8c Mon Sep 17 00:00:00 2001 From: Juan Cruz-Benito Date: Fri, 28 Feb 2020 17:31:19 +0100 Subject: [PATCH 021/156] =?UTF-8?q?Removing=20print=20statements=20?= =?UTF-8?q?=F0=9F=A4=A6=E2=80=8D=E2=99=82=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- jupyterhub/pagination.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/jupyterhub/pagination.py b/jupyterhub/pagination.py index 95e5293b..3f82eb35 100644 --- a/jupyterhub/pagination.py +++ b/jupyterhub/pagination.py @@ -125,7 +125,6 @@ class Pagination: """ pages_to_render = self.calculate_pages_window() - print(f"pages_to_render {pages_to_render} ") links = [' {% endblock user_row %} + {% endfor %} {% endfor %} From 485c7b72c29ba8ff37ca43e55a9c38de20dea1f4 Mon Sep 17 00:00:00 2001 From: Josh Meek Date: Thu, 16 Apr 2020 09:36:52 -0400 Subject: [PATCH 031/156] Fix use of auxiliary verb on index.rst --- docs/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index c9722089..8d51a6bc 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,7 +7,7 @@ It can be used in a class of students, a corporate data science group or scienti research group. It is a multi-user **Hub** that spawns, manages, and proxies multiple instances of the single-user `Jupyter notebook`_ server. -To make life easier, JupyterHub have distributions. Be sure to +To make life easier, JupyterHub has distributions. Be sure to take a look at them before continuing with the configuration of the broad original system of `JupyterHub`_. Today, you can find two main cases: From 8648285375ef47f9504e6c5f06bde8918531ee04 Mon Sep 17 00:00:00 2001 From: Thijs Walcarius Date: Fri, 17 Apr 2020 10:00:25 +0200 Subject: [PATCH 032/156] Fix broken test due to BeautifulSoup 4.9.0 behavior change cfr. https://bugs.launchpad.net/beautifulsoup/+bug/1871335 --- jupyterhub/tests/test_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyterhub/tests/test_pages.py b/jupyterhub/tests/test_pages.py index b9d6bba5..20f3ea45 100644 --- a/jupyterhub/tests/test_pages.py +++ b/jupyterhub/tests/test_pages.py @@ -354,7 +354,7 @@ async def test_spawn_pending(app, username, slow_spawn): assert page.find('div', {'class': 'progress'}) # validate event source url by consuming it - script = page.body.find('script').text + script = page.body.find('script').string assert 'EventSource' in script # find EventSource url in javascript # maybe not the most robust way to check this? From c234463a6765491c6a8baee10541ad67be28b468 Mon Sep 17 00:00:00 2001 From: Richard Darst Date: Thu, 16 Apr 2020 11:47:07 +0300 Subject: [PATCH 033/156] sphinx conf.py: update add_stylesheet -> add_css_file - Seems to be added in 1.0: https://www.sphinx-doc.org/en/latest/changes.html#release-1-0-jul-23-2010 --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index fd0d4b92..14b548ee 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -62,7 +62,7 @@ from recommonmark.transform import AutoStructify def setup(app): app.add_config_value('recommonmark_config', {'enable_eval_rst': True}, True) - app.add_stylesheet('custom.css') + app.add_css_file('custom.css') app.add_transform(AutoStructify) From e882e7954c0473d2941c0fbc7d5f8c8e04a8658a Mon Sep 17 00:00:00 2001 From: Richard Darst Date: Thu, 16 Apr 2020 16:59:05 +0300 Subject: [PATCH 034/156] docs: use recommonmark as an extension - source_parsers deprecated in sphinx 3.0 - Since sphinx 1.4, it can (should) be used as a direct extension: https://github.com/readthedocs/recommonmark/pull/43 --- docs/source/conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 14b548ee..64e4939a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -20,6 +20,7 @@ extensions = [ 'autodoc_traits', 'sphinx_copybutton', 'sphinx-jsonschema', + 'recommonmark', ] templates_path = ['_templates'] @@ -66,8 +67,6 @@ def setup(app): app.add_transform(AutoStructify) -source_parsers = {'.md': 'recommonmark.parser.CommonMarkParser'} - source_suffix = ['.rst', '.md'] # source_encoding = 'utf-8-sig' From 9d6e8e6b6fb02ca7952b158bf665b3ad8a8a0223 Mon Sep 17 00:00:00 2001 From: Richard Darst Date: Thu, 16 Apr 2020 11:45:35 +0300 Subject: [PATCH 035/156] Temporary patch autodoc-traits to fix build error [temporary] - This commit should be removed later after autodoc-traits is fixed upstream --- docs/requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index e0ea5aa4..472fcc28 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,9 @@ -r ../requirements.txt alabaster_jupyterhub -autodoc-traits +# Temporary fix of #3021. Revert back to released autodoc-traits when +# 0.1.0 released. +https://github.com/jupyterhub/autodoc-traits/archive/75885ee24636efbfebfceed1043459715049cd84.zip pydata-sphinx-theme recommonmark>=0.6 sphinx-copybutton From 6f2e409fb9bb4959310f26f2316bc4d4d228acc0 Mon Sep 17 00:00:00 2001 From: Thijs Walcarius Date: Mon, 6 Apr 2020 16:26:16 +0200 Subject: [PATCH 036/156] Allow bypassing of spawn form by calling options in query arguments of /spawn --- jupyterhub/handlers/pages.py | 26 ++++++++++++++++++++++++++ jupyterhub/tests/test_pages.py | 22 ++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index 5cd1f4a7..6f1a43f2 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -173,6 +173,32 @@ class SpawnHandler(BaseHandler): auth_state = await user.get_auth_state() await spawner.run_auth_state_hook(auth_state) + # Try to start server directly when query arguments are passed. + form_options = {} + for key, byte_list in self.request.query_arguments.items(): + form_options[key] = [bs.decode('utf8') for bs in byte_list] + + # 'next' is reserved argument for redirect after spawn + form_options.pop('next', None) + + if len(form_options) > 0: + try: + self.log.debug( + "Triggering spawn with supplied query arguments for %s", + spawner._log_name, + ) + options = await maybe_future(spawner.options_from_form(form_options)) + pending_url = self._get_pending_url(user, server_name) + return await self._wrap_spawn_single_user( + user, server_name, spawner, pending_url, options + ) + except Exception as e: + self.log.error( + "Failed to spawn single-user server with query arguments", + exc_info=True, + ) + # fallback to behavior without failing query arguments + spawner_options_form = await spawner.get_options_form() if spawner_options_form: self.log.debug("Serving options form for %s", spawner._log_name) diff --git a/jupyterhub/tests/test_pages.py b/jupyterhub/tests/test_pages.py index 20f3ea45..7ca4cb22 100644 --- a/jupyterhub/tests/test_pages.py +++ b/jupyterhub/tests/test_pages.py @@ -255,6 +255,28 @@ async def test_spawn_page_admin(app, admin_access): assert "Spawning server for {}".format(u.name) in r.text +async def test_spawn_with_query_arguments(app): + with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): + base_url = ujoin(public_host(app), app.hub.base_url) + cookies = await app.login_user('jones') + orm_u = orm.User.find(app.db, 'jones') + u = app.users[orm_u] + await u.stop() + next_url = ujoin(app.base_url, 'user/jones/tree') + r = await async_requests.get( + url_concat( + ujoin(base_url, 'spawn'), {'next': next_url, 'energy': '510keV'}, + ), + cookies=cookies, + ) + r.raise_for_status() + assert r.history + assert u.spawner.user_options == { + 'energy': '510keV', + 'notspecified': 5, + } + + async def test_spawn_form(app): with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): base_url = ujoin(public_host(app), app.hub.base_url) From 24387664184846db36cc0a13f984b5d3a07212a8 Mon Sep 17 00:00:00 2001 From: Thijs Walcarius Date: Tue, 7 Apr 2020 13:56:32 +0200 Subject: [PATCH 037/156] Show error message when spawning via query-arguments failed. Add options_from_query function --- jupyterhub/handlers/pages.py | 16 ++++++++++------ jupyterhub/spawner.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index 6f1a43f2..93c59fdb 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -174,20 +174,21 @@ class SpawnHandler(BaseHandler): await spawner.run_auth_state_hook(auth_state) # Try to start server directly when query arguments are passed. - form_options = {} + error_message = '' + query_options = {} for key, byte_list in self.request.query_arguments.items(): - form_options[key] = [bs.decode('utf8') for bs in byte_list] + query_options[key] = [bs.decode('utf8') for bs in byte_list] # 'next' is reserved argument for redirect after spawn - form_options.pop('next', None) + query_options.pop('next', None) - if len(form_options) > 0: + if len(query_options) > 0: try: self.log.debug( "Triggering spawn with supplied query arguments for %s", spawner._log_name, ) - options = await maybe_future(spawner.options_from_form(form_options)) + options = await maybe_future(spawner.options_from_query(query_options)) pending_url = self._get_pending_url(user, server_name) return await self._wrap_spawn_single_user( user, server_name, spawner, pending_url, options @@ -197,13 +198,16 @@ class SpawnHandler(BaseHandler): "Failed to spawn single-user server with query arguments", exc_info=True, ) + error_message = str(e) # fallback to behavior without failing query arguments spawner_options_form = await spawner.get_options_form() if spawner_options_form: self.log.debug("Serving options form for %s", spawner._log_name) form = await self._render_form( - for_user=user, spawner_options_form=spawner_options_form + for_user=user, + spawner_options_form=spawner_options_form, + message=error_message, ) self.finish(form) else: diff --git a/jupyterhub/spawner.py b/jupyterhub/spawner.py index 4abc355b..54875837 100644 --- a/jupyterhub/spawner.py +++ b/jupyterhub/spawner.py @@ -384,6 +384,37 @@ class Spawner(LoggingConfigurable): """ return form_data + def options_from_query(self, query_data): + """Interpret query arguments passed to /spawn + + Query arguments will always arrive as a dict of unicode strings. + Override this function to understand single-values, numbers, etc. + + By default, options_from_form is called from this function. You can however override + this function if you need to process the query arguments differently. + + This should coerce form data into the structure expected by self.user_options, + which must be a dict, and should be JSON-serializeable, + though it can contain bytes in addition to standard JSON data types. + + This method should not have any side effects. + Any handling of `user_options` should be done in `.start()` + to ensure consistent behavior across servers + spawned via the API and form submission page. + + Instances will receive this data on self.user_options, after passing through this function, + prior to `Spawner.start`. + + .. versionadded:: 1.2 + user_options are persisted in the JupyterHub database to be reused + on subsequent spawns if no options are given. + user_options is serialized to JSON as part of this persistence + (with additional support for bytes in case of uploaded file data), + and any non-bytes non-jsonable values will be replaced with None + if the user_options are re-used. + """ + return self.options_from_form(query_data) + user_options = Dict( help=""" Dict of user specified options for the user's spawned instance of a single-user server. From 6283e7ec837e25a80de3add190d11d4a9b46e9c3 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 15 Apr 2020 23:21:53 +0200 Subject: [PATCH 038/156] support kubespawner running on a IPv6 only cluster --- jupyterhub/app.py | 10 +++++++--- jupyterhub/objects.py | 13 +++++++++---- jupyterhub/services/service.py | 2 +- jupyterhub/user.py | 7 ++++++- jupyterhub/utils.py | 4 ++-- 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 727b964f..50809e19 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -561,7 +561,11 @@ class JupyterHub(Application): def _url_part_changed(self, change): """propagate deprecated ip/port/base_url config to the bind_url""" urlinfo = urlparse(self.bind_url) - urlinfo = urlinfo._replace(netloc='%s:%i' % (self.ip, self.port)) + if ':' in self.ip: + fmt = '[%s]:%i' + else: + fmt = '%s:%i' + urlinfo = urlinfo._replace(netloc=fmt % (self.ip, self.port)) urlinfo = urlinfo._replace(path=self.base_url) bind_url = urlunparse(urlinfo) if bind_url != self.bind_url: @@ -727,10 +731,10 @@ class JupyterHub(Application): help="""The ip or hostname for proxies and spawners to use for connecting to the Hub. - Use when the bind address (`hub_ip`) is 0.0.0.0 or otherwise different + Use when the bind address (`hub_ip`) is 0.0.0.0, :: or otherwise different from the connect address. - Default: when `hub_ip` is 0.0.0.0, use `socket.gethostname()`, otherwise use `hub_ip`. + Default: when `hub_ip` is 0.0.0.0 or ::, use `socket.gethostname()`, otherwise use `hub_ip`. Note: Some spawners or proxy implementations might not support hostnames. Check your spawner or proxy documentation to see if they have extra requirements. diff --git a/jupyterhub/objects.py b/jupyterhub/objects.py index de2ac7a6..e5aa9c1c 100644 --- a/jupyterhub/objects.py +++ b/jupyterhub/objects.py @@ -53,7 +53,7 @@ class Server(HasTraits): Never used in APIs, only logging, since it can be non-connectable value, such as '', meaning all interfaces. """ - if self.ip in {'', '0.0.0.0'}: + if self.ip in {'', '0.0.0.0', '::'}: return self.url.replace(self._connect_ip, self.ip or '*', 1) return self.url @@ -87,13 +87,13 @@ class Server(HasTraits): """The address to use when connecting to this server When `ip` is set to a real ip address, the same value is used. - When `ip` refers to 'all interfaces' (e.g. '0.0.0.0'), + When `ip` refers to 'all interfaces' (e.g. '0.0.0.0' or '::'), clients connect via hostname by default. Setting `connect_ip` explicitly overrides any default behavior. """ if self.connect_ip: return self.connect_ip - elif self.ip in {'', '0.0.0.0'}: + elif self.ip in {'', '0.0.0.0', '::'}: # if listening on all interfaces, default to hostname for connect return socket.gethostname() else: @@ -149,7 +149,12 @@ class Server(HasTraits): if self.connect_url: parsed = urlparse(self.connect_url) return "{proto}://{host}".format(proto=parsed.scheme, host=parsed.netloc) - return "{proto}://{ip}:{port}".format( + + if ':' in self._connect_ip: + fmt = "{proto}://[{ip}]:{port}" + else: + fmt = "{proto}://{ip}:{port}" + return fmt.format( proto=self.proto, ip=self._connect_ip, port=self._connect_port ) diff --git a/jupyterhub/services/service.py b/jupyterhub/services/service.py index fbf1f331..dcd28946 100644 --- a/jupyterhub/services/service.py +++ b/jupyterhub/services/service.py @@ -342,7 +342,7 @@ class Service(LoggingConfigurable): env['JUPYTERHUB_SERVICE_PREFIX'] = self.server.base_url hub = self.hub - if self.hub.ip in ('0.0.0.0', ''): + if self.hub.ip in ('', '0.0.0.0', '::'): # if the Hub is listening on all interfaces, # tell services to connect via localhost # since they are always local subprocesses diff --git a/jupyterhub/user.py b/jupyterhub/user.py index 71ceac4e..4f049817 100644 --- a/jupyterhub/user.py +++ b/jupyterhub/user.py @@ -566,7 +566,12 @@ class User: else: # >= 0.7 returns (ip, port) proto = 'https' if self.settings['internal_ssl'] else 'http' - url = '%s://%s:%i' % ((proto,) + url) + + # check if spawner returned an IPv6 address + if ':' in url[0]: + url = '%s://[%s]:%i' % ((proto,) + url) + else: + url = '%s://%s:%i' % ((proto,) + url) urlinfo = urlparse(url) server.proto = urlinfo.scheme server.ip = urlinfo.hostname diff --git a/jupyterhub/utils.py b/jupyterhub/utils.py index 851b4d53..82bcc0c8 100644 --- a/jupyterhub/utils.py +++ b/jupyterhub/utils.py @@ -66,7 +66,7 @@ def can_connect(ip, port): Return True if we can connect, False otherwise. """ - if ip in {'', '0.0.0.0'}: + if ip in {'', '0.0.0.0', '::'}: ip = '127.0.0.1' try: socket.create_connection((ip, port)).close() @@ -179,7 +179,7 @@ async def exponential_backoff( async def wait_for_server(ip, port, timeout=10): """Wait for any server to show up at ip:port.""" - if ip in {'', '0.0.0.0'}: + if ip in {'', '0.0.0.0', '::'}: ip = '127.0.0.1' await exponential_backoff( lambda: can_connect(ip, port), From 17dde3a2a9d14b836ad7774b5b755a3391279ff6 Mon Sep 17 00:00:00 2001 From: Michael Blackmon Date: Mon, 20 Apr 2020 10:38:19 -0400 Subject: [PATCH 039/156] remove margin styling from submit button --- share/jupyterhub/static/less/login.less | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/share/jupyterhub/static/less/login.less b/share/jupyterhub/static/less/login.less index d0040908..f5ca78e5 100644 --- a/share/jupyterhub/static/less/login.less +++ b/share/jupyterhub/static/less/login.less @@ -6,7 +6,7 @@ .bg-warning(); padding:10px; } - + .service-login { text-align: center; display: table-cell; @@ -27,9 +27,9 @@ } input[type=submit] { - margin-top: 16px; + margin-top: 0px; } - + .form-control:focus, input[type=submit]:focus { box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px @jupyter-orange; border-color: @jupyter-orange; From 5fbd2838c9edbcc7bc40003597174bab30f50939 Mon Sep 17 00:00:00 2001 From: Michael Blackmon Date: Mon, 20 Apr 2020 10:39:57 -0400 Subject: [PATCH 040/156] add style class for feedback, widget and container --- share/jupyterhub/static/less/page.less | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/share/jupyterhub/static/less/page.less b/share/jupyterhub/static/less/page.less index 6193b357..328ffe7e 100644 --- a/share/jupyterhub/static/less/page.less +++ b/share/jupyterhub/static/less/page.less @@ -26,3 +26,19 @@ // .progress-log-event:hover { // background: rgba(66, 165, 245, 0.2); // } + + +.feedback { + &-container { + margin-top: 16px; + } + + &-widget { + padding: 5px 0px 0px 6px; + i { + font-size: 2em; + color: lightgrey; + } + } + +} From 42e7d1a3fbc2d997258a88f98716d80dc2aa6798 Mon Sep 17 00:00:00 2001 From: Michael Blackmon Date: Mon, 20 Apr 2020 10:59:34 -0400 Subject: [PATCH 041/156] put submit button & widget in feedback-container; extend template to include script block with form onsubmit handler --- share/jupyterhub/templates/spawn.html | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/share/jupyterhub/templates/spawn.html b/share/jupyterhub/templates/spawn.html index f1330b1f..d44bdd0d 100644 --- a/share/jupyterhub/templates/spawn.html +++ b/share/jupyterhub/templates/spawn.html @@ -23,9 +23,26 @@
{{spawner_options_form | safe}}
- + {% endblock %} + +{% block script %} +{{ super() }} + +{% endblock %} From 10bb5ef3c0ce25cb131aa134d89433bb4519f556 Mon Sep 17 00:00:00 2001 From: Michael Blackmon Date: Mon, 20 Apr 2020 11:00:40 -0400 Subject: [PATCH 042/156] wrap button & widget in feedback-container; add js block with onsubmit handler --- share/jupyterhub/templates/login.html | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/share/jupyterhub/templates/login.html b/share/jupyterhub/templates/login.html index 130339c9..cd585ded 100644 --- a/share/jupyterhub/templates/login.html +++ b/share/jupyterhub/templates/login.html @@ -56,13 +56,18 @@ tabindex="2" /> - + {% endif %} @@ -79,6 +84,11 @@ if (window.location.protocol === "http:") { var warning = document.getElementById('insecure-login-warning'); warning.className = warning.className.replace(/\bhidden\b/, ''); } +// setup onSubmit feedback +$('form').submit(() => { + $('.feedback-container>input').attr('disabled', true); + $('.feedback-container>*').toggleClass('hidden'); + $('.feedback-widget>*').toggleClass('fa-pulse'); +}); - {% endblock %} From debd2974943c03fadb1943ab6d8efb9634f3dff5 Mon Sep 17 00:00:00 2001 From: Michael Blackmon Date: Mon, 20 Apr 2020 11:33:38 -0400 Subject: [PATCH 043/156] restrict submit handler to only operate on targeted form --- share/jupyterhub/templates/login.html | 9 +++++---- share/jupyterhub/templates/spawn.html | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/share/jupyterhub/templates/login.html b/share/jupyterhub/templates/login.html index cd585ded..5c00d31d 100644 --- a/share/jupyterhub/templates/login.html +++ b/share/jupyterhub/templates/login.html @@ -85,10 +85,11 @@ if (window.location.protocol === "http:") { warning.className = warning.className.replace(/\bhidden\b/, ''); } // setup onSubmit feedback -$('form').submit(() => { - $('.feedback-container>input').attr('disabled', true); - $('.feedback-container>*').toggleClass('hidden'); - $('.feedback-widget>*').toggleClass('fa-pulse'); +$('form').submit((e) => { + var form = $(e.target); + form.find('.feedback-container>input').attr('disabled', true); + form.find('.feedback-container>*').toggleClass('hidden'); + form.find('.feedback-widget>*').toggleClass('fa-pulse'); }); {% endblock %} diff --git a/share/jupyterhub/templates/spawn.html b/share/jupyterhub/templates/spawn.html index d44bdd0d..773a55e7 100644 --- a/share/jupyterhub/templates/spawn.html +++ b/share/jupyterhub/templates/spawn.html @@ -39,10 +39,11 @@ {{ super() }} {% endblock %} From 137591f4582dfe470cb2c4e708d796caea16149a Mon Sep 17 00:00:00 2001 From: Will Starms Date: Fri, 1 May 2020 19:09:19 -0500 Subject: [PATCH 044/156] remove fixed position, causes Z ordering issues with the bottom of the users list --- share/jupyterhub/static/less/admin.less | 1 - 1 file changed, 1 deletion(-) diff --git a/share/jupyterhub/static/less/admin.less b/share/jupyterhub/static/less/admin.less index 6189e4e9..86dd6bcd 100644 --- a/share/jupyterhub/static/less/admin.less +++ b/share/jupyterhub/static/less/admin.less @@ -3,7 +3,6 @@ i.sort-icon { } .version_footer { - position: fixed; bottom: 0; width: 100%; } From da34c6cb34db6d677d3cebd0cadbc76f585fa5af Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 6 May 2020 10:44:53 +0200 Subject: [PATCH 045/156] remove hardcoded path from pagination links allows pagination of other pages --- jupyterhub/pagination.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/jupyterhub/pagination.py b/jupyterhub/pagination.py index c64a3ded..7f2450ca 100644 --- a/jupyterhub/pagination.py +++ b/jupyterhub/pagination.py @@ -126,10 +126,12 @@ class Pagination: @property def links(self): - """Sets the links for the pagination. - Getting the input from calculate_pages_window(), generates the HTML code + """Get the links for the pagination. + Getting the input from calculate_pages_window(), generates the HTML code for the pages to render, plus the arrows to go onwards and backwards (if needed). """ + if self.total_pages == 1: + return [] pages_to_render = self.calculate_pages_window() @@ -139,9 +141,7 @@ class Pagination: if self.page > 1: prev_page = self.page - 1 links.append( - '
  • «
    • '.format( - prev_page=prev_page - ) + '
  • «
    • '.format(prev_page=prev_page) ) else: links.append( @@ -161,17 +161,13 @@ class Pagination: ) else: links.append( - '
  • {page}
    • '.format( - page=page - ) + '
  • {page}
    • '.format(page=page) ) if self.page >= 1 and self.page < self.total_pages: next_page = self.page + 1 links.append( - '
  • »
    • '.format( - next_page=next_page - ) + '
  • »
    • '.format(next_page=next_page) ) else: links.append( From 0d245fe4e4979ac3aae9f8b801ea3537b9111ccf Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 6 May 2020 10:47:08 +0200 Subject: [PATCH 046/156] move pagination info next to pagination links at the bottom --- share/jupyterhub/static/less/admin.less | 4 ++++ share/jupyterhub/templates/admin.html | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/share/jupyterhub/static/less/admin.less b/share/jupyterhub/static/less/admin.less index 6189e4e9..88e36fd0 100644 --- a/share/jupyterhub/static/less/admin.less +++ b/share/jupyterhub/static/less/admin.less @@ -2,6 +2,10 @@ i.sort-icon { margin-left: 4px; } +tr.pagination-row > td.pagination-page-info { + vertical-align: middle; +} + .version_footer { position: fixed; bottom: 0; diff --git a/share/jupyterhub/templates/admin.html b/share/jupyterhub/templates/admin.html index b98108de..a13ba6c8 100644 --- a/share/jupyterhub/templates/admin.html +++ b/share/jupyterhub/templates/admin.html @@ -18,9 +18,6 @@ {% block main %}
    -
    - Displaying users {{ pagination.info.start|safe }} - {{ pagination.info.end|safe }}. Total users {{ pagination.info.total|safe }} -
    @@ -104,16 +101,19 @@ {% endfor %} {% endfor %} + + + + + +
    + {% if pagination.links %} + + {% endif %} + + Displaying users {{ pagination.info.start|safe }} - {{ pagination.info.end|safe }} of {{ pagination.info.total|safe }} +
    - {% if pagination.links %} - - - - - - - - {% endif %}
    From 4e29342711cce0a84331850fc203fb861b2f6452 Mon Sep 17 00:00:00 2001 From: Thijs Walcarius Date: Wed, 6 May 2020 11:04:30 +0200 Subject: [PATCH 047/156] Test error path when parsing query arguments --- jupyterhub/tests/mocking.py | 3 +++ jupyterhub/tests/test_pages.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/jupyterhub/tests/mocking.py b/jupyterhub/tests/mocking.py index 5ac2406d..64af46a8 100644 --- a/jupyterhub/tests/mocking.py +++ b/jupyterhub/tests/mocking.py @@ -173,6 +173,9 @@ class FormSpawner(MockSpawner): options['energy'] = form_data['energy'][0] if 'hello_file' in form_data: options['hello'] = form_data['hello_file'][0] + + if 'illegal_argument' in form_data: + raise ValueError("You are not allowed to specify 'illegal_argument'") return options diff --git a/jupyterhub/tests/test_pages.py b/jupyterhub/tests/test_pages.py index 7ca4cb22..a1825d43 100644 --- a/jupyterhub/tests/test_pages.py +++ b/jupyterhub/tests/test_pages.py @@ -277,6 +277,25 @@ async def test_spawn_with_query_arguments(app): } +async def test_spawn_with_query_arguments_error(app): + with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): + base_url = ujoin(public_host(app), app.hub.base_url) + cookies = await app.login_user('jones') + orm_u = orm.User.find(app.db, 'jones') + u = app.users[orm_u] + await u.stop() + next_url = ujoin(app.base_url, 'user/jones/tree') + r = await async_requests.get( + url_concat( + ujoin(base_url, 'spawn'), + {'next': next_url, 'energy': '510keV', 'illegal_argument': '42'}, + ), + cookies=cookies, + ) + r.raise_for_status() + assert "You are not allowed to specify " in r.text + + async def test_spawn_form(app): with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): base_url = ujoin(public_host(app), app.hub.base_url) From 8859bf8842b3daa91bfb9f8bf3faeaa239453024 Mon Sep 17 00:00:00 2001 From: romainx Date: Fri, 8 May 2020 06:06:42 +0200 Subject: [PATCH 048/156] #1018 PAM added in prerequisites --- README.md | 2 ++ docs/source/quickstart.md | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/README.md b/README.md index 98ab6fa4..9fb2fe76 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,8 @@ for administration of the Hub and its users. The `nodejs-legacy` package installs the `node` executable and is currently required for npm to work on Debian/Ubuntu. +- A [pluggable authentication module (PAM)](https://en.wikipedia.org/wiki/Pluggable_authentication_module) + to use the default Authenticator - TLS certificate and key for HTTPS communication - Domain name diff --git a/docs/source/quickstart.md b/docs/source/quickstart.md index f9c4beaa..b621aab2 100644 --- a/docs/source/quickstart.md +++ b/docs/source/quickstart.md @@ -26,6 +26,10 @@ Before installing JupyterHub, you will need: The `nodejs-legacy` package installs the `node` executable and is currently required for npm to work on Debian/Ubuntu. +- A [pluggable authentication module (PAM)](https://en.wikipedia.org/wiki/Pluggable_authentication_module) + to use the [default Authenticator](./getting-started/authenticators-users-basics.md). + PAM is often available by default on most distributions, if this is not the case it can be installed by + using the operating system's package manager. - TLS certificate and key for HTTPS communication - Domain name From d67b71b7aedd3f19f6522eb32e834fab22c3da14 Mon Sep 17 00:00:00 2001 From: ragar64 Date: Mon, 9 Mar 2020 13:04:20 +0530 Subject: [PATCH 049/156] " #2925 : Changing start my server button link to spawn url once server is stopped" --- share/jupyterhub/static/js/home.js | 1 + 1 file changed, 1 insertion(+) diff --git a/share/jupyterhub/static/js/home.js b/share/jupyterhub/static/js/home.js index 635f89b4..909c2b73 100644 --- a/share/jupyterhub/static/js/home.js +++ b/share/jupyterhub/static/js/home.js @@ -104,6 +104,7 @@ require(["jquery", "moment", "jhapi"], function( .text("Start My Server") .attr("title", "Start your default server") .attr("disabled", false) + .attr("href", base_url + "spawn/" + user) .off("click"); }, }); From 0e59ab003a8c6797de8efb1168a288afa639f947 Mon Sep 17 00:00:00 2001 From: romainx Date: Mon, 11 May 2020 14:54:32 +0200 Subject: [PATCH 050/156] Readme updated according to review --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 9fb2fe76..a6a65107 100644 --- a/README.md +++ b/README.md @@ -74,8 +74,7 @@ for administration of the Hub and its users. The `nodejs-legacy` package installs the `node` executable and is currently required for npm to work on Debian/Ubuntu. -- A [pluggable authentication module (PAM)](https://en.wikipedia.org/wiki/Pluggable_authentication_module) - to use the default Authenticator +- If using the default PAM Authenticator, a [pluggable authentication module (PAM)](https://en.wikipedia.org/wiki/Pluggable_authentication_module). - TLS certificate and key for HTTPS communication - Domain name From a36a56b4ff8530bab4e0b34d789f94faa02eb4be Mon Sep 17 00:00:00 2001 From: dp Date: Thu, 14 May 2020 16:16:07 -0700 Subject: [PATCH 051/156] docs: add proxy_http_version 1.1 add proxy_http_version 1.1 as it is required for KeepAlive connections --- docs/source/reference/config-proxy.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/reference/config-proxy.md b/docs/source/reference/config-proxy.md index 564f2efc..64ccb0ed 100644 --- a/docs/source/reference/config-proxy.md +++ b/docs/source/reference/config-proxy.md @@ -83,6 +83,7 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; # websocket headers + proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; } From 2991d2d1f1532cfe75a2a0b455659346f94ff2af Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Fri, 15 May 2020 11:34:03 +0200 Subject: [PATCH 052/156] Log successful /health requests on the debug level They are less relevant than other request and could very well end up cluttering the logs. It is not uncomming for these requests to be made every second or every other second. --- jupyterhub/log.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jupyterhub/log.py b/jupyterhub/log.py index d60b358a..f9fbffe8 100644 --- a/jupyterhub/log.py +++ b/jupyterhub/log.py @@ -12,6 +12,7 @@ from tornado.log import LogFormatter from tornado.web import HTTPError from tornado.web import StaticFileHandler +from .handlers.pages import HealthCheckHandler from .metrics import prometheus_log_method @@ -127,7 +128,9 @@ def log_request(handler): """ status = handler.get_status() request = handler.request - if status == 304 or (status < 300 and isinstance(handler, StaticFileHandler)): + if status == 304 or ( + status < 300 and isinstance(handler, (StaticFileHandler, HealthCheckHandler)) + ): # static-file success and 304 Found are debug-level log_method = access_log.debug elif status < 400: From 0dbad9bd9969b5b29e1079856a26635daf27558f Mon Sep 17 00:00:00 2001 From: Mohammad Wasil Date: Fri, 15 May 2020 13:51:50 +0200 Subject: [PATCH 053/156] Disable proxy_buffering to make the progress bar working when using nginx reverse proxy --- docs/source/reference/config-proxy.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/reference/config-proxy.md b/docs/source/reference/config-proxy.md index 564f2efc..9e493cf4 100644 --- a/docs/source/reference/config-proxy.md +++ b/docs/source/reference/config-proxy.md @@ -85,6 +85,8 @@ server { # websocket headers proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; + + proxy_buffering off; } # Managing requests to verify letsencrypt host From d60fa9a400c7b2a8a89ef692299e29bb57c0d51a Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Mon, 25 May 2020 16:10:08 +0200 Subject: [PATCH 054/156] Update links to the black GitHub repository --- .pre-commit-config.yaml | 2 +- CONTRIBUTING.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 02a918fa..e8f6663e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: rev: v1.9.0 hooks: - id: reorder-python-imports -- repo: https://github.com/ambv/black +- repo: https://github.com/psf/black rev: 19.10b0 hooks: - id: black diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 76ddf057..0f5ec05a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,7 +66,7 @@ pre-commit run which should run any autoformatting on your code and tell you about any errors it couldn't fix automatically. -You may also install [black integration](https://github.com/ambv/black#editor-integration) +You may also install [black integration](https://github.com/psf/black#editor-integration) into your text editor to format code automatically. If you have already committed files before setting up the pre-commit From 6bf4f3b2aae73c0f2f45ec39b5b01d83e3bbb0ba Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 26 May 2020 13:40:21 +0200 Subject: [PATCH 055/156] document upgrading from api_tokens to services config --- docs/source/reference/rest.md | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/source/reference/rest.md b/docs/source/reference/rest.md index 95e9ea8b..0432f677 100644 --- a/docs/source/reference/rest.md +++ b/docs/source/reference/rest.md @@ -57,6 +57,9 @@ generating an API token is available from the JupyterHub user interface: ## Add API tokens to the config file +**This is deprecated. We are in no rush to remove this feature, +but please consider if service tokens are right for you.** + You may also add a dictionary of API tokens and usernames to the hub's configuration file, `jupyterhub_config.py` (note that the **key** is the 'secret-token' while the **value** is the 'username'): @@ -67,6 +70,41 @@ c.JupyterHub.api_tokens = { } ``` +### Updating to admin services + +The `api_tokens` configuration has been softly deprecated since the introduction of services. +We have no plans to remove it, +but users are encouraged to use service configuration instead. + +If you have been using `api_tokens` to create an admin user +and a token for that user to perform some automations, +the services mechanism may be a better fit. +If you have the following configuration: + +```python +c.JupyterHub.admin_users = {"service-admin",} +c.JupyterHub.api_tokens = { + "secret-token": "service-admin", +} +``` + +This can be updated to create an admin service, with the following configuration: + +```python +c.JupyterHub.services = [ + { + "name": "service-token", + "admin": True, + "api_token": "secret-token", + }, +] +``` + +The token will have the same admin permissions, +but there will no longer be a user account created to house it. +The main noticeable difference is that there will be no notebook server associated with the account +and the service will not show up in the various user list pages and APIs. + ## Make an API request To authenticate your requests, pass the API token in the request's From cd367338588374b09c97c5b22c4590a659f3bd2e Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Tue, 26 May 2020 14:49:36 +0300 Subject: [PATCH 056/156] Remove the issue templates so that the ones from the central repo jupyterhub/.github take effect --- .github/ISSUE_TEMPLATE/bug_report.md | 33 ----------------------- .github/ISSUE_TEMPLATE/config.yml | 4 --- .github/ISSUE_TEMPLATE/feature_request.md | 25 ----------------- 3 files changed, 62 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 .github/ISSUE_TEMPLATE/config.yml delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 2fe07734..00000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -name: "\U0001F41B Bug report" -about: Create a report to help us repair something that is currently broken - ---- - - -### Bug description - - -#### Expected behaviour - - -#### Actual behaviour - - -### How to reproduce - - -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -### Your personal set up - - - - OS: - - - Version: - - - Configuration: - diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index 8574d777..00000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1,4 +0,0 @@ -contact_links: - - name: "\U0001F914 All other questions, including if you're not sure what to do." - url: https://discourse.jupyter.org - about: Search on Discourse for similar questions or ask for help there. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index b027cbcf..00000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -name: "\U0001F680 Feature request" -about: Suggest a new feature or a big change to JupyterHub - ---- - - -### Proposed change - - - -### Alternative options - - - -### Who would use this feature? - - - -### How much effort will adding it take? - - - -### Who can do this work? - From bcbc68dd82f06b69756b713dc6c998e3bfdec547 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Wed, 27 May 2020 19:55:47 +0300 Subject: [PATCH 057/156] Warn if both bind_url and ip/port/base_url are set --- jupyterhub/app.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 50809e19..ebc5938e 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -568,7 +568,16 @@ class JupyterHub(Application): urlinfo = urlinfo._replace(netloc=fmt % (self.ip, self.port)) urlinfo = urlinfo._replace(path=self.base_url) bind_url = urlunparse(urlinfo) + + # Warn if both bind_url and ip/port/base_url are set if bind_url != self.bind_url: + if self.bind_url != "http://:8000" and self.bind_url != "https://:8000": + self.log.warning( + "Both bind_url and ip/port/base_url have been configured. " + "JupyterHub.ip, JupyterHub.port, JupyterHub.base_url are" + " deprecated in JupyterHub 0.9," + " please use JupyterHub.bind_url instead." + ) self.bind_url = bind_url bind_url = Unicode( From e4e4d472b866309074b78701440b1200244639c8 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Thu, 28 May 2020 17:35:42 +0300 Subject: [PATCH 058/156] Add JupyterHub Demo docker image --- demo-image/Dockerfile | 16 ++++++++++++++++ demo-image/README.md | 25 +++++++++++++++++++++++++ demo-image/jupyterhub_config.py | 7 +++++++ 3 files changed, 48 insertions(+) create mode 100644 demo-image/Dockerfile create mode 100644 demo-image/README.md create mode 100644 demo-image/jupyterhub_config.py diff --git a/demo-image/Dockerfile b/demo-image/Dockerfile new file mode 100644 index 00000000..b5d69553 --- /dev/null +++ b/demo-image/Dockerfile @@ -0,0 +1,16 @@ +# Demo JupyterHub Docker image +# +# This should only be used for demo or testing and not as a base image to build on. +# +# It includes the notebook package and it uses the DummyAuthenticator and the SimpleLocalProcessSpawner. + +FROM jupyterhub/jupyterhub-onbuild + +# Install the notebook package +RUN python3 -m pip install notebook + +# Create a demo user +RUN useradd --create-home demo +RUN chown demo . + +USER demo diff --git a/demo-image/README.md b/demo-image/README.md new file mode 100644 index 00000000..7c9681b8 --- /dev/null +++ b/demo-image/README.md @@ -0,0 +1,25 @@ +## Demo Dockerfile + +This is a demo JupyterHub Docker image to help you get a quick overview of what +JupyterHub is and how it works. + +It uses the SimpleLocalProcessSpawner to spawn new user servers and +DummyAuthenticator for authentication. +The DummyAuthenticator allows you to log in with any username & password and the +SimpleLocalProcessSpawner allows starting servers without having to create a +local user for each JupyterHub user. + +### Important! + +This should only be used for demo or testing purposes! +It shouldn't be used as a base image to build on. + +### Try it +1. `cd` to the root of your jupyterhub repo. + +2. Build the demo image with `docker build -t jupyterhub-demo demo-image`. + +3. Run the demo image with `docker run -d -p 8000:8000 jupyterhub-demo`. + +4. Visit http://localhost:8000 and login with any username and password +5. Happy demo-ing :tada:! diff --git a/demo-image/jupyterhub_config.py b/demo-image/jupyterhub_config.py new file mode 100644 index 00000000..99a0ff13 --- /dev/null +++ b/demo-image/jupyterhub_config.py @@ -0,0 +1,7 @@ +# Configuration file for jupyterhub-demo + +c = get_config() + +# Use DummyAuthenticator and SympleSpawner +c.JupyterHub.spawner_class = "simple" +c.JupyterHub.authenticator_class = "dummy" From da3fedb5aa6a70b9e1b578594266ae969e182100 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Mon, 1 Jun 2020 12:41:22 -0700 Subject: [PATCH 059/156] pin jupyter_telemetry dependency --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 825fd409..07cc96e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ async_generator>=1.8 certipy>=0.1.2 entrypoints jinja2 -jupyter_telemetry +jupyter_telemetry==0.1.0 oauthlib>=3.0 pamela prometheus_client>=0.0.21 From 66c3760b02b57e2571feaa573f4a9768e6d1e6ab Mon Sep 17 00:00:00 2001 From: Georgiana Elena Date: Wed, 3 Jun 2020 14:30:51 +0300 Subject: [PATCH 060/156] Update jupyterhub/app.py Co-authored-by: Erik Sundell --- jupyterhub/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index ebc5938e..8b2119ed 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -571,7 +571,7 @@ class JupyterHub(Application): # Warn if both bind_url and ip/port/base_url are set if bind_url != self.bind_url: - if self.bind_url != "http://:8000" and self.bind_url != "https://:8000": + if self.bind_url != self._bind_url_default(): self.log.warning( "Both bind_url and ip/port/base_url have been configured. " "JupyterHub.ip, JupyterHub.port, JupyterHub.base_url are" From a8bc4f8a4a8d51201020e74290da52f3833cea12 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 5 Jun 2020 15:04:02 -0700 Subject: [PATCH 061/156] Test with some master dependencies. This does some of the test with the latest traitlets. We are looking into making a 5.0 release and would like to have some confidence that it does not break too many things. --- .travis.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8e2c56b0..f4de85ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,10 @@ before_install: install: - pip install --upgrade pip - pip install --upgrade --pre -r dev-requirements.txt . + - | + if [[ "$MASTER_DEPENDENCIES" == "True" ]]; then + pip install git+https://github.com/ipython/traitlets#egg=traitlets --force + fi - pip freeze # run tests @@ -87,6 +91,14 @@ jobs: - PGPASSWORD=hub[test/:? # The password in url below is url-encoded with: urllib.parse.quote($PGPASSWORD, safe='') - JUPYTERHUB_TEST_DB_URL=postgresql://jupyterhub:hub%5Btest%2F%3A%3F@127.0.0.1/jupyterhub + - name: python:3.8 + master dependencies + python: 3.8 + env: + - PGUSER=jupyterhub + - PGPASSWORD=hub[test/:? + # The password in url below is url-encoded with: urllib.parse.quote($PGPASSWORD, safe='') + - JUPYTERHUB_TEST_DB_URL=postgresql://jupyterhub:hub%5Btest%2F%3A%3F@127.0.0.1/jupyterhub + - MASTER_DEPENDENCIES=True - name: python:nightly python: nightly allow_failures: From 522ef3daeae97c663890cfbf26ddd3a9ce2df190 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Mon, 8 Jun 2020 23:49:31 +1200 Subject: [PATCH 062/156] Add Configuration Reference --- docs/source/conf.py | 54 ++++++++++++++++++++++ docs/source/reference/config-reference.rst | 30 ++++++++++++ docs/source/reference/index.rst | 1 + 3 files changed, 85 insertions(+) create mode 100644 docs/source/reference/config-reference.rst diff --git a/docs/source/conf.py b/docs/source/conf.py index 64e4939a..e0eb3bb9 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -60,11 +60,65 @@ default_role = 'literal' import recommonmark from recommonmark.transform import AutoStructify +# -- Config ------------------------------------------------------------- +from jupyterhub.app import JupyterHub +from docutils import nodes +from sphinx.directives.other import SphinxDirective +from contextlib import redirect_stdout +from io import StringIO + +# create a temp instance of JupyterHub just to get the output of the generate-config +# and help --all commands. +jupyterhub_app = JupyterHub() + + +class ConfigDirective(SphinxDirective): + """Generate the configuration file output for use in the documentation.""" + + has_content = False + required_arguments = 0 + optional_arguments = 0 + final_argument_whitespace = False + option_spec = {} + + def run(self): + # The generated configuration file for this version + generated_config = jupyterhub_app.generate_config_file() + # post-process output + home_dir = os.environ['HOME'] + generated_config = generated_config.replace(home_dir, '$HOME', 1) + par = nodes.literal_block(text=generated_config) + return [par] + + +class HelpAllDirective(SphinxDirective): + """Print the output of jupyterhub help --all for use in the documentation.""" + + has_content = False + required_arguments = 0 + optional_arguments = 0 + final_argument_whitespace = False + option_spec = {} + + def run(self): + # The output of the help command for this version + buffer = StringIO() + with redirect_stdout(buffer): + jupyterhub_app.print_help('--help-all') + all_help = buffer.getvalue() + # post-process output + home_dir = os.environ['HOME'] + all_help = all_help.replace(home_dir, '$HOME', 1) + par = nodes.literal_block(text=all_help) + return [par] + def setup(app): app.add_config_value('recommonmark_config', {'enable_eval_rst': True}, True) app.add_css_file('custom.css') app.add_transform(AutoStructify) + app.add_directive('jupyterhub-generate-config', ConfigDirective) + app.add_directive('jupyterhub-help-all', HelpAllDirective) source_suffix = ['.rst', '.md'] diff --git a/docs/source/reference/config-reference.rst b/docs/source/reference/config-reference.rst new file mode 100644 index 00000000..5b3c2acb --- /dev/null +++ b/docs/source/reference/config-reference.rst @@ -0,0 +1,30 @@ +============================== +Configuration Reference +============================== + +.. important:: + + Make sure the version of JupyterHub for this documentation matches your + installation version, as the output of this command may change between versions. + +JupyterHub configuration +------------------------ + +As explained in the `Configuration Basics <../getting-started/config-basics.html#generate-a-default-config-file>`_ +section, the ``jupyterhub-config.py`` can be automatically generated via + + .. code-block:: bash + + jupyterhub --generate-config + + +The following contains the output of that command for reference. + +.. jupyterhub-generate-config:: + +JupyterHub help command output +------------------------------ + +This section contains the output of the command ``jupyterhub --help-all``. + +.. jupyterhub-help-all:: diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst index b900025d..4008a723 100644 --- a/docs/source/reference/index.rst +++ b/docs/source/reference/index.rst @@ -24,3 +24,4 @@ what happens under-the-hood when you deploy and configure your JupyterHub. config-ghoauth config-proxy config-sudo + config-reference From b92ea54eda68bc922b702fe91d12a1ed32125fd2 Mon Sep 17 00:00:00 2001 From: niklas netter <40384506+gatoniel@users.noreply.github.com> Date: Wed, 10 Jun 2020 14:16:36 +0200 Subject: [PATCH 063/156] proxy settings might cause authentication errors --- docs/source/troubleshooting.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/troubleshooting.md b/docs/source/troubleshooting.md index dc222942..1addff21 100644 --- a/docs/source/troubleshooting.md +++ b/docs/source/troubleshooting.md @@ -196,6 +196,10 @@ After this, when you start your server via JupyterHub, it will build a new container. If this was the underlying cause of the issue, you should see your server again. +##### Proxy settings + +When your whole JupyterHub sits behind a organization proxy (*not* a reverse proxy like NGINX as part of your setup and *not* the configurable-http-proxy) the environment variables `HTTP_PROXY`, `HTTPS_PROXY`, `http_proxy` and `https_proxy` might be set. This confuses the jupyterhub-singleuser servers: When connecting to the Hub for authorization they connect via the proxy instead of directly connecting to the Hub on localhost. The proxy might deny the request. This results in the singleuser server thinking it has a wrong auth token. To circumvent this you should unset these environment variables. Using the environment variables `NO_PROXY` and `no_proxy` seems not to work. + ### Launching Jupyter Notebooks to run as an externally managed JupyterHub service with the `jupyterhub-singleuser` command returns a `JUPYTERHUB_API_TOKEN` error [JupyterHub services](https://jupyterhub.readthedocs.io/en/stable/reference/services.html) allow processes to interact with JupyterHub's REST API. Example use-cases include: From 4eb22821f24a6bd44ad85ef712346233322e4c64 Mon Sep 17 00:00:00 2001 From: niklas netter <40384506+gatoniel@users.noreply.github.com> Date: Wed, 10 Jun 2020 14:51:37 +0200 Subject: [PATCH 064/156] no_proxy does work --- docs/source/troubleshooting.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/troubleshooting.md b/docs/source/troubleshooting.md index 1addff21..d92c594b 100644 --- a/docs/source/troubleshooting.md +++ b/docs/source/troubleshooting.md @@ -152,7 +152,7 @@ You should see a similar 200 message, as above, in the Hub log when you first visit your single-user notebook server. If you don't see this message in the log, it may mean that your single-user notebook server isn't connecting to your Hub. -If you see 403 (forbidden) like this, it's a token problem: +If you see 403 (forbidden) like this, it's likely a token problem: ``` 403 GET /hub/api/authorizations/cookie/jupyterhub-token-name/[secret] (@10.0.1.4) 4.14ms @@ -196,9 +196,9 @@ After this, when you start your server via JupyterHub, it will build a new container. If this was the underlying cause of the issue, you should see your server again. -##### Proxy settings +##### Proxy settings (403 GET) -When your whole JupyterHub sits behind a organization proxy (*not* a reverse proxy like NGINX as part of your setup and *not* the configurable-http-proxy) the environment variables `HTTP_PROXY`, `HTTPS_PROXY`, `http_proxy` and `https_proxy` might be set. This confuses the jupyterhub-singleuser servers: When connecting to the Hub for authorization they connect via the proxy instead of directly connecting to the Hub on localhost. The proxy might deny the request. This results in the singleuser server thinking it has a wrong auth token. To circumvent this you should unset these environment variables. Using the environment variables `NO_PROXY` and `no_proxy` seems not to work. +When your whole JupyterHub sits behind a organization proxy (*not* a reverse proxy like NGINX as part of your setup and *not* the configurable-http-proxy) the environment variables `HTTP_PROXY`, `HTTPS_PROXY`, `http_proxy` and `https_proxy` might be set. This confuses the jupyterhub-singleuser servers: When connecting to the Hub for authorization they connect via the proxy instead of directly connecting to the Hub on localhost. The proxy might deny the request (403 GET). This results in the singleuser server thinking it has a wrong auth token. To circumvent this you should add `,,localhost,127.0.0.1` to the environment variables `NO_PROXY` and `no_proxy`. ### Launching Jupyter Notebooks to run as an externally managed JupyterHub service with the `jupyterhub-singleuser` command returns a `JUPYTERHUB_API_TOKEN` error From c357d02b566b76fc5801ae240ae7326bd8fd37cd Mon Sep 17 00:00:00 2001 From: Simon Li Date: Wed, 10 Jun 2020 14:53:58 +0100 Subject: [PATCH 065/156] Allow `python:3.8 + master dependencies` to fail Follow-up from https://github.com/jupyterhub/jupyterhub/pull/3076 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index f4de85ae..dd24b9c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -103,4 +103,5 @@ jobs: python: nightly allow_failures: - name: python:nightly + - name: python:3.8 + master dependencies fast_finish: true From df35268bfee2ae9eb6ab51ac29e9521b8ae8ed1f Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 11 Jun 2020 09:45:18 +0200 Subject: [PATCH 066/156] make Spawner.environment config highest priority so that it can override 'default' env variables like JUPYTERHUB_API_URL use with caution! --- jupyterhub/spawner.py | 27 +++++++++++++++++---------- jupyterhub/tests/test_spawner.py | 12 ++++++++++++ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/jupyterhub/spawner.py b/jupyterhub/spawner.py index 54875837..7c0bf9c4 100644 --- a/jupyterhub/spawner.py +++ b/jupyterhub/spawner.py @@ -467,6 +467,11 @@ class Spawner(LoggingConfigurable): Note that the spawner class' interface is not guaranteed to be exactly same across upgrades, so if you are using the callable take care to verify it continues to work after upgrades! + + .. versionchanged:: 1.2 + environment from this configuration has highest priority, + allowing override of 'default' env variables, + such as JUPYTERHUB_API_URL. """ ).tag(config=True) @@ -740,16 +745,6 @@ class Spawner(LoggingConfigurable): if key in os.environ: env[key] = os.environ[key] - # config overrides. If the value is a callable, it will be called with - # one parameter - the current spawner instance - and the return value - # will be assigned to the environment variable. This will be called at - # spawn time. - for key, value in self.environment.items(): - if callable(value): - env[key] = value(self) - else: - env[key] = value - env['JUPYTERHUB_API_TOKEN'] = self.api_token # deprecated (as of 0.7.2), for old versions of singleuser env['JPY_API_TOKEN'] = self.api_token @@ -797,6 +792,18 @@ class Spawner(LoggingConfigurable): env['JUPYTERHUB_SSL_CERTFILE'] = self.cert_paths['certfile'] env['JUPYTERHUB_SSL_CLIENT_CA'] = self.cert_paths['cafile'] + # env overrides from config. If the value is a callable, it will be called with + # one parameter - the current spawner instance - and the return value + # will be assigned to the environment variable. This will be called at + # spawn time. + # Called last to ensure highest priority, in case of overriding other + # 'default' variables like the API url + for key, value in self.environment.items(): + if callable(value): + env[key] = value(self) + else: + env[key] = value + return env async def get_url(self): diff --git a/jupyterhub/tests/test_spawner.py b/jupyterhub/tests/test_spawner.py index fa0cbeab..99b84393 100644 --- a/jupyterhub/tests/test_spawner.py +++ b/jupyterhub/tests/test_spawner.py @@ -404,3 +404,15 @@ async def test_spawner_routing(app, name): assert r.url == url assert r.text == urlparse(url).path await user.stop() + + +async def test_spawner_env(db): + env_overrides = { + "JUPYTERHUB_API_URL": "https://test.horse/hub/api", + "TEST_KEY": "value", + } + spawner = new_spawner(db, environment=env_overrides) + env = spawner.get_env() + for key, value in env_overrides.items(): + assert key in env + assert env[key] == value From aba55cc09319f9c402dcf96dd08efd6563ab5616 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 4 Jun 2020 10:20:50 +0200 Subject: [PATCH 067/156] implement UserDict.get behaves more like one would expect (same as try get-key, except: return default) without relying on cache presence or underlying key type (integer only) --- jupyterhub/handlers/base.py | 5 ++-- jupyterhub/tests/test_user.py | 22 +++++++++++++++ jupyterhub/user.py | 51 +++++++++++++++++++++++++++++++++-- 3 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 jupyterhub/tests/test_user.py diff --git a/jupyterhub/handlers/base.py b/jupyterhub/handlers/base.py index 9606fa61..07c7026e 100644 --- a/jupyterhub/handlers/base.py +++ b/jupyterhub/handlers/base.py @@ -676,9 +676,10 @@ class BaseHandler(RequestHandler): raise ValueError("Username doesn't match! %s != %s" % (username, user.name)) if user is None: - new_user = username not in self.users - user = self.user_from_username(username) + user = self.find_user(username) + new_user = user is None if new_user: + user = self.user_from_username(username) await maybe_future(self.authenticator.add_user(user)) # Only set `admin` if the authenticator returned an explicit value. if admin is not None and admin != user.admin: diff --git a/jupyterhub/tests/test_user.py b/jupyterhub/tests/test_user.py new file mode 100644 index 00000000..df7bc8a2 --- /dev/null +++ b/jupyterhub/tests/test_user.py @@ -0,0 +1,22 @@ +import pytest + +from ..user import UserDict +from .utils import add_user + + +@pytest.mark.parametrize("attr", ["self", "id", "name"]) +async def test_userdict_get(db, attr): + u = add_user(db, name="rey", app=False) + userdict = UserDict(db_factory=lambda: db, settings={}) + + if attr == "self": + key = u + else: + key = getattr(u, attr) + + # `in` checks cache only + assert key not in userdict + assert userdict.get(key) + assert userdict.get(key).id == u.id + # `in` should find it now + assert key in userdict diff --git a/jupyterhub/user.py b/jupyterhub/user.py index 4f049817..cfebe149 100644 --- a/jupyterhub/user.py +++ b/jupyterhub/user.py @@ -34,7 +34,23 @@ from .utils import url_path_join class UserDict(dict): """Like defaultdict, but for users - Getting by a user id OR an orm.User instance returns a User wrapper around the orm user. + Users can be retrieved by: + + - integer database id + - orm.User object + - username str + + A User wrapper object is always returned. + + This dict contains at least all active users, + but not necessarily all users in the database. + + Checking `key in userdict` returns whether + an item is already in the cache, + *not* whether it is in the database. + + .. versionchanged:: 1.2 + ``'username' in userdict`` pattern is now supported """ def __init__(self, db_factory, settings): @@ -57,11 +73,28 @@ class UserDict(dict): return self[orm_user.id] def __contains__(self, key): + """key in userdict checks presence in the cache + + it does not check if the user is in the database + """ if isinstance(key, (User, orm.User)): key = key.id + elif isinstance(key, str): + # username lookup, O(N) + for user in self.values(): + if user.name == key: + key = user.id + break return dict.__contains__(self, key) def __getitem__(self, key): + """UserDict allows retrieval of user by any of: + + - User object + - orm.User object + - username (str) + - orm.User.id int (actual key used in underlying dict) + """ if isinstance(key, User): key = key.id elif isinstance(key, str): @@ -69,7 +102,7 @@ class UserDict(dict): if orm_user is None: raise KeyError("No such user: %s" % key) else: - key = orm_user + key = orm_user.id if isinstance(key, orm.User): # users[orm_user] returns User(orm_user) orm_user = key @@ -92,6 +125,20 @@ class UserDict(dict): else: raise KeyError(repr(key)) + def get(self, key, default=None): + """Retrieve a User object if it can be found, else default + + Lookup can be by User object, id, or name + + .. versionchanged:: 1.2 + ``get()`` accesses the database instead of just the cache by integer id, + so is equivalent to catching KeyErrors on attempted lookup. + """ + try: + return self[key] + except KeyError: + return default + def __delitem__(self, key): user = self[key] for orm_spawner in user.orm_user._orm_spawners: From d4b5373c05c2614cb1af01128f18bc241d48e02c Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 4 Jun 2020 12:04:44 +0200 Subject: [PATCH 068/156] synchronize implementation of expiring values - base Expiring class - ensures expiring values (OAuthCode, OAuthAccessToken, APIToken) are not returned from `find` - all expire appropriately via purge_expired --- jupyterhub/app.py | 16 ++++-- jupyterhub/oauth/provider.py | 7 ++- jupyterhub/orm.py | 96 +++++++++++++++++++++++++++--------- jupyterhub/tests/test_orm.py | 77 ++++++++++++++++++++++++++++- 4 files changed, 166 insertions(+), 30 deletions(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 50809e19..174d5c71 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -1832,17 +1832,27 @@ class JupyterHub(Application): # purge expired tokens hourly purge_expired_tokens_interval = 3600 + def purge_expired_tokens(self): + """purge all expiring token objects from the database + + run periodically + """ + # this should be all the subclasses of Expiring + for cls in (orm.APIToken, orm.OAuthAccessToken, orm.OAuthCode): + self.log.debug("Purging expired {name}s".format(name=cls.__name__)) + cls.purge_expired(self.db) + async def init_api_tokens(self): """Load predefined API tokens (for services) into database""" await self._add_tokens(self.service_tokens, kind='service') await self._add_tokens(self.api_tokens, kind='user') - purge_expired_tokens = partial(orm.APIToken.purge_expired, self.db) - purge_expired_tokens() + + self.purge_expired_tokens() # purge expired tokens hourly # we don't need to be prompt about this # because expired tokens cannot be used anyway pc = PeriodicCallback( - purge_expired_tokens, 1e3 * self.purge_expired_tokens_interval + self.purge_expired_tokens, 1e3 * self.purge_expired_tokens_interval ) pc.start() diff --git a/jupyterhub/oauth/provider.py b/jupyterhub/oauth/provider.py index 6157223f..7c31d90d 100644 --- a/jupyterhub/oauth/provider.py +++ b/jupyterhub/oauth/provider.py @@ -2,7 +2,6 @@ implements https://oauthlib.readthedocs.io/en/latest/oauth2/server.html """ -from datetime import datetime from urllib.parse import urlparse from oauthlib import uri_validate @@ -250,7 +249,7 @@ class JupyterHubRequestValidator(RequestValidator): client=orm_client, code=code['code'], # oauth has 5 minutes to complete - expires_at=int(datetime.utcnow().timestamp() + 300), + expires_at=int(orm.OAuthCode.now() + 300), # TODO: persist oauth scopes # scopes=request.scopes, user=request.user.orm_user, @@ -347,7 +346,7 @@ class JupyterHubRequestValidator(RequestValidator): orm_access_token = orm.OAuthAccessToken( client=client, grant_type=orm.GrantType.authorization_code, - expires_at=datetime.utcnow().timestamp() + token['expires_in'], + expires_at=orm.OAuthAccessToken.now() + token['expires_in'], refresh_token=token['refresh_token'], # TODO: save scopes, # scopes=scopes, @@ -441,7 +440,7 @@ class JupyterHubRequestValidator(RequestValidator): Method is used by: - Authorization Code Grant """ - orm_code = self.db.query(orm.OAuthCode).filter_by(code=code).first() + orm_code = orm.OAuthCode.find(self.db, code=code) if orm_code is None: app_log.debug("No such code: %s", code) return False diff --git a/jupyterhub/orm.py b/jupyterhub/orm.py index 130d16f0..cc96ca88 100644 --- a/jupyterhub/orm.py +++ b/jupyterhub/orm.py @@ -311,7 +311,46 @@ class Service(Base): return db.query(cls).filter(cls.name == name).first() -class Hashed(object): +class Expiring: + """Mixin for expiring entries + + Subclass must define at least expires_at property, + which should be unix timestamp or datetime object + """ + + now = utcnow # funciton, must return float timestamp or datetime + expires_at = None # must be defined + + @property + def expires_in(self): + """Property returning expiration in seconds from now + + or None + """ + if self.expires_at: + delta = self.expires_at - self.now() + if isinstance(delta, timedelta): + delta = delta.total_seconds() + return delta + else: + return None + + @classmethod + def purge_expired(cls, db): + """Purge expired API Tokens from the database""" + now = cls.now() + deleted = False + for obj in ( + db.query(cls).filter(cls.expires_at != None).filter(cls.expires_at < now) + ): + app_log.debug("Purging expired %s", obj) + deleted = True + db.delete(obj) + if deleted: + db.commit() + + +class Hashed(Expiring): """Mixin for tables with hashed tokens""" prefix_length = 4 @@ -368,11 +407,21 @@ class Hashed(object): """Start the query for matching token. Returns an SQLAlchemy query already filtered by prefix-matches. + + .. versionchanged:: 1.2 + + Excludes expired matches. """ prefix = token[: cls.prefix_length] # since we can't filter on hashed values, filter on prefix # so we aren't comparing with all tokens - return db.query(cls).filter(bindparam('prefix', prefix).startswith(cls.prefix)) + prefix_match = db.query(cls).filter( + bindparam('prefix', prefix).startswith(cls.prefix) + ) + prefix_match = prefix_match.filter( + or_(cls.expires_at == None, cls.expires_at >= cls.now()) + ) + return prefix_match @classmethod def find(cls, db, token): @@ -408,6 +457,7 @@ class APIToken(Hashed, Base): return 'a%i' % self.id # token metadata for bookkeeping + now = datetime.utcnow # for expiry created = Column(DateTime, default=datetime.utcnow) expires_at = Column(DateTime, default=None, nullable=True) last_activity = Column(DateTime) @@ -428,20 +478,6 @@ class APIToken(Hashed, Base): cls=self.__class__.__name__, pre=self.prefix, kind=kind, name=name ) - @classmethod - def purge_expired(cls, db): - """Purge expired API Tokens from the database""" - now = utcnow() - deleted = False - for token in ( - db.query(cls).filter(cls.expires_at != None).filter(cls.expires_at < now) - ): - app_log.debug("Purging expired %s", token) - deleted = True - db.delete(token) - if deleted: - db.commit() - @classmethod def find(cls, db, token, *, kind=None): """Find a token object by value. @@ -452,9 +488,6 @@ class APIToken(Hashed, Base): `kind='service'` only returns API tokens for services """ prefix_match = cls.find_prefix(db, token) - prefix_match = prefix_match.filter( - or_(cls.expires_at == None, cls.expires_at >= utcnow()) - ) if kind == 'user': prefix_match = prefix_match.filter(cls.user_id != None) elif kind == 'service': @@ -497,7 +530,7 @@ class APIToken(Hashed, Base): assert service.id is not None orm_token.service = service if expires_in is not None: - orm_token.expires_at = utcnow() + timedelta(seconds=expires_in) + orm_token.expires_at = cls.now() + timedelta(seconds=expires_in) db.add(orm_token) db.commit() return token @@ -521,6 +554,10 @@ class OAuthAccessToken(Hashed, Base): __tablename__ = 'oauth_access_tokens' id = Column(Integer, primary_key=True, autoincrement=True) + @staticmethod + def now(): + return datetime.utcnow().timestamp() + @property def api_id(self): return 'o%i' % self.id @@ -547,11 +584,12 @@ class OAuthAccessToken(Hashed, Base): last_activity = Column(DateTime, nullable=True) def __repr__(self): - return "<{cls}('{prefix}...', client_id={client_id!r}, user={user!r}>".format( + return "<{cls}('{prefix}...', client_id={client_id!r}, user={user!r}, expires_in={expires_in}>".format( cls=self.__class__.__name__, client_id=self.client_id, user=self.user and self.user.name, prefix=self.prefix, + expires_in=self.expires_in, ) @classmethod @@ -568,8 +606,9 @@ class OAuthAccessToken(Hashed, Base): return orm_token -class OAuthCode(Base): +class OAuthCode(Expiring, Base): __tablename__ = 'oauth_codes' + id = Column(Integer, primary_key=True, autoincrement=True) client_id = Column( Unicode(255), ForeignKey('oauth_clients.identifier', ondelete='CASCADE') @@ -581,6 +620,19 @@ class OAuthCode(Base): # state = Column(Unicode(1023)) user_id = Column(Integer, ForeignKey('users.id', ondelete='CASCADE')) + @staticmethod + def now(): + return datetime.utcnow().timestamp() + + @classmethod + def find(cls, db, code): + return ( + db.query(cls) + .filter(cls.code == code) + .filter(or_(cls.expires_at == None, cls.expires_at >= cls.now())) + .first() + ) + class OAuthClient(Base): __tablename__ = 'oauth_clients' diff --git a/jupyterhub/tests/test_orm.py b/jupyterhub/tests/test_orm.py index 4790f386..0c125c5a 100644 --- a/jupyterhub/tests/test_orm.py +++ b/jupyterhub/tests/test_orm.py @@ -134,7 +134,7 @@ def test_token_expiry(db): assert orm_token.expires_at > now + timedelta(seconds=50) assert orm_token.expires_at < now + timedelta(seconds=70) the_future = mock.patch( - 'jupyterhub.orm.utcnow', lambda: now + timedelta(seconds=70) + 'jupyterhub.orm.APIToken.now', lambda: now + timedelta(seconds=70) ) with the_future: found = orm.APIToken.find(db, token=token) @@ -482,3 +482,78 @@ def test_group_delete_cascade(db): db.delete(user1) db.commit() assert user1 not in group1.users + + +def test_expiring_api_token(app, user): + db = app.db + token = orm.APIToken.new(expires_in=30, user=user) + orm_token = orm.APIToken.find(db, token, kind='user') + assert orm_token + + # purge_expired doesn't delete non-expired + orm.APIToken.purge_expired(db) + found = orm.APIToken.find(db, token) + assert found is orm_token + + with mock.patch.object( + orm.APIToken, 'now', lambda: datetime.utcnow() + timedelta(seconds=60) + ): + found = orm.APIToken.find(db, token) + assert found is None + assert orm_token in db.query(orm.APIToken) + orm.APIToken.purge_expired(db) + assert orm_token not in db.query(orm.APIToken) + + +def test_expiring_oauth_token(app, user): + db = app.db + token = "abc123" + now = orm.OAuthAccessToken.now + client = orm.OAuthClient(identifier="xxx", secret="yyy") + db.add(client) + orm_token = orm.OAuthAccessToken( + token=token, + grant_type=orm.GrantType.authorization_code, + client=client, + user=user, + expires_at=now() + 30, + ) + db.add(orm_token) + db.commit() + + found = orm.OAuthAccessToken.find(db, token) + assert found is orm_token + # purge_expired doesn't delete non-expired + orm.OAuthAccessToken.purge_expired(db) + found = orm.OAuthAccessToken.find(db, token) + assert found is orm_token + + with mock.patch.object(orm.OAuthAccessToken, 'now', lambda: now() + 60): + found = orm.OAuthAccessToken.find(db, token) + assert found is None + assert orm_token in db.query(orm.OAuthAccessToken) + orm.OAuthAccessToken.purge_expired(db) + assert orm_token not in db.query(orm.OAuthAccessToken) + + +def test_expiring_oauth_code(app, user): + db = app.db + code = "abc123" + now = orm.OAuthCode.now + orm_code = orm.OAuthCode(code=code, expires_at=now() + 30) + db.add(orm_code) + db.commit() + + found = orm.OAuthCode.find(db, code) + assert found is orm_code + # purge_expired doesn't delete non-expired + orm.OAuthCode.purge_expired(db) + found = orm.OAuthCode.find(db, code) + assert found is orm_code + + with mock.patch.object(orm.OAuthCode, 'now', lambda: now() + 60): + found = orm.OAuthCode.find(db, code) + assert found is None + assert orm_code in db.query(orm.OAuthCode) + orm.OAuthCode.purge_expired(db) + assert orm_code not in db.query(orm.OAuthCode) From 703703a6486d0d67dd709fbd1a5d04e5f5a70ad9 Mon Sep 17 00:00:00 2001 From: Jason Anderson Date: Thu, 11 Jun 2020 14:05:16 -0500 Subject: [PATCH 069/156] Ensure client dependencies build before wheel Bug #2852 describes an issue where templates cannot be found by JupyterHub when using the Docker images built out of this repo. The issue turned out to be due to missing node_modules at the time of build. There is a hook in the `package.json` that causes node_modules to be copied to the static/components directory post-install. If this is not run, those components are not in the static directory and thus are not included in the wheel when it is built. Fix #2905 fixed one problem--the `bower-lite` hook script wasn't copied to the Docker image, and so the hook couldn't run, but the other issue is that the client dependencies are never explicitly built. They must be built prior to the wheel build, and the hook script must have run so they are copied to the ./static folder, which is included in the wheel build thanks to [MANIFEST.in][1] .. note:: This removes the verbose flag from the wheel build command. The reason is that it generates a lot of writes to stdout. It seems that wheel can (or always) is switching to non-blocking mode, which can cause EAGAIN to be raised, which leads to fun errors like: BlockingIOError(.., 'write could not complete without blocking', ..) The wheels fail to build if this error is raised. Removing the verbosity flag is a quick solution (it drastically reduces writes to STDOUT), but comes at the cost of more trouble debugging a failed wheel build. Adding the "-v" back in the Dockerfile when debugging a build failure is still possible. [Credit: @vbraun][2] .. note:: This commit also removes some extraneous COPY operations during the Docker build, in particular the /src/jupyterhub/share directory is not used unless users have explicitly override their jupyterhub_config.py to include it somehow. If the default data_files_path behavior is used, JupyterHub should find the proper static directory when the application loads. Fixes: #2852 [1]: https://packaging.python.org/guides/using-manifest-in/ [2]: https://github.com/travis-ci/travis-ci/issues/4704#issuecomment-348435959 --- Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 36baddea..8b786bf0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,12 +45,15 @@ RUN apt-get update \ # compromise between needing to rebuild and maintaining # what needs to be part of the build COPY . /src/jupyterhub/ -COPY jupyterhub/ /src/jupyterhub/jupyterhub -COPY share/ /src/jupyterhub/share WORKDIR /src/jupyterhub + RUN python3 -m pip install --upgrade setuptools pip wheel -RUN python3 -m pip wheel -v --wheel-dir wheelhouse . + +# Build client component packages (they will be copied into ./share and +# packaged with the built wheel.) +RUN npm install +RUN python3 -m pip wheel --wheel-dir wheelhouse . FROM $BASE_IMAGE @@ -87,7 +90,6 @@ RUN npm install -g configurable-http-proxy@^4.2.0 \ # install the wheels we built in the first stage COPY --from=builder /src/jupyterhub/wheelhouse /tmp/wheelhouse -COPY --from=builder /src/jupyterhub/share /src/jupyterhub/share RUN python3 -m pip install --no-cache /tmp/wheelhouse/* RUN mkdir -p /srv/jupyterhub/ From 98dc1f71dbbf688529b076d91cea52211c3501e3 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 12 Jun 2020 10:03:34 +0200 Subject: [PATCH 070/156] build jupyterhub/jupyterhub-demo image on docker hub --- demo-image/Dockerfile | 4 ++-- demo-image/jupyterhub_config.py | 2 +- hooks/post_build | 3 +++ hooks/post_push | 34 ++++++++++++++++----------------- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/demo-image/Dockerfile b/demo-image/Dockerfile index b5d69553..c5add75e 100644 --- a/demo-image/Dockerfile +++ b/demo-image/Dockerfile @@ -3,8 +3,8 @@ # This should only be used for demo or testing and not as a base image to build on. # # It includes the notebook package and it uses the DummyAuthenticator and the SimpleLocalProcessSpawner. - -FROM jupyterhub/jupyterhub-onbuild +ARG BASE_IMAGE=jupyterhub/jupyterhub-onbuild +FROM ${BASE_IMAGE} # Install the notebook package RUN python3 -m pip install notebook diff --git a/demo-image/jupyterhub_config.py b/demo-image/jupyterhub_config.py index 99a0ff13..312df765 100644 --- a/demo-image/jupyterhub_config.py +++ b/demo-image/jupyterhub_config.py @@ -2,6 +2,6 @@ c = get_config() -# Use DummyAuthenticator and SympleSpawner +# Use DummyAuthenticator and SimpleSpawner c.JupyterHub.spawner_class = "simple" c.JupyterHub.authenticator_class = "dummy" diff --git a/hooks/post_build b/hooks/post_build index 874fd6d5..70d712e2 100755 --- a/hooks/post_build +++ b/hooks/post_build @@ -1,4 +1,7 @@ #!/bin/bash set -exuo pipefail +# build jupyterhub-onbuild image docker build --build-arg BASE_IMAGE=$DOCKER_REPO:$DOCKER_TAG -t ${DOCKER_REPO}-onbuild:$DOCKER_TAG onbuild +# build jupyterhub-demo image +docker build --build-arg BASE_IMAGE=${DOCKER_REPO}-onbuild:$DOCKER_TAG -t ${DOCKER_REPO}-demo:$DOCKER_TAG demo-image diff --git a/hooks/post_push b/hooks/post_push index 1b19f796..4db84638 100755 --- a/hooks/post_push +++ b/hooks/post_push @@ -2,8 +2,11 @@ set -exuo pipefail export ONBUILD=${DOCKER_REPO}-onbuild +export DEMO=${DOCKER_REPO}-demo +export REPOS="${DOCKER_REPO} ${ONBUILD} ${DEMO}" # push ONBUILD image docker push $ONBUILD:$DOCKER_TAG +docker push $DEMO:$DOCKER_TAG function get_hub_version() { rm -f hub_version @@ -20,25 +23,20 @@ function get_hub_version() { fi } - get_hub_version -# when building master, push 0.9.0.dev as well -docker tag $DOCKER_REPO:$DOCKER_TAG $DOCKER_REPO:$hub_xyz -docker push $DOCKER_REPO:$hub_xyz -docker tag $ONBUILD:$DOCKER_TAG $ONBUILD:$hub_xyz -docker push $ONBUILD:$hub_xyz +for repo in ${REPOS}; do + # when building master, push 0.9.0.dev as well + docker tag $repo:$DOCKER_TAG $repo:$hub_xyz + docker push $repo:$hub_xyz -# when building 0.9.x, push 0.9 as well -docker tag $DOCKER_REPO:$DOCKER_TAG $DOCKER_REPO:$hub_xy -docker push $DOCKER_REPO:$hub_xy -docker tag $ONBUILD:$DOCKER_TAG $ONBUILD:$hub_xy -docker push $ONBUILD:$hub_xyz + # when building 0.9.x, push 0.9 as well + docker tag $repo:$DOCKER_TAG $repo:$hub_xy + docker push $repo:$hub_xy -# if building a stable release, tag latest as well -if [[ "$latest" == "1" ]]; then - docker tag $DOCKER_REPO:$DOCKER_TAG $DOCKER_REPO:latest - docker push $DOCKER_REPO:latest - docker tag $ONBUILD:$DOCKER_TAG $ONBUILD:latest - docker push $ONBUILD:latest -fi + # if building a stable release, tag latest as well + if [[ "$latest" == "1" ]]; then + docker tag $repo:$DOCKER_TAG $repo:latest + docker push $repo:latest + fi +done From 10f09f4f7041d7414d5fb9ba57f1c233888214a2 Mon Sep 17 00:00:00 2001 From: Leo Gallucci Date: Fri, 12 Jun 2020 18:00:00 +0200 Subject: [PATCH 071/156] Remove already done named servers from roadmap Remove already done "UI for managing named servers" from the roadmap --- docs/source/contributing/roadmap.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/contributing/roadmap.md b/docs/source/contributing/roadmap.md index 8ad33fbf..c1a6194e 100644 --- a/docs/source/contributing/roadmap.md +++ b/docs/source/contributing/roadmap.md @@ -83,7 +83,6 @@ these will be moved at a future review of the roadmap. - (prometheus?) API for resource monitoring - tracking activity on single-user servers instead of the proxy - notes and activity tracking per API token - - UI for managing named servers ### Later From 2d1f91e527721af9ce095acd65cd82df9b41f544 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Sat, 13 Jun 2020 11:47:34 +0100 Subject: [PATCH 072/156] Replace zonca/remotespawner with NERSC/sshspawner https://github.com/zonca/remotespawner is archived, the readme recommends https://github.com/jupyterhub/batchspawner --- docs/source/reference/spawners.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/reference/spawners.md b/docs/source/reference/spawners.md index d2a21472..78a78d3f 100644 --- a/docs/source/reference/spawners.md +++ b/docs/source/reference/spawners.md @@ -27,8 +27,8 @@ Some examples include: servers using batch systems - [YarnSpawner](https://github.com/jupyterhub/yarnspawner) for spawning notebook servers in YARN containers on a Hadoop cluster -- [RemoteSpawner](https://github.com/zonca/remotespawner) to spawn notebooks - and a remote server and tunnel the port via SSH +- [SSHSpawner](https://github.com/NERSC/sshspawner) to spawn notebooks + on a remote server using SSH ## Spawner control methods From 5bbf584cb7d691d4f41e5a725de611b98bef4ea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Collonval?= Date: Sat, 13 Jun 2020 15:58:46 +0200 Subject: [PATCH 073/156] Make `delete_invalid_users` configurable --- jupyterhub/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jupyterhub/auth.py b/jupyterhub/auth.py index 9a43d280..ef18fead 100644 --- a/jupyterhub/auth.py +++ b/jupyterhub/auth.py @@ -206,6 +206,7 @@ class Authenticator(LoggingConfigurable): delete_invalid_users = Bool( False, + config=True, help="""Delete any users from the database that do not pass validation When JupyterHub starts, `.add_user` will be called From cc8e7806530466dce8968567d1bbd2b39a7afa26 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 15 Jun 2020 14:40:20 +0200 Subject: [PATCH 074/156] rename white/blacklist allowed/blocked - group_whitelist -> allowed_groups still todo: handle deprecated signatures in check_whitelist methods while preserving subclass overrides --- jupyterhub/apihandlers/auth.py | 9 +- jupyterhub/app.py | 48 +++--- jupyterhub/auth.py | 212 +++++++++++++++++-------- jupyterhub/services/auth.py | 8 +- jupyterhub/spawner.py | 6 +- jupyterhub/tests/test_app.py | 2 +- jupyterhub/tests/test_auth.py | 61 ++++--- jupyterhub/tests/test_pages.py | 2 +- jupyterhub/tests/test_services_auth.py | 10 +- 9 files changed, 219 insertions(+), 139 deletions(-) diff --git a/jupyterhub/apihandlers/auth.py b/jupyterhub/apihandlers/auth.py index 15a4d783..b7c4df0e 100644 --- a/jupyterhub/apihandlers/auth.py +++ b/jupyterhub/apihandlers/auth.py @@ -201,7 +201,7 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler): def needs_oauth_confirm(self, user, oauth_client): """Return whether the given oauth client needs to prompt for access for the given user - Checks whitelist for oauth clients + Checks list for oauth clients that don't need confirmation (i.e. the user's own server) @@ -214,9 +214,8 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler): if ( # it's the user's own server oauth_client.identifier in own_oauth_client_ids - # or it's in the global whitelist - or oauth_client.identifier - in self.settings.get('oauth_no_confirm_whitelist', set()) + # or it's in the global no-confirm list + or oauth_client.identifier in self.settings.get('oauth_no_confirm', set()) ): return False # default: require confirmation @@ -229,7 +228,7 @@ class OAuthAuthorizeHandler(OAuthHandler, BaseHandler): Render oauth confirmation page: "Server at ... would like permission to ...". - Users accessing their own server or a service whitelist + Users accessing their own server or a blessed service will skip confirmation. """ diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 8b2119ed..ba549aa4 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -1689,22 +1689,22 @@ class JupyterHub(Application): # the admin_users config variable will never be used after this point. # only the database values will be referenced. - whitelist = [ + allowed = [ self.authenticator.normalize_username(name) - for name in self.authenticator.whitelist + for name in self.authenticator.allowed ] - self.authenticator.whitelist = set(whitelist) # force normalization - for username in whitelist: + self.authenticator.allowed = set(allowed) # force normalization + for username in allowed: if not self.authenticator.validate_username(username): raise ValueError("username %r is not valid" % username) - if not whitelist: + if not allowed: self.log.info( - "Not using whitelist. Any authenticated user will be allowed." + "Not using allowed user list. Any authenticated user will be allowed." ) - # add whitelisted users to the db - for name in whitelist: + # add allowed users to the db + for name in allowed: user = orm.User.find(db, name) if user is None: user = orm.User(name=name) @@ -1714,9 +1714,9 @@ class JupyterHub(Application): db.commit() # Notify authenticator of all users. - # This ensures Auth whitelist is up-to-date with the database. - # This lets whitelist be used to set up initial list, - # but changes to the whitelist can occur in the database, + # This ensures Authenticator.allowed is up-to-date with the database. + # This lets .allowed be used to set up initial list, + # but changes to the allowed list can occur in the database, # and persist across sessions. total_users = 0 for user in db.query(orm.User): @@ -1753,9 +1753,9 @@ class JupyterHub(Application): user.created = user.last_activity or datetime.utcnow() db.commit() - # The whitelist set and the users in the db are now the same. + # The allowed set and the users in the db are now the same. # From this point on, any user changes should be done simultaneously - # to the whitelist set and user db, unless the whitelist is empty (all users allowed). + # to the allowed set and user db, unless the allowed set is empty (all users allowed). TOTAL_USERS.set(total_users) @@ -1770,11 +1770,11 @@ class JupyterHub(Application): for username in usernames: username = self.authenticator.normalize_username(username) if not ( - await maybe_future( - self.authenticator.check_whitelist(username, None) - ) + await maybe_future(self.authenticator.check_allowed(username, None)) ): - raise ValueError("Username %r is not in whitelist" % username) + raise ValueError( + "Username %r is not in Authenticator.allowed" % username + ) user = orm.User.find(db, name=username) if user is None: if not self.authenticator.validate_username(username): @@ -1798,11 +1798,13 @@ class JupyterHub(Application): if kind == 'user': name = self.authenticator.normalize_username(name) if not ( - await maybe_future(self.authenticator.check_whitelist(name, None)) + await maybe_future(self.authenticator.check_allowed(name, None)) ): - raise ValueError("Token name %r is not in whitelist" % name) + raise ValueError( + "Token user name %r is not in Authenticator.allowed" % name + ) if not self.authenticator.validate_username(name): - raise ValueError("Token name %r is not valid" % name) + raise ValueError("Token user name %r is not valid" % name) if kind == 'service': if not any(service["name"] == name for service in self.services): self.log.warning( @@ -2183,14 +2185,14 @@ class JupyterHub(Application): else: version_hash = datetime.now().strftime("%Y%m%d%H%M%S") - oauth_no_confirm_whitelist = set() + oauth_no_confirm_list = set() for service in self._service_map.values(): if service.oauth_no_confirm: self.log.warning( "Allowing service %s to complete OAuth without confirmation on an authorization web page", service.name, ) - oauth_no_confirm_whitelist.add(service.oauth_client_id) + oauth_no_confirm_list.add(service.oauth_client_id) settings = dict( log_function=log_request, @@ -2226,7 +2228,7 @@ class JupyterHub(Application): default_server_name=self._default_server_name, named_server_limit_per_user=self.named_server_limit_per_user, oauth_provider=self.oauth_provider, - oauth_no_confirm_whitelist=oauth_no_confirm_whitelist, + oauth_no_confirm_list=oauth_no_confirm_list, concurrent_spawn_limit=self.concurrent_spawn_limit, spawn_throttle_retry_range=self.spawn_throttle_retry_range, active_server_limit=self.active_server_limit, diff --git a/jupyterhub/auth.py b/jupyterhub/auth.py index 9a43d280..476e7441 100644 --- a/jupyterhub/auth.py +++ b/jupyterhub/auth.py @@ -7,6 +7,7 @@ import re import sys import warnings from concurrent.futures import ThreadPoolExecutor +from functools import partial from shutil import which from subprocess import PIPE from subprocess import Popen @@ -100,41 +101,74 @@ class Authenticator(LoggingConfigurable): """ ).tag(config=True) - whitelist = Set( + whitelist = Set(help="Deprecated, use `Authenticator.allowed`", config=True,) + + allowed = Set( help=""" - Whitelist of usernames that are allowed to log in. + Set of usernames that are allowed to log in. Use this with supported authenticators to restrict which users can log in. This is an - additional whitelist that further restricts users, beyond whatever restrictions the + additional list that further restricts users, beyond whatever restrictions the authenticator has in place. If empty, does not perform any additional restriction. + + .. versionchanged:: 1.2 + `Authenticator.whitelist` renamed to `allowed` """ ).tag(config=True) - blacklist = Set( + blocked = Set( help=""" - Blacklist of usernames that are not allowed to log in. + Set of usernames that are not allowed to log in. Use this with supported authenticators to restrict which users can not log in. This is an - additional blacklist that further restricts users, beyond whatever restrictions the + additional block list that further restricts users, beyond whatever restrictions the authenticator has in place. If empty, does not perform any additional restriction. .. versionadded: 0.9 + + .. versionchanged:: 1.2 + `Authenticator.blacklist` renamed to `blocked` """ ).tag(config=True) - @observe('whitelist') - def _check_whitelist(self, change): + _deprecated_aliases = { + "whitelist": ("allowed", "1.2"), + "blacklist": ("blocked", "1.2"), + } + + @observe(*list(_deprecated_aliases)) + def _deprecated_trait(self, change): + """observer for deprecated traits""" + old_attr = change.name + new_attr, version = self._deprecated_aliases.get(old_attr) + new_value = getattr(self, new_attr) + if new_value != change.new: + # only warn if different + # protects backward-compatible config from warnings + # if they set the same value under both names + self.log.warning( + "{cls}.{old} is deprecated in JupyterHub {version}, use {cls}.{new} instead".format( + cls=self.__class__.__name__, + old=old_attr, + new=new_attr, + version=version, + ) + ) + setattr(self, new_attr, change.new) + + @observe('allowed') + def _check_allowed(self, change): short_names = [name for name in change['new'] if len(name) <= 1] if short_names: sorted_names = sorted(short_names) single = ''.join(sorted_names) string_set_typo = "set('%s')" % single self.log.warning( - "whitelist contains single-character names: %s; did you mean set([%r]) instead of %s?", + "Allowed list contains single-character names: %s; did you mean set([%r]) instead of %s?", sorted_names[:8], single, string_set_typo, @@ -260,6 +294,8 @@ class Authenticator(LoggingConfigurable): def __init__(self, **kwargs): super().__init__(**kwargs) + # TODO: properly handle deprecated signature *and* name + # with correct subclass override priority! for method_name in ( 'check_whitelist', 'check_blacklist', @@ -326,39 +362,45 @@ class Authenticator(LoggingConfigurable): username = self.username_map.get(username, username) return username - def check_whitelist(self, username, authentication=None): - """Check if a username is allowed to authenticate based on whitelist configuration + def check_allowed(self, username, authentication=None): + """Check if a username is allowed to authenticate based on configuration Return True if username is allowed, False otherwise. - No whitelist means any username is allowed. + No allowed set means any username is allowed. - Names are normalized *before* being checked against the whitelist. + Names are normalized *before* being checked against the allowed set. .. versionchanged:: 1.0 Signature updated to accept authentication data and any future changes - """ - if not self.whitelist: - # No whitelist means any name is allowed - return True - return username in self.whitelist - def check_blacklist(self, username, authentication=None): - """Check if a username is blocked to authenticate based on blacklist configuration + .. versionchanged:: 1.2 + Renamed check_whitelist to check_allowed + """ + if not self.allowed: + # No allowed set means any name is allowed + return True + return username in self.allowed + + def check_blocked(self, username, authentication=None): + """Check if a username is blocked to authenticate based on Authenticator.blocked configuration Return True if username is allowed, False otherwise. - No blacklist means any username is allowed. + No block list means any username is allowed. - Names are normalized *before* being checked against the blacklist. + Names are normalized *before* being checked against the block list. .. versionadded: 0.9 .. versionchanged:: 1.0 Signature updated to accept authentication data as second argument + + .. versionchanged:: 1.2 + Renamed check_blacklist to check_blocked """ - if not self.blacklist: - # No blacklist means any name is allowed + if not self.blocked: + # No block list means any name is allowed return True - return username not in self.blacklist + return username not in self.blocked async def get_authenticated_user(self, handler, data): """Authenticate the user who is attempting to log in @@ -367,7 +409,7 @@ class Authenticator(LoggingConfigurable): This calls `authenticate`, which should be overridden in subclasses, normalizes the username if any normalization should be done, - and then validates the name in the whitelist. + and then validates the name in the allowed set. This is the outer API for authenticating a user. Subclasses should not override this method. @@ -375,7 +417,7 @@ class Authenticator(LoggingConfigurable): The various stages can be overridden separately: - `authenticate` turns formdata into a username - `normalize_username` normalizes the username - - `check_whitelist` checks against the user whitelist + - `check_allowed` checks against the user allowed .. versionchanged:: 0.8 return dict instead of username @@ -389,7 +431,7 @@ class Authenticator(LoggingConfigurable): else: authenticated = {'name': authenticated} authenticated.setdefault('auth_state', None) - # Leave the default as None, but reevaluate later post-whitelist + # Leave the default as None, but reevaluate later post-allowed-check authenticated.setdefault('admin', None) # normalize the username @@ -400,20 +442,16 @@ class Authenticator(LoggingConfigurable): self.log.warning("Disallowing invalid username %r.", username) return - blacklist_pass = await maybe_future( - self.check_blacklist(username, authenticated) - ) - whitelist_pass = await maybe_future( - self.check_whitelist(username, authenticated) - ) + blocked_pass = await maybe_future(self.check_blocked(username, authenticated)) + allowed_pass = await maybe_future(self.check_allowed(username, authenticated)) - if blacklist_pass: + if blocked_pass: pass else: - self.log.warning("User %r in blacklist. Stop authentication", username) + self.log.warning("User %r blocked. Stop authentication", username) return - if whitelist_pass: + if allowed_pass: if authenticated['admin'] is None: authenticated['admin'] = await maybe_future( self.is_admin(handler, authenticated) @@ -423,7 +461,7 @@ class Authenticator(LoggingConfigurable): return authenticated else: - self.log.warning("User %r not in whitelist.", username) + self.log.warning("User %r not allowed.", username) return async def refresh_user(self, user, handler=None): @@ -479,7 +517,7 @@ class Authenticator(LoggingConfigurable): It must return the username on successful authentication, and return None on failed authentication. - Checking the whitelist is handled separately by the caller. + Checking allowed/blocked is handled separately by the caller. .. versionchanged:: 0.8 Allow `authenticate` to return a dict containing auth_state. @@ -520,10 +558,10 @@ class Authenticator(LoggingConfigurable): This method may be a coroutine. - By default, this just adds the user to the whitelist. + By default, this just adds the user to the allowed set. Subclasses may do more extensive things, such as adding actual unix users, - but they should call super to ensure the whitelist is updated. + but they should call super to ensure the allowed set is updated. Note that this should be idempotent, since it is called whenever the hub restarts for all users. @@ -533,19 +571,19 @@ class Authenticator(LoggingConfigurable): """ if not self.validate_username(user.name): raise ValueError("Invalid username: %s" % user.name) - if self.whitelist: - self.whitelist.add(user.name) + if self.allowed: + self.allowed.add(user.name) def delete_user(self, user): """Hook called when a user is deleted - Removes the user from the whitelist. - Subclasses should call super to ensure the whitelist is updated. + Removes the user from the allowed set. + Subclasses should call super to ensure the allowed set is updated. Args: user (User): The User wrapper object """ - self.whitelist.discard(user.name) + self.allowed.discard(user.name) auto_login = Bool( False, @@ -610,6 +648,38 @@ class Authenticator(LoggingConfigurable): return [('/login', LoginHandler)] +def _deprecated_method(old_name, new_name, version, self, *args, **kwargs): + """Method wrapper for a deprecated method name""" + + warnings.warn( + ( + "{cls}.{old_name} is deprecated in JupyterHub {version}." + " Please use {cls}.{new_name} instead." + ).format( + cls=self.__class__.__name__, + old_name=old_name, + new_name=new_name, + version=version, + ), + DeprecationWarning, + stacklevel=2, + ) + old_method = getattr(self, new_name) + return old_method(*args, **kwargs) + + +# deprecate white/blacklist method names +for _old_name, _new_name, _version in [ + ("check_whitelist", "check_allowed", "1.2"), + ("check_blacklist", "check_blocked", "1.2"), +]: + setattr( + Authenticator, + _old_name, + partial(_deprecated_method, _old_name, _new_name, _version), + ) + + class LocalAuthenticator(Authenticator): """Base class for Authenticators that work with local Linux/UNIX users @@ -669,37 +739,37 @@ class LocalAuthenticator(Authenticator): """ ).tag(config=True) - group_whitelist = Set( - help=""" - Whitelist all users from this UNIX group. + group_whitelist = Set(help="""DEPRECATED: use allowed_groups""",).tag(config=True) - This makes the username whitelist ineffective. + allowed_groups = Set( + help=""" + Allow login from all users in these UNIX groups. + + If set, allowed username set is ignored. """ ).tag(config=True) - @observe('group_whitelist') - def _group_whitelist_changed(self, change): - """ - Log a warning if both group_whitelist and user whitelist are set. - """ - if self.whitelist: + @observe('allowed_groups') + def _allowed_groups_changed(self, change): + """Log a warning if mutually exclusive user and group allowed sets are specified.""" + if self.allowed: self.log.warning( - "Ignoring username whitelist because group whitelist supplied!" + "Ignoring Authenticator.allowed set because Authenticator.allowed_groups supplied!" ) - def check_whitelist(self, username, authentication=None): - if self.group_whitelist: - return self.check_group_whitelist(username, authentication) + def check_allowed(self, username, authentication=None): + if self.allowed_groups: + return self.check_allowed_groups(username, authentication) else: - return super().check_whitelist(username, authentication) + return super().check_allowed(username, authentication) - def check_group_whitelist(self, username, authentication=None): + def check_allowed_groups(self, username, authentication=None): """ - If group_whitelist is configured, check if authenticating user is part of group. + If allowed_groups is configured, check if authenticating user is part of group. """ - if not self.group_whitelist: + if not self.allowed_groups: return False - for grnam in self.group_whitelist: + for grnam in self.allowed_groups: try: group = self._getgrnam(grnam) except KeyError: @@ -843,7 +913,7 @@ class PAMAuthenticator(LocalAuthenticator): Authoritative list of user groups that determine admin access. Users not in these groups can still be granted admin status through admin_users. - White/blacklisting rules still apply. + allowed/blocked rules still apply. """ ).tag(config=True) @@ -986,6 +1056,16 @@ class PAMAuthenticator(LocalAuthenticator): return super().normalize_username(username) +for _old_name, _new_name, _version in [ + ("check_group_whitelist", "check_group_allowed", "1.2"), +]: + setattr( + LocalAuthenticator, + _old_name, + partial(_deprecated_method, _old_name, _new_name, _version), + ) + + class DummyAuthenticator(Authenticator): """Dummy Authenticator for testing diff --git a/jupyterhub/services/auth.py b/jupyterhub/services/auth.py index 518fd7e8..5ad6d163 100644 --- a/jupyterhub/services/auth.py +++ b/jupyterhub/services/auth.py @@ -860,15 +860,15 @@ class HubAuthenticated(object): if kind == 'service': # it's a service, check hub_services if self.hub_services and name in self.hub_services: - app_log.debug("Allowing whitelisted Hub service %s", name) + app_log.debug("Allowing Hub service %s", name) return model else: app_log.warning("Not allowing Hub service %s", name) raise UserNotAllowed(model) if self.hub_users and name in self.hub_users: - # user in whitelist - app_log.debug("Allowing whitelisted Hub user %s", name) + # user in allowed list + app_log.debug("Allowing Hub user %s", name) return model elif self.hub_groups and set(model['groups']).intersection(self.hub_groups): allowed_groups = set(model['groups']).intersection(self.hub_groups) @@ -877,7 +877,7 @@ class HubAuthenticated(object): name, ','.join(sorted(allowed_groups)), ) - # group in whitelist + # group in allowed list return model else: app_log.warning("Not allowing Hub user %s", name) diff --git a/jupyterhub/spawner.py b/jupyterhub/spawner.py index 7c0bf9c4..cdca3c74 100644 --- a/jupyterhub/spawner.py +++ b/jupyterhub/spawner.py @@ -435,9 +435,9 @@ class Spawner(LoggingConfigurable): 'LC_ALL', ], help=""" - Whitelist of environment variables for the single-user server to inherit from the JupyterHub process. + List of environment variables for the single-user server to inherit from the JupyterHub process. - This whitelist is used to ensure that sensitive information in the JupyterHub process's environment + This list is used to ensure that sensitive information in the JupyterHub process's environment (such as `CONFIGPROXY_AUTH_TOKEN`) is not passed to the single-user server's process. """, ).tag(config=True) @@ -456,7 +456,7 @@ class Spawner(LoggingConfigurable): Environment variables that end up in the single-user server's process come from 3 sources: - This `environment` configurable - - The JupyterHub process' environment variables that are whitelisted in `env_keep` + - The JupyterHub process' environment variables that are listed in `env_keep` - Variables to establish contact between the single-user notebook and the hub (such as JUPYTERHUB_API_TOKEN) The `environment` configurable should be set by JupyterHub administrators to add diff --git a/jupyterhub/tests/test_app.py b/jupyterhub/tests/test_app.py index bc59176b..55000bc7 100644 --- a/jupyterhub/tests/test_app.py +++ b/jupyterhub/tests/test_app.py @@ -93,7 +93,7 @@ def test_generate_config(): os.remove(cfg_file) assert cfg_file in out assert 'Spawner.cmd' in cfg_text - assert 'Authenticator.whitelist' in cfg_text + assert 'Authenticator.allowed' in cfg_text async def test_init_tokens(request): diff --git a/jupyterhub/tests/test_auth.py b/jupyterhub/tests/test_auth.py index 10ae0b1a..3c9b79ec 100644 --- a/jupyterhub/tests/test_auth.py +++ b/jupyterhub/tests/test_auth.py @@ -6,6 +6,7 @@ from unittest import mock import pytest from requests import HTTPError +from traitlets.config import Config from .mocking import MockPAMAuthenticator from .mocking import MockStructGroup @@ -137,8 +138,8 @@ async def test_pam_auth_admin_groups(): assert authorized['admin'] is False -async def test_pam_auth_whitelist(): - authenticator = MockPAMAuthenticator(whitelist={'wash', 'kaylee'}) +async def test_pam_auth_allowed(): + authenticator = MockPAMAuthenticator(allowed={'wash', 'kaylee'}) authorized = await authenticator.get_authenticated_user( None, {'username': 'kaylee', 'password': 'kaylee'} ) @@ -155,11 +156,11 @@ async def test_pam_auth_whitelist(): assert authorized is None -async def test_pam_auth_group_whitelist(): +async def test_pam_auth_allowed_groups(): def getgrnam(name): return MockStructGroup('grp', ['kaylee']) - authenticator = MockPAMAuthenticator(group_whitelist={'group'}) + authenticator = MockPAMAuthenticator(allowed_groups={'group'}) with mock.patch.object(authenticator, '_getgrnam', getgrnam): authorized = await authenticator.get_authenticated_user( @@ -174,7 +175,7 @@ async def test_pam_auth_group_whitelist(): assert authorized is None -async def test_pam_auth_blacklist(): +async def test_pam_auth_blocked(): # Null case compared to next case authenticator = MockPAMAuthenticator() authorized = await authenticator.get_authenticated_user( @@ -183,50 +184,41 @@ async def test_pam_auth_blacklist(): assert authorized['name'] == 'wash' # Blacklist basics - authenticator = MockPAMAuthenticator(blacklist={'wash'}) + authenticator = MockPAMAuthenticator(blocked={'wash'}) authorized = await authenticator.get_authenticated_user( None, {'username': 'wash', 'password': 'wash'} ) assert authorized is None - # User in both white and blacklists: default deny. Make error someday? - authenticator = MockPAMAuthenticator( - blacklist={'wash'}, whitelist={'wash', 'kaylee'} - ) + # User in both allowed and blocked: default deny. Make error someday? + authenticator = MockPAMAuthenticator(blocked={'wash'}, allowed={'wash', 'kaylee'}) authorized = await authenticator.get_authenticated_user( None, {'username': 'wash', 'password': 'wash'} ) assert authorized is None - # User not in blacklist can log in - authenticator = MockPAMAuthenticator( - blacklist={'wash'}, whitelist={'wash', 'kaylee'} - ) + # User not in blocked set can log in + authenticator = MockPAMAuthenticator(blocked={'wash'}, allowed={'wash', 'kaylee'}) authorized = await authenticator.get_authenticated_user( None, {'username': 'kaylee', 'password': 'kaylee'} ) assert authorized['name'] == 'kaylee' - # User in whitelist, blacklist irrelevent - authenticator = MockPAMAuthenticator( - blacklist={'mal'}, whitelist={'wash', 'kaylee'} - ) + # User in allowed, blocked irrelevent + authenticator = MockPAMAuthenticator(blocked={'mal'}, allowed={'wash', 'kaylee'}) authorized = await authenticator.get_authenticated_user( None, {'username': 'wash', 'password': 'wash'} ) assert authorized['name'] == 'wash' # User in neither list - authenticator = MockPAMAuthenticator( - blacklist={'mal'}, whitelist={'wash', 'kaylee'} - ) + authenticator = MockPAMAuthenticator(blocked={'mal'}, allowed={'wash', 'kaylee'}) authorized = await authenticator.get_authenticated_user( None, {'username': 'simon', 'password': 'simon'} ) assert authorized is None - # blacklist == {} - authenticator = MockPAMAuthenticator(blacklist=set(), whitelist={'wash', 'kaylee'}) + authenticator = MockPAMAuthenticator(blocked=set(), allowed={'wash', 'kaylee'}) authorized = await authenticator.get_authenticated_user( None, {'username': 'kaylee', 'password': 'kaylee'} ) @@ -253,7 +245,7 @@ async def test_deprecated_signatures(): async def test_pam_auth_no_such_group(): - authenticator = MockPAMAuthenticator(group_whitelist={'nosuchcrazygroup'}) + authenticator = MockPAMAuthenticator(allowed_groups={'nosuchcrazygroup'}) authorized = await authenticator.get_authenticated_user( None, {'username': 'kaylee', 'password': 'kaylee'} ) @@ -262,7 +254,7 @@ async def test_pam_auth_no_such_group(): async def test_wont_add_system_user(): user = orm.User(name='lioness4321') - authenticator = auth.PAMAuthenticator(whitelist={'mal'}) + authenticator = auth.PAMAuthenticator(allowed={'mal'}) authenticator.create_system_users = False with pytest.raises(KeyError): await authenticator.add_user(user) @@ -270,7 +262,7 @@ async def test_wont_add_system_user(): async def test_cant_add_system_user(): user = orm.User(name='lioness4321') - authenticator = auth.PAMAuthenticator(whitelist={'mal'}) + authenticator = auth.PAMAuthenticator(allowed={'mal'}) authenticator.add_user_cmd = ['jupyterhub-fake-command'] authenticator.create_system_users = True @@ -296,7 +288,7 @@ async def test_cant_add_system_user(): async def test_add_system_user(): user = orm.User(name='lioness4321') - authenticator = auth.PAMAuthenticator(whitelist={'mal'}) + authenticator = auth.PAMAuthenticator(allowed={'mal'}) authenticator.create_system_users = True authenticator.add_user_cmd = ['echo', '/home/USERNAME'] @@ -317,13 +309,13 @@ async def test_add_system_user(): async def test_delete_user(): user = orm.User(name='zoe') - a = MockPAMAuthenticator(whitelist={'mal'}) + a = MockPAMAuthenticator(allowed={'mal'}) - assert 'zoe' not in a.whitelist + assert 'zoe' not in a.allowed await a.add_user(user) - assert 'zoe' in a.whitelist + assert 'zoe' in a.allowed a.delete_user(user) - assert 'zoe' not in a.whitelist + assert 'zoe' not in a.allowed def test_urls(): @@ -461,3 +453,10 @@ async def test_post_auth_hook(): ) assert authorized['testkey'] == 'testvalue' + + +async def test_deprecations(): + cfg = Config() + cfg.Authenticator.whitelist = {'user'} + authenticator = auth.Authenticator(config=cfg) + assert authenticator.allowed == {'user'} diff --git a/jupyterhub/tests/test_pages.py b/jupyterhub/tests/test_pages.py index a1825d43..44e67332 100644 --- a/jupyterhub/tests/test_pages.py +++ b/jupyterhub/tests/test_pages.py @@ -686,7 +686,7 @@ async def test_shutdown_on_logout(app, shutdown_on_logout): assert spawner.ready == (not shutdown_on_logout) -async def test_login_no_whitelist_adds_user(app): +async def test_login_no_allowed_adds_user(app): auth = app.authenticator mock_add_user = mock.Mock() with mock.patch.object(auth, 'add_user', mock_add_user): diff --git a/jupyterhub/tests/test_services_auth.py b/jupyterhub/tests/test_services_auth.py index de4d73e9..752091d2 100644 --- a/jupyterhub/tests/test_services_auth.py +++ b/jupyterhub/tests/test_services_auth.py @@ -185,7 +185,7 @@ def test_hub_authenticated(request): m.get(good_url, text=json.dumps(mock_model)) - # no whitelist + # no specific allowed user r = requests.get( 'http://127.0.0.1:%i' % port, cookies={'jubal': 'early'}, @@ -194,7 +194,7 @@ def test_hub_authenticated(request): r.raise_for_status() assert r.status_code == 200 - # pass whitelist + # pass allowed user TestHandler.hub_users = {'jubalearly'} r = requests.get( 'http://127.0.0.1:%i' % port, @@ -204,7 +204,7 @@ def test_hub_authenticated(request): r.raise_for_status() assert r.status_code == 200 - # no pass whitelist + # no pass allowed ser TestHandler.hub_users = {'kaylee'} r = requests.get( 'http://127.0.0.1:%i' % port, @@ -213,7 +213,7 @@ def test_hub_authenticated(request): ) assert r.status_code == 403 - # pass group whitelist + # pass allowed group TestHandler.hub_groups = {'lions'} r = requests.get( 'http://127.0.0.1:%i' % port, @@ -223,7 +223,7 @@ def test_hub_authenticated(request): r.raise_for_status() assert r.status_code == 200 - # no pass group whitelist + # no pass allowed group TestHandler.hub_groups = {'tigers'} r = requests.get( 'http://127.0.0.1:%i' % port, From 14378f4cc2da8ced635841547d396d07027f56d7 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Mon, 15 Jun 2020 21:45:33 +1200 Subject: [PATCH 075/156] Include the query string parameters when redirecting to a new URL --- jupyterhub/handlers/base.py | 28 +++++++++++++++++++++++++++ jupyterhub/handlers/login.py | 11 +++++++++-- jupyterhub/handlers/pages.py | 2 ++ share/jupyterhub/templates/login.html | 2 +- 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/jupyterhub/handlers/base.py b/jupyterhub/handlers/base.py index 9606fa61..f2d21e89 100644 --- a/jupyterhub/handlers/base.py +++ b/jupyterhub/handlers/base.py @@ -654,8 +654,36 @@ class BaseHandler(RequestHandler): next_url = url_path_join(self.hub.base_url, 'spawn') else: next_url = url_path_join(self.hub.base_url, 'home') + + next_url = self.append_query_parameters(next_url, exclude=['next']) return next_url + def append_query_parameters(self, url, exclude=None): + """Append the current request's query parameters to the given URL. + + Supports an extra optional parameter ``exclude`` that when provided must + contain a list of parameters to be ignored, i.e. these parameters will + not be added to the URL. + + This is important to avoid infinite loops with the next parameter being + added over and over, for instance. + + :param str url: a URL + :param list exclude: optional list of parameters to be ignored + :rtype (str) + """ + if not exclude: + exclude = [] + if self.request.query: + query_string = [ + param + for param in parse_qsl(self.request.query) + if param[0] not in exclude + ] + if query_string: + url = url_concat(url, query_string) + return url + async def auth_to_user(self, authenticated, user=None): """Persist data from .authenticate() or .refresh_user() to the User database diff --git a/jupyterhub/handlers/login.py b/jupyterhub/handlers/login.py index f1bd7c45..85654564 100644 --- a/jupyterhub/handlers/login.py +++ b/jupyterhub/handlers/login.py @@ -90,13 +90,20 @@ class LoginHandler(BaseHandler): """Render the login page.""" def _render(self, login_error=None, username=None): + # login url with any parameters specified by the user/application + login_url = self.append_query_parameters( + self.settings['login_url'], exclude=['next'] + ) + # the next url for after the login + next_url = url_escape(self.get_argument('next', default='')) + # now concat the login_url with the next_url + login_url = url_concat(login_url, {'next': next_url}) return self.render_template( 'login.html', - next=url_escape(self.get_argument('next', default='')), username=username, login_error=login_error, custom_html=self.authenticator.custom_html, - login_url=self.settings['login_url'], + login_url=login_url, authenticator_login_url=url_concat( self.authenticator.login_url(self.hub.base_url), {'next': self.get_argument('next', '')}, diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index ffc44bb5..ea0c1d34 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -285,6 +285,8 @@ class SpawnHandler(BaseHandler): self.hub.base_url, "spawn-pending", user.escaped_name, server_name ) + pending_url = self.append_query_parameters(pending_url, exclude=['next']) + if self.get_argument('next', None): # preserve `?next=...` through spawn-pending pending_url = url_concat(pending_url, {'next': self.get_argument('next')}) diff --git a/share/jupyterhub/templates/login.html b/share/jupyterhub/templates/login.html index 130339c9..6242ab71 100644 --- a/share/jupyterhub/templates/login.html +++ b/share/jupyterhub/templates/login.html @@ -19,7 +19,7 @@
    {% else %} -
    +
    Sign in
    From ec710f4d901f39001363b45fe0608cb409abe0a9 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 18 Jun 2020 11:50:44 +0200 Subject: [PATCH 076/156] test subclass priority when overriding old methods --- jupyterhub/auth.py | 104 +++++++++++++++++++++++----------- jupyterhub/tests/test_auth.py | 49 +++++++++++++++- 2 files changed, 118 insertions(+), 35 deletions(-) diff --git a/jupyterhub/auth.py b/jupyterhub/auth.py index 476e7441..cfc82541 100644 --- a/jupyterhub/auth.py +++ b/jupyterhub/auth.py @@ -294,41 +294,74 @@ class Authenticator(LoggingConfigurable): def __init__(self, **kwargs): super().__init__(**kwargs) + self._init_deprecated_methods() + + def _init_deprecated_methods(self): # TODO: properly handle deprecated signature *and* name # with correct subclass override priority! - for method_name in ( - 'check_whitelist', - 'check_blacklist', - 'check_group_whitelist', + for old_name, new_name in ( + ('check_whitelist', 'check_allowed'), + ('check_blacklist', 'check_blocked'), + ('check_group_whitelist', 'check_allowed_groups'), ): - original_method = getattr(self, method_name, None) - if original_method is None: + old_method = getattr(self, old_name, None) + if old_method is None: # no such method (check_group_whitelist is optional) continue - signature = inspect.signature(original_method) - if 'authentication' not in signature.parameters: + + # allow old name to have higher priority + # if and only if it's defined in a later subclass + # than the new name + for cls in self.__class__.mro(): + has_old_name = old_name in cls.__dict__ + has_new_name = new_name in cls.__dict__ + if has_new_name: + break + if has_old_name and not has_new_name: + warnings.warn( + "{0}.{1} should be renamed to {0}.{2} for JupyterHub >= 1.2".format( + cls.__name__, old_name, new_name + ), + DeprecationWarning, + ) + # use old name instead of new + # if old name is overridden in subclass + def _new_calls_old(old_name, *args, **kwargs): + return getattr(self, old_name)(*args, **kwargs) + + setattr(self, new_name, partial(_new_calls_old, old_name)) + break + + # deprecate pre-1.0 method signatures + signature = inspect.signature(old_method) + if 'authentication' not in signature.parameters and not any( + param.kind == inspect.Parameter.VAR_KEYWORD + for param in signature.parameters.values() + ): # adapt to pre-1.0 signature for compatibility warnings.warn( """ {0}.{1} does not support the authentication argument, - added in JupyterHub 1.0. + added in JupyterHub 1.0. and is renamed to {2} in JupyterHub 1.2. It should have the signature: - def {1}(self, username, authentication=None): + def {2}(self, username, authentication=None): ... Adapting for compatibility. """.format( - self.__class__.__name__, method_name + self.__class__.__name__, old_name, new_name ), DeprecationWarning, ) - def wrapped_method(username, authentication=None, **kwargs): + def wrapped_method( + original_method, username, authentication=None, **kwargs + ): return original_method(username, **kwargs) - setattr(self, method_name, wrapped_method) + setattr(self, old_name, partial(wrapped_method, old_method)) async def run_post_auth_hook(self, handler, authentication): """ @@ -648,25 +681,30 @@ class Authenticator(LoggingConfigurable): return [('/login', LoginHandler)] -def _deprecated_method(old_name, new_name, version, self, *args, **kwargs): - """Method wrapper for a deprecated method name""" +def _deprecated_method(old_name, new_name, version): + """Create a deprecated method wrapper for a deprecated method name""" - warnings.warn( - ( - "{cls}.{old_name} is deprecated in JupyterHub {version}." - " Please use {cls}.{new_name} instead." - ).format( - cls=self.__class__.__name__, - old_name=old_name, - new_name=new_name, - version=version, - ), - DeprecationWarning, - stacklevel=2, - ) - old_method = getattr(self, new_name) - return old_method(*args, **kwargs) + def deprecated(self, *args, **kwargs): + warnings.warn( + ( + "{cls}.{old_name} is deprecated in JupyterHub {version}." + " Please use {cls}.{new_name} instead." + ).format( + cls=self.__class__.__name__, + old_name=old_name, + new_name=new_name, + version=version, + ), + DeprecationWarning, + stacklevel=2, + ) + old_method = getattr(self, new_name) + return old_method(*args, **kwargs) + return deprecated + + +import types # deprecate white/blacklist method names for _old_name, _new_name, _version in [ @@ -674,9 +712,7 @@ for _old_name, _new_name, _version in [ ("check_blacklist", "check_blocked", "1.2"), ]: setattr( - Authenticator, - _old_name, - partial(_deprecated_method, _old_name, _new_name, _version), + Authenticator, _old_name, _deprecated_method(_old_name, _new_name, _version), ) @@ -1062,7 +1098,7 @@ for _old_name, _new_name, _version in [ setattr( LocalAuthenticator, _old_name, - partial(_deprecated_method, _old_name, _new_name, _version), + _deprecated_method(_old_name, _new_name, _version), ) diff --git a/jupyterhub/tests/test_auth.py b/jupyterhub/tests/test_auth.py index 3c9b79ec..0021b1da 100644 --- a/jupyterhub/tests/test_auth.py +++ b/jupyterhub/tests/test_auth.py @@ -1,7 +1,9 @@ """Tests for PAM authentication""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +import logging import os +import warnings from unittest import mock import pytest @@ -455,8 +457,53 @@ async def test_post_auth_hook(): assert authorized['testkey'] == 'testvalue' -async def test_deprecations(): +class MyAuthenticator(auth.Authenticator): + def check_whitelist(self, username, authentication=None): + return username == "subclass-allowed" + + +def test_deprecated_config(caplog): + cfg = Config() + cfg.Authenticator.whitelist = {'user'} + log = logging.getLogger("testlog") + authenticator = auth.Authenticator(config=cfg, log=log) + assert caplog.record_tuples == [ + ( + log.name, + logging.WARNING, + 'Authenticator.whitelist is deprecated in JupyterHub 1.2, use ' + 'Authenticator.allowed instead', + ) + ] + assert authenticator.allowed == {'user'} + + +def test_deprecated_methods(): cfg = Config() cfg.Authenticator.whitelist = {'user'} authenticator = auth.Authenticator(config=cfg) + + assert authenticator.check_allowed("user") + with pytest.deprecated_call(): + assert authenticator.check_whitelist("user") + assert not authenticator.check_allowed("otheruser") + with pytest.deprecated_call(): + assert not authenticator.check_whitelist("otheruser") + + +def test_deprecated_config_subclass(): + cfg = Config() + cfg.MyAuthenticator.whitelist = {'user'} + with pytest.deprecated_call(): + authenticator = MyAuthenticator(config=cfg) assert authenticator.allowed == {'user'} + + +def test_deprecated_methods_subclass(): + with pytest.deprecated_call(): + authenticator = MyAuthenticator() + + assert authenticator.check_allowed("subclass-allowed") + assert authenticator.check_whitelist("subclass-allowed") + assert not authenticator.check_allowed("otheruser") + assert not authenticator.check_whitelist("otheruser") From d4ce3aa731511aec60dcb5767b4b47715d319a33 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Sat, 20 Jun 2020 22:19:54 +1200 Subject: [PATCH 077/156] Add unit tests --- jupyterhub/tests/test_pages.py | 46 +++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/jupyterhub/tests/test_pages.py b/jupyterhub/tests/test_pages.py index a1825d43..544b54c9 100644 --- a/jupyterhub/tests/test_pages.py +++ b/jupyterhub/tests/test_pages.py @@ -516,6 +516,19 @@ async def test_user_redirect_deprecated(app, username): ) +async def test_login_page(app): + url = url_concat('login', dict(next='foo', param1='test')) + r = await get_page(url, app) + assert r.url.endswith('/hub/login?next=foo¶m1=test') + # now the login.html rendered template must include the given parameters in the form + # action URL, including the next URL + page = BeautifulSoup(r.text, "html.parser") + form = page.find("form", method="post") + action = form.attrs['action'] + assert 'next=foo' in action + assert 'param1=test' in action + + async def test_login_fail(app): name = 'wash' base_url = public_url(app) @@ -546,26 +559,29 @@ async def test_login_strip(app): @pytest.mark.parametrize( - 'running, next_url, location', + 'running, next_url, location, params', [ # default URL if next not specified, for both running and not - (True, '', ''), - (False, '', ''), + (True, '', '', None), + (False, '', '', None), # next_url is respected - (False, '/hub/admin', '/hub/admin'), - (False, '/user/other', '/hub/user/other'), - (False, '/absolute', '/absolute'), - (False, '/has?query#andhash', '/has?query#andhash'), + (False, '/hub/admin', '/hub/admin', None), + (False, '/user/other', '/hub/user/other', None), + (False, '/absolute', '/absolute', None), + (False, '/has?query#andhash', '/has?query#andhash', None), # next_url outside is not allowed - (False, 'relative/path', ''), - (False, 'https://other.domain', ''), - (False, 'ftp://other.domain', ''), - (False, '//other.domain', ''), - (False, '///other.domain/triple', ''), - (False, '\\\\other.domain/backslashes', ''), + (False, 'relative/path', '', None), + (False, 'https://other.domain', '', None), + (False, 'ftp://other.domain', '', None), + (False, '//other.domain', '', None), + (False, '///other.domain/triple', '', None), + (False, '\\\\other.domain/backslashes', '', None), + # params are handled correctly + (True, '/hub/admin', 'hub/admin?left=1&right=2', dict(left=1, right=2)), + (False, '/hub/admin', 'hub/admin?left=1&right=2', dict(left=1, right=2)), ], ) -async def test_login_redirect(app, running, next_url, location): +async def test_login_redirect(app, running, next_url, location, params): cookies = await app.login_user('river') user = app.users['river'] if location: @@ -577,6 +593,8 @@ async def test_login_redirect(app, running, next_url, location): location = ujoin(app.base_url, 'hub/spawn') url = 'login' + if params: + url = url_concat(url, params) if next_url: if '//' not in next_url and next_url.startswith('/'): next_url = ujoin(app.base_url, next_url, '') From 8528684dc42ef4ca9ee633bf6ac26ed2599afb55 Mon Sep 17 00:00:00 2001 From: Matt Riedemann Date: Mon, 22 Jun 2020 15:35:15 -0500 Subject: [PATCH 078/156] Remove old context-less print statement This was added in PR #2721 and by default results in just printing out "10" without any context when starting the hub service. This simply removes the orphan print statement. I'm open to changing this to a debug log statement with context if someone finds that useful, e.g.: `self.log.debug('Effective init_spawners_timeout: %s', init_spawners_timeout)` --- jupyterhub/app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 8b2119ed..dd8f6db2 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -2360,7 +2360,6 @@ class JupyterHub(Application): if init_spawners_timeout < 0: # negative timeout means forever (previous, most stable behavior) init_spawners_timeout = 86400 - print(init_spawners_timeout) init_start_time = time.perf_counter() init_spawners_future = asyncio.ensure_future(self.init_spawners()) From 7ad4b0c7cb1a71c2fc39f064a8e73a04ed75ddb2 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 23 Jun 2020 10:10:07 +0200 Subject: [PATCH 079/156] update allowed/blocked language in docs our words matter, let's be more mindful --- docs/source/changelog.md | 10 ++++++---- .../authenticators-users-basics.md | 20 +++++++++---------- docs/source/reference/config-ghoauth.md | 2 +- docs/source/reference/config-sudo.md | 2 +- docs/source/reference/services.md | 2 +- docs/source/troubleshooting.md | 15 +++++++------- 6 files changed, 26 insertions(+), 25 deletions(-) diff --git a/docs/source/changelog.md b/docs/source/changelog.md index 5589b968..6eb740e7 100644 --- a/docs/source/changelog.md +++ b/docs/source/changelog.md @@ -7,6 +7,8 @@ command line for details. ## [Unreleased] + + ## 1.1 ### [1.1.0] 2020-01-17 @@ -116,7 +118,7 @@ Thanks to everyone who has contributed to this release! - Log JupyterHub version on startup [#2752](https://github.com/jupyterhub/jupyterhub/pull/2752) ([@consideRatio](https://github.com/consideRatio)) - Reduce verbosity for "Failing suspected API request to not-running server" (new) [#2751](https://github.com/jupyterhub/jupyterhub/pull/2751) ([@rkdarst](https://github.com/rkdarst)) - Add missing package for json schema doc build [#2744](https://github.com/jupyterhub/jupyterhub/pull/2744) ([@willingc](https://github.com/willingc)) -- blacklist urllib3 versions with encoding bug [#2743](https://github.com/jupyterhub/jupyterhub/pull/2743) ([@minrk](https://github.com/minrk)) +- block urllib3 versions with encoding bug [#2743](https://github.com/jupyterhub/jupyterhub/pull/2743) ([@minrk](https://github.com/minrk)) - Remove tornado deprecated/unnecessary AsyncIOMainLoop().install() call [#2740](https://github.com/jupyterhub/jupyterhub/pull/2740) ([@kinow](https://github.com/kinow)) - Fix deprecated call [#2739](https://github.com/jupyterhub/jupyterhub/pull/2739) ([@kinow](https://github.com/kinow)) - Remove duplicate hub and authenticator traitlets from Spawner [#2736](https://github.com/jupyterhub/jupyterhub/pull/2736) ([@eslavich](https://github.com/eslavich)) @@ -231,8 +233,8 @@ whether it was through discussion, testing, documentation, or development. This hook may transform the return value of `Authenticator.authenticate()` and return a new authentication dictionary, e.g. specifying admin privileges, group membership, - or custom white/blacklisting logic. - This hook is called *after* existing normalization and whitelist checking. + or custom allowed/blocked logic. + This hook is called *after* existing normalization and allowed-username checking. - `Spawner.options_from_form` may now be async - Added `JupyterHub.shutdown_on_logout` option to trigger shutdown of a user's servers when they log out. @@ -418,7 +420,7 @@ and tornado < 5.0. launching an IPython session connected to your JupyterHub database. - Include `User.auth_state` in user model on single-user REST endpoints for admins only. - Include `Server.state` in server model on REST endpoints for admins only. -- Add `Authenticator.blacklist` for blacklisting users instead of whitelisting. +- Add `Authenticator.blacklist` for blocking users instead of allowing. - Pass `c.JupyterHub.tornado_settings['cookie_options']` down to Spawners so that cookie options (e.g. `expires_days`) can be set globally for the whole application. - SIGINFO (`ctrl-t`) handler showing the current status of all running threads, diff --git a/docs/source/getting-started/authenticators-users-basics.md b/docs/source/getting-started/authenticators-users-basics.md index b954f88d..a2647976 100644 --- a/docs/source/getting-started/authenticators-users-basics.md +++ b/docs/source/getting-started/authenticators-users-basics.md @@ -4,23 +4,23 @@ The default Authenticator uses [PAM][] to authenticate system users with their username and password. With the default Authenticator, any user with an account and password on the system will be allowed to login. -## Create a whitelist of users +## Create a set of allowed users -You can restrict which users are allowed to login with a whitelist, -`Authenticator.whitelist`: +You can restrict which users are allowed to login with a set, +`Authenticator.allowed`: ```python -c.Authenticator.whitelist = {'mal', 'zoe', 'inara', 'kaylee'} +c.Authenticator.allowed = {'mal', 'zoe', 'inara', 'kaylee'} ``` -Users in the whitelist are added to the Hub database when the Hub is +Users in the allowed set are added to the Hub database when the Hub is started. ## Configure admins (`admin_users`) Admin users of JupyterHub, `admin_users`, can add and remove users from -the user `whitelist`. `admin_users` can take actions on other users' +the user `allowed` set. `admin_users` can take actions on other users' behalf, such as stopping and restarting their servers. A set of initial admin users, `admin_users` can configured be as follows: @@ -28,7 +28,7 @@ A set of initial admin users, `admin_users` can configured be as follows: ```python c.Authenticator.admin_users = {'mal', 'zoe'} ``` -Users in the admin list are automatically added to the user `whitelist`, +Users in the admin set are automatically added to the user `allowed` set, if they are not already present. Each authenticator may have different ways of determining whether a user is an @@ -53,12 +53,12 @@ sure your users know if admin_access is enabled.** Users can be added to and removed from the Hub via either the admin panel or the REST API. When a user is **added**, the user will be -automatically added to the whitelist and database. Restarting the Hub -will not require manually updating the whitelist in your config file, +automatically added to the allowed set and database. Restarting the Hub +will not require manually updating the allowed set in your config file, as the users will be loaded from the database. After starting the Hub once, it is not sufficient to **remove** a user -from the whitelist in your config file. You must also remove the user +from the allowed set in your config file. You must also remove the user from the Hub's database, either by deleting the user from JupyterHub's admin page, or you can clear the `jupyterhub.sqlite` database and start fresh. diff --git a/docs/source/reference/config-ghoauth.md b/docs/source/reference/config-ghoauth.md index b120b3b2..4183ddec 100644 --- a/docs/source/reference/config-ghoauth.md +++ b/docs/source/reference/config-ghoauth.md @@ -52,7 +52,7 @@ c.GitHubOAuthenticator.oauth_callback_url = os.environ['OAUTH_CALLBACK_URL'] c.LocalAuthenticator.create_system_users = True # specify users and admin -c.Authenticator.whitelist = {'rgbkrk', 'minrk', 'jhamrick'} +c.Authenticator.allowed = {'rgbkrk', 'minrk', 'jhamrick'} c.Authenticator.admin_users = {'jhamrick', 'rgbkrk'} # uses the default spawner diff --git a/docs/source/reference/config-sudo.md b/docs/source/reference/config-sudo.md index fc0d4865..310f1a93 100644 --- a/docs/source/reference/config-sudo.md +++ b/docs/source/reference/config-sudo.md @@ -57,7 +57,7 @@ To do this we add to `/etc/sudoers` (use `visudo` for safe editing of sudoers): For example: ```bash -# comma-separated whitelist of users that can spawn single-user servers +# comma-separated list of users that can spawn single-user servers # this should include all of your Hub users Runas_Alias JUPYTER_USERS = rhea, zoe, wash diff --git a/docs/source/reference/services.md b/docs/source/reference/services.md index ec446c8e..d940034b 100644 --- a/docs/source/reference/services.md +++ b/docs/source/reference/services.md @@ -313,7 +313,7 @@ class MyHandler(HubAuthenticated, web.RequestHandler): The HubAuth will automatically load the desired configuration from the Service environment variables. -If you want to limit user access, you can whitelist users through either the +If you want to limit user access, you can specify allowed users through either the `.hub_users` attribute or `.hub_groups`. These are sets that check against the username and user group list, respectively. If a user matches neither the user list nor the group list, they will not be allowed access. If both are left diff --git a/docs/source/troubleshooting.md b/docs/source/troubleshooting.md index d92c594b..3958e590 100644 --- a/docs/source/troubleshooting.md +++ b/docs/source/troubleshooting.md @@ -7,8 +7,8 @@ problem and how to resolve it. [*Behavior*](#behavior) - JupyterHub proxy fails to start - sudospawner fails to run -- What is the default behavior when none of the lists (admin, whitelist, - group whitelist) are set? +- What is the default behavior when none of the lists (admin, allowed, + allowed groups) are set? - JupyterHub Docker container not accessible at localhost [*Errors*](#errors) @@ -55,14 +55,14 @@ or add: to the config file, `jupyterhub_config.py`. -### What is the default behavior when none of the lists (admin, whitelist, group whitelist) are set? +### What is the default behavior when none of the lists (admin, allowed, allowed groups) are set? When nothing is given for these lists, there will be no admins, and all users who can authenticate on the system (i.e. all the unix users on the server with -a password) will be allowed to start a server. The whitelist lets you limit -this to a particular set of users, and the admin_users lets you specify who +a password) will be allowed to start a server. The allowed username set lets you limit +this to a particular set of users, and admin_users lets you specify who among them may use the admin interface (not necessary, unless you need to do -things like inspect other users' servers, or modify the userlist at runtime). +things like inspect other users' servers, or modify the user list at runtime). ### JupyterHub Docker container not accessible at localhost @@ -332,8 +332,7 @@ notebook servers to default to JupyterLab: ### How do I set up JupyterHub for a workshop (when users are not known ahead of time)? 1. Set up JupyterHub using OAuthenticator for GitHub authentication -2. Configure whitelist to be an empty list in` jupyterhub_config.py` -3. Configure admin list to have workshop leaders be listed with administrator privileges. +2. Configure admin list to have workshop leaders be listed with administrator privileges. Users will need a GitHub account to login and be authenticated by the Hub. From 3fece09dda032a67b6d33636c66b92e3ee09222b Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 23 Jun 2020 10:13:31 +0200 Subject: [PATCH 080/156] loosen jupyter-telemetry pin we don't want strict pinning in package dependencies --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 07cc96e8..a74d1d16 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ async_generator>=1.8 certipy>=0.1.2 entrypoints jinja2 -jupyter_telemetry==0.1.0 +jupyter_telemetry>=0.1.0 oauthlib>=3.0 pamela prometheus_client>=0.0.21 From 73f1211286668c13350fbbb51e2c88a986ddd3d0 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Tue, 23 Jun 2020 20:11:49 +1200 Subject: [PATCH 081/156] Update append_query_parameters to have exclude=["none"] by default, and avoid using dicts with url_concat, to have consistent tests as otherwise in Python 3.5 the generated URL's could have parameters in random order. --- jupyterhub/handlers/base.py | 10 ++++++++-- jupyterhub/tests/test_pages.py | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/jupyterhub/handlers/base.py b/jupyterhub/handlers/base.py index f2d21e89..b08a6ac7 100644 --- a/jupyterhub/handlers/base.py +++ b/jupyterhub/handlers/base.py @@ -668,12 +668,18 @@ class BaseHandler(RequestHandler): This is important to avoid infinite loops with the next parameter being added over and over, for instance. + The default value for ``exclude`` is an array with "next". This is useful + as most use cases in JupyterHub (all?) won't want to include the next + parameter twice (the next parameter is added elsewhere to the query + parameters). + :param str url: a URL - :param list exclude: optional list of parameters to be ignored + :param list exclude: optional list of parameters to be ignored, defaults to + a list with "next" (to avoid redirect-loops) :rtype (str) """ if not exclude: - exclude = [] + exclude = ['next'] if self.request.query: query_string = [ param diff --git a/jupyterhub/tests/test_pages.py b/jupyterhub/tests/test_pages.py index 544b54c9..3dc478ef 100644 --- a/jupyterhub/tests/test_pages.py +++ b/jupyterhub/tests/test_pages.py @@ -517,7 +517,7 @@ async def test_user_redirect_deprecated(app, username): async def test_login_page(app): - url = url_concat('login', dict(next='foo', param1='test')) + url = url_concat('login', [('next', 'foo'), ('param1', 'test')]) r = await get_page(url, app) assert r.url.endswith('/hub/login?next=foo¶m1=test') # now the login.html rendered template must include the given parameters in the form @@ -577,8 +577,8 @@ async def test_login_strip(app): (False, '///other.domain/triple', '', None), (False, '\\\\other.domain/backslashes', '', None), # params are handled correctly - (True, '/hub/admin', 'hub/admin?left=1&right=2', dict(left=1, right=2)), - (False, '/hub/admin', 'hub/admin?left=1&right=2', dict(left=1, right=2)), + (True, '/hub/admin', 'hub/admin?left=1&right=2', [('left', 1), ('right', 2)]), + (False, '/hub/admin', 'hub/admin?left=1&right=2', [('left', 1), ('right', 2)]), ], ) async def test_login_redirect(app, running, next_url, location, params): From 1bdc9aa29794d5fcb69c05990204ae06d5b5c7da Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Wed, 24 Jun 2020 00:18:55 +1200 Subject: [PATCH 082/156] Escape/encode parameters with the next URL, add more tests --- jupyterhub/handlers/login.py | 11 ++---- jupyterhub/tests/test_pages.py | 50 ++++++++++++++++++++++++--- share/jupyterhub/templates/login.html | 2 +- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/jupyterhub/handlers/login.py b/jupyterhub/handlers/login.py index 85654564..f1bd7c45 100644 --- a/jupyterhub/handlers/login.py +++ b/jupyterhub/handlers/login.py @@ -90,20 +90,13 @@ class LoginHandler(BaseHandler): """Render the login page.""" def _render(self, login_error=None, username=None): - # login url with any parameters specified by the user/application - login_url = self.append_query_parameters( - self.settings['login_url'], exclude=['next'] - ) - # the next url for after the login - next_url = url_escape(self.get_argument('next', default='')) - # now concat the login_url with the next_url - login_url = url_concat(login_url, {'next': next_url}) return self.render_template( 'login.html', + next=url_escape(self.get_argument('next', default='')), username=username, login_error=login_error, custom_html=self.authenticator.custom_html, - login_url=login_url, + login_url=self.settings['login_url'], authenticator_login_url=url_concat( self.authenticator.login_url(self.hub.base_url), {'next': self.get_argument('next', '')}, diff --git a/jupyterhub/tests/test_pages.py b/jupyterhub/tests/test_pages.py index 3dc478ef..74de03a5 100644 --- a/jupyterhub/tests/test_pages.py +++ b/jupyterhub/tests/test_pages.py @@ -9,6 +9,7 @@ from urllib.parse import urlparse import pytest from bs4 import BeautifulSoup from tornado import gen +from tornado.escape import url_escape from tornado.httputil import url_concat from .. import orm @@ -516,17 +517,56 @@ async def test_user_redirect_deprecated(app, username): ) -async def test_login_page(app): - url = url_concat('login', [('next', 'foo'), ('param1', 'test')]) +@pytest.mark.parametrize( + 'url, params, redirected_url, form_action', + [ + ( + # spawn?param=value + # will encode given parameters for an unauthenticated URL in the next url + # the next parameter will contain the app base URL (replaces BASE_URL in tests) + 'spawn', + [('param', 'value')], + '/hub/login?next={{BASE_URL}}hub%2Fspawn%3Fparam%3Dvalue', + '/hub/login?next={{BASE_URL}}hub%2Fspawn%3Fparam%3Dvalue', + ), + ( + # login?param=fromlogin&next=encoded(/hub/spawn?param=value) + # will drop parameters given to the login page, passing only the next url + 'login', + [('param', 'fromlogin'), ('next', '/hub/spawn?param=value')], + '/hub/login?param=fromlogin&next=%2Fhub%2Fspawn%3Fparam%3Dvalue', + '/hub/login?next=%2Fhub%2Fspawn%3Fparam%3Dvalue', + ), + ( + # login?param=value&anotherparam=anothervalue + # will drop parameters given to the login page, and use an empty next url + 'login', + [('param', 'value'), ('anotherparam', 'anothervalue')], + '/hub/login?param=value&anotherparam=anothervalue', + '/hub/login?next=', + ), + ( + # login + # simplest case, accessing the login URL, gives an empty next url + 'login', + [], + '/hub/login', + '/hub/login?next=', + ), + ], +) +async def test_login_page(app, url, params, redirected_url, form_action): + url = url_concat(url, params) r = await get_page(url, app) - assert r.url.endswith('/hub/login?next=foo¶m1=test') + redirected_url = redirected_url.replace('{{BASE_URL}}', url_escape(app.base_url)) + assert r.url.endswith(redirected_url) # now the login.html rendered template must include the given parameters in the form # action URL, including the next URL page = BeautifulSoup(r.text, "html.parser") form = page.find("form", method="post") action = form.attrs['action'] - assert 'next=foo' in action - assert 'param1=test' in action + form_action = form_action.replace('{{BASE_URL}}', url_escape(app.base_url)) + assert action.endswith(form_action) async def test_login_fail(app): diff --git a/share/jupyterhub/templates/login.html b/share/jupyterhub/templates/login.html index 6242ab71..130339c9 100644 --- a/share/jupyterhub/templates/login.html +++ b/share/jupyterhub/templates/login.html @@ -19,7 +19,7 @@
    {% else %} - +
    Sign in
    From 6e988bf58781c65f559d32393cdf66633b71a1b1 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 24 Jun 2020 13:29:42 +0200 Subject: [PATCH 083/156] call it allowed_users be clearer since it's users vs groups, etc. --- .../authenticators-users-basics.md | 16 ++--- docs/source/reference/config-ghoauth.md | 2 +- jupyterhub/app.py | 29 +++++---- jupyterhub/auth.py | 64 ++++++++++--------- jupyterhub/tests/test_app.py | 2 +- jupyterhub/tests/test_auth.py | 44 ++++++++----- 6 files changed, 85 insertions(+), 72 deletions(-) diff --git a/docs/source/getting-started/authenticators-users-basics.md b/docs/source/getting-started/authenticators-users-basics.md index a2647976..ec078fed 100644 --- a/docs/source/getting-started/authenticators-users-basics.md +++ b/docs/source/getting-started/authenticators-users-basics.md @@ -7,20 +7,20 @@ with an account and password on the system will be allowed to login. ## Create a set of allowed users You can restrict which users are allowed to login with a set, -`Authenticator.allowed`: +`Authenticator.allowed_users`: ```python -c.Authenticator.allowed = {'mal', 'zoe', 'inara', 'kaylee'} +c.Authenticator.allowed_users = {'mal', 'zoe', 'inara', 'kaylee'} ``` -Users in the allowed set are added to the Hub database when the Hub is +Users in the `allowed_users` set are added to the Hub database when the Hub is started. ## Configure admins (`admin_users`) Admin users of JupyterHub, `admin_users`, can add and remove users from -the user `allowed` set. `admin_users` can take actions on other users' +the user `allowed_users` set. `admin_users` can take actions on other users' behalf, such as stopping and restarting their servers. A set of initial admin users, `admin_users` can configured be as follows: @@ -28,7 +28,7 @@ A set of initial admin users, `admin_users` can configured be as follows: ```python c.Authenticator.admin_users = {'mal', 'zoe'} ``` -Users in the admin set are automatically added to the user `allowed` set, +Users in the admin set are automatically added to the user `allowed_users` set, if they are not already present. Each authenticator may have different ways of determining whether a user is an @@ -53,12 +53,12 @@ sure your users know if admin_access is enabled.** Users can be added to and removed from the Hub via either the admin panel or the REST API. When a user is **added**, the user will be -automatically added to the allowed set and database. Restarting the Hub -will not require manually updating the allowed set in your config file, +automatically added to the allowed users set and database. Restarting the Hub +will not require manually updating the allowed users set in your config file, as the users will be loaded from the database. After starting the Hub once, it is not sufficient to **remove** a user -from the allowed set in your config file. You must also remove the user +from the allowed users set in your config file. You must also remove the user from the Hub's database, either by deleting the user from JupyterHub's admin page, or you can clear the `jupyterhub.sqlite` database and start fresh. diff --git a/docs/source/reference/config-ghoauth.md b/docs/source/reference/config-ghoauth.md index 4183ddec..6ec46e1c 100644 --- a/docs/source/reference/config-ghoauth.md +++ b/docs/source/reference/config-ghoauth.md @@ -52,7 +52,7 @@ c.GitHubOAuthenticator.oauth_callback_url = os.environ['OAUTH_CALLBACK_URL'] c.LocalAuthenticator.create_system_users = True # specify users and admin -c.Authenticator.allowed = {'rgbkrk', 'minrk', 'jhamrick'} +c.Authenticator.allowed_users = {'rgbkrk', 'minrk', 'jhamrick'} c.Authenticator.admin_users = {'jhamrick', 'rgbkrk'} # uses the default spawner diff --git a/jupyterhub/app.py b/jupyterhub/app.py index ba549aa4..4135f8f0 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -1689,22 +1689,22 @@ class JupyterHub(Application): # the admin_users config variable will never be used after this point. # only the database values will be referenced. - allowed = [ + allowed_users = [ self.authenticator.normalize_username(name) - for name in self.authenticator.allowed + for name in self.authenticator.allowed_users ] - self.authenticator.allowed = set(allowed) # force normalization - for username in allowed: + self.authenticator.allowed_users = set(allowed_users) # force normalization + for username in allowed_users: if not self.authenticator.validate_username(username): raise ValueError("username %r is not valid" % username) - if not allowed: + if not allowed_users: self.log.info( - "Not using allowed user list. Any authenticated user will be allowed." + "Not using allowed_users. Any authenticated user will be allowed." ) # add allowed users to the db - for name in allowed: + for name in allowed_users: user = orm.User.find(db, name) if user is None: user = orm.User(name=name) @@ -1714,9 +1714,9 @@ class JupyterHub(Application): db.commit() # Notify authenticator of all users. - # This ensures Authenticator.allowed is up-to-date with the database. - # This lets .allowed be used to set up initial list, - # but changes to the allowed list can occur in the database, + # This ensures Authenticator.allowed_users is up-to-date with the database. + # This lets .allowed_users be used to set up initial list, + # but changes to the allowed_users set can occur in the database, # and persist across sessions. total_users = 0 for user in db.query(orm.User): @@ -1753,9 +1753,9 @@ class JupyterHub(Application): user.created = user.last_activity or datetime.utcnow() db.commit() - # The allowed set and the users in the db are now the same. + # The allowed_users set and the users in the db are now the same. # From this point on, any user changes should be done simultaneously - # to the allowed set and user db, unless the allowed set is empty (all users allowed). + # to the allowed_users set and user db, unless the allowed set is empty (all users allowed). TOTAL_USERS.set(total_users) @@ -1773,7 +1773,7 @@ class JupyterHub(Application): await maybe_future(self.authenticator.check_allowed(username, None)) ): raise ValueError( - "Username %r is not in Authenticator.allowed" % username + "Username %r is not in Authenticator.allowed_users" % username ) user = orm.User.find(db, name=username) if user is None: @@ -1801,7 +1801,8 @@ class JupyterHub(Application): await maybe_future(self.authenticator.check_allowed(name, None)) ): raise ValueError( - "Token user name %r is not in Authenticator.allowed" % name + "Token user name %r is not in Authenticator.allowed_users" + % name ) if not self.authenticator.validate_username(name): raise ValueError("Token user name %r is not valid" % name) diff --git a/jupyterhub/auth.py b/jupyterhub/auth.py index cfc82541..dfbfab79 100644 --- a/jupyterhub/auth.py +++ b/jupyterhub/auth.py @@ -101,9 +101,9 @@ class Authenticator(LoggingConfigurable): """ ).tag(config=True) - whitelist = Set(help="Deprecated, use `Authenticator.allowed`", config=True,) + whitelist = Set(help="Deprecated, use `Authenticator.allowed_users`", config=True,) - allowed = Set( + allowed_users = Set( help=""" Set of usernames that are allowed to log in. @@ -114,11 +114,11 @@ class Authenticator(LoggingConfigurable): If empty, does not perform any additional restriction. .. versionchanged:: 1.2 - `Authenticator.whitelist` renamed to `allowed` + `Authenticator.whitelist` renamed to `allowed_users` """ ).tag(config=True) - blocked = Set( + blocked_users = Set( help=""" Set of usernames that are not allowed to log in. @@ -131,13 +131,13 @@ class Authenticator(LoggingConfigurable): .. versionadded: 0.9 .. versionchanged:: 1.2 - `Authenticator.blacklist` renamed to `blocked` + `Authenticator.blacklist` renamed to `blocked_users` """ ).tag(config=True) _deprecated_aliases = { - "whitelist": ("allowed", "1.2"), - "blacklist": ("blocked", "1.2"), + "whitelist": ("allowed_users", "1.2"), + "blacklist": ("blocked_users", "1.2"), } @observe(*list(_deprecated_aliases)) @@ -160,15 +160,15 @@ class Authenticator(LoggingConfigurable): ) setattr(self, new_attr, change.new) - @observe('allowed') - def _check_allowed(self, change): + @observe('allowed_users') + def _check_allowed_users(self, change): short_names = [name for name in change['new'] if len(name) <= 1] if short_names: sorted_names = sorted(short_names) single = ''.join(sorted_names) string_set_typo = "set('%s')" % single self.log.warning( - "Allowed list contains single-character names: %s; did you mean set([%r]) instead of %s?", + "Allowed set contains single-character names: %s; did you mean set([%r]) instead of %s?", sorted_names[:8], single, string_set_typo, @@ -301,7 +301,7 @@ class Authenticator(LoggingConfigurable): # with correct subclass override priority! for old_name, new_name in ( ('check_whitelist', 'check_allowed'), - ('check_blacklist', 'check_blocked'), + ('check_blacklist', 'check_blocked_users'), ('check_group_whitelist', 'check_allowed_groups'), ): old_method = getattr(self, old_name, None) @@ -399,7 +399,7 @@ class Authenticator(LoggingConfigurable): """Check if a username is allowed to authenticate based on configuration Return True if username is allowed, False otherwise. - No allowed set means any username is allowed. + No allowed_users set means any username is allowed. Names are normalized *before* being checked against the allowed set. @@ -409,12 +409,12 @@ class Authenticator(LoggingConfigurable): .. versionchanged:: 1.2 Renamed check_whitelist to check_allowed """ - if not self.allowed: + if not self.allowed_users: # No allowed set means any name is allowed return True - return username in self.allowed + return username in self.allowed_users - def check_blocked(self, username, authentication=None): + def check_blocked_users(self, username, authentication=None): """Check if a username is blocked to authenticate based on Authenticator.blocked configuration Return True if username is allowed, False otherwise. @@ -428,12 +428,12 @@ class Authenticator(LoggingConfigurable): Signature updated to accept authentication data as second argument .. versionchanged:: 1.2 - Renamed check_blacklist to check_blocked + Renamed check_blacklist to check_blocked_users """ - if not self.blocked: + if not self.blocked_users: # No block list means any name is allowed return True - return username not in self.blocked + return username not in self.blocked_users async def get_authenticated_user(self, handler, data): """Authenticate the user who is attempting to log in @@ -450,7 +450,7 @@ class Authenticator(LoggingConfigurable): The various stages can be overridden separately: - `authenticate` turns formdata into a username - `normalize_username` normalizes the username - - `check_allowed` checks against the user allowed + - `check_allowed` checks against the allowed usernames .. versionchanged:: 0.8 return dict instead of username @@ -475,7 +475,9 @@ class Authenticator(LoggingConfigurable): self.log.warning("Disallowing invalid username %r.", username) return - blocked_pass = await maybe_future(self.check_blocked(username, authenticated)) + blocked_pass = await maybe_future( + self.check_blocked_users(username, authenticated) + ) allowed_pass = await maybe_future(self.check_allowed(username, authenticated)) if blocked_pass: @@ -550,7 +552,7 @@ class Authenticator(LoggingConfigurable): It must return the username on successful authentication, and return None on failed authentication. - Checking allowed/blocked is handled separately by the caller. + Checking allowed_users/blocked_users is handled separately by the caller. .. versionchanged:: 0.8 Allow `authenticate` to return a dict containing auth_state. @@ -591,10 +593,10 @@ class Authenticator(LoggingConfigurable): This method may be a coroutine. - By default, this just adds the user to the allowed set. + By default, this just adds the user to the allowed_users set. Subclasses may do more extensive things, such as adding actual unix users, - but they should call super to ensure the allowed set is updated. + but they should call super to ensure the allowed_users set is updated. Note that this should be idempotent, since it is called whenever the hub restarts for all users. @@ -604,19 +606,19 @@ class Authenticator(LoggingConfigurable): """ if not self.validate_username(user.name): raise ValueError("Invalid username: %s" % user.name) - if self.allowed: - self.allowed.add(user.name) + if self.allowed_users: + self.allowed_users.add(user.name) def delete_user(self, user): """Hook called when a user is deleted - Removes the user from the allowed set. - Subclasses should call super to ensure the allowed set is updated. + Removes the user from the allowed_users set. + Subclasses should call super to ensure the allowed_users set is updated. Args: user (User): The User wrapper object """ - self.allowed.discard(user.name) + self.allowed_users.discard(user.name) auto_login = Bool( False, @@ -709,7 +711,7 @@ import types # deprecate white/blacklist method names for _old_name, _new_name, _version in [ ("check_whitelist", "check_allowed", "1.2"), - ("check_blacklist", "check_blocked", "1.2"), + ("check_blacklist", "check_blocked_users", "1.2"), ]: setattr( Authenticator, _old_name, _deprecated_method(_old_name, _new_name, _version), @@ -788,9 +790,9 @@ class LocalAuthenticator(Authenticator): @observe('allowed_groups') def _allowed_groups_changed(self, change): """Log a warning if mutually exclusive user and group allowed sets are specified.""" - if self.allowed: + if self.allowed_users: self.log.warning( - "Ignoring Authenticator.allowed set because Authenticator.allowed_groups supplied!" + "Ignoring Authenticator.allowed_users set because Authenticator.allowed_groups supplied!" ) def check_allowed(self, username, authentication=None): diff --git a/jupyterhub/tests/test_app.py b/jupyterhub/tests/test_app.py index 55000bc7..e00c699e 100644 --- a/jupyterhub/tests/test_app.py +++ b/jupyterhub/tests/test_app.py @@ -93,7 +93,7 @@ def test_generate_config(): os.remove(cfg_file) assert cfg_file in out assert 'Spawner.cmd' in cfg_text - assert 'Authenticator.allowed' in cfg_text + assert 'Authenticator.allowed_users' in cfg_text async def test_init_tokens(request): diff --git a/jupyterhub/tests/test_auth.py b/jupyterhub/tests/test_auth.py index 0021b1da..51cd4c2a 100644 --- a/jupyterhub/tests/test_auth.py +++ b/jupyterhub/tests/test_auth.py @@ -141,7 +141,7 @@ async def test_pam_auth_admin_groups(): async def test_pam_auth_allowed(): - authenticator = MockPAMAuthenticator(allowed={'wash', 'kaylee'}) + authenticator = MockPAMAuthenticator(allowed_users={'wash', 'kaylee'}) authorized = await authenticator.get_authenticated_user( None, {'username': 'kaylee', 'password': 'kaylee'} ) @@ -186,41 +186,51 @@ async def test_pam_auth_blocked(): assert authorized['name'] == 'wash' # Blacklist basics - authenticator = MockPAMAuthenticator(blocked={'wash'}) + authenticator = MockPAMAuthenticator(blocked_users={'wash'}) authorized = await authenticator.get_authenticated_user( None, {'username': 'wash', 'password': 'wash'} ) assert authorized is None # User in both allowed and blocked: default deny. Make error someday? - authenticator = MockPAMAuthenticator(blocked={'wash'}, allowed={'wash', 'kaylee'}) + authenticator = MockPAMAuthenticator( + blocked_users={'wash'}, allowed_users={'wash', 'kaylee'} + ) authorized = await authenticator.get_authenticated_user( None, {'username': 'wash', 'password': 'wash'} ) assert authorized is None # User not in blocked set can log in - authenticator = MockPAMAuthenticator(blocked={'wash'}, allowed={'wash', 'kaylee'}) + authenticator = MockPAMAuthenticator( + blocked_users={'wash'}, allowed_users={'wash', 'kaylee'} + ) authorized = await authenticator.get_authenticated_user( None, {'username': 'kaylee', 'password': 'kaylee'} ) assert authorized['name'] == 'kaylee' # User in allowed, blocked irrelevent - authenticator = MockPAMAuthenticator(blocked={'mal'}, allowed={'wash', 'kaylee'}) + authenticator = MockPAMAuthenticator( + blocked_users={'mal'}, allowed_users={'wash', 'kaylee'} + ) authorized = await authenticator.get_authenticated_user( None, {'username': 'wash', 'password': 'wash'} ) assert authorized['name'] == 'wash' # User in neither list - authenticator = MockPAMAuthenticator(blocked={'mal'}, allowed={'wash', 'kaylee'}) + authenticator = MockPAMAuthenticator( + blocked_users={'mal'}, allowed_users={'wash', 'kaylee'} + ) authorized = await authenticator.get_authenticated_user( None, {'username': 'simon', 'password': 'simon'} ) assert authorized is None - authenticator = MockPAMAuthenticator(blocked=set(), allowed={'wash', 'kaylee'}) + authenticator = MockPAMAuthenticator( + blocked_users=set(), allowed_users={'wash', 'kaylee'} + ) authorized = await authenticator.get_authenticated_user( None, {'username': 'kaylee', 'password': 'kaylee'} ) @@ -256,7 +266,7 @@ async def test_pam_auth_no_such_group(): async def test_wont_add_system_user(): user = orm.User(name='lioness4321') - authenticator = auth.PAMAuthenticator(allowed={'mal'}) + authenticator = auth.PAMAuthenticator(allowed_users={'mal'}) authenticator.create_system_users = False with pytest.raises(KeyError): await authenticator.add_user(user) @@ -264,7 +274,7 @@ async def test_wont_add_system_user(): async def test_cant_add_system_user(): user = orm.User(name='lioness4321') - authenticator = auth.PAMAuthenticator(allowed={'mal'}) + authenticator = auth.PAMAuthenticator(allowed_users={'mal'}) authenticator.add_user_cmd = ['jupyterhub-fake-command'] authenticator.create_system_users = True @@ -290,7 +300,7 @@ async def test_cant_add_system_user(): async def test_add_system_user(): user = orm.User(name='lioness4321') - authenticator = auth.PAMAuthenticator(allowed={'mal'}) + authenticator = auth.PAMAuthenticator(allowed_users={'mal'}) authenticator.create_system_users = True authenticator.add_user_cmd = ['echo', '/home/USERNAME'] @@ -311,13 +321,13 @@ async def test_add_system_user(): async def test_delete_user(): user = orm.User(name='zoe') - a = MockPAMAuthenticator(allowed={'mal'}) + a = MockPAMAuthenticator(allowed_users={'mal'}) - assert 'zoe' not in a.allowed + assert 'zoe' not in a.allowed_users await a.add_user(user) - assert 'zoe' in a.allowed + assert 'zoe' in a.allowed_users a.delete_user(user) - assert 'zoe' not in a.allowed + assert 'zoe' not in a.allowed_users def test_urls(): @@ -472,10 +482,10 @@ def test_deprecated_config(caplog): log.name, logging.WARNING, 'Authenticator.whitelist is deprecated in JupyterHub 1.2, use ' - 'Authenticator.allowed instead', + 'Authenticator.allowed_users instead', ) ] - assert authenticator.allowed == {'user'} + assert authenticator.allowed_users == {'user'} def test_deprecated_methods(): @@ -496,7 +506,7 @@ def test_deprecated_config_subclass(): cfg.MyAuthenticator.whitelist = {'user'} with pytest.deprecated_call(): authenticator = MyAuthenticator(config=cfg) - assert authenticator.allowed == {'user'} + assert authenticator.allowed_users == {'user'} def test_deprecated_methods_subclass(): From cceb65203973126b9d0168bd13e6f8700e554164 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 24 Jun 2020 20:19:44 +0200 Subject: [PATCH 084/156] TODO is TODONE Co-authored-by: Georgiana Elena --- jupyterhub/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyterhub/auth.py b/jupyterhub/auth.py index dfbfab79..48e02bda 100644 --- a/jupyterhub/auth.py +++ b/jupyterhub/auth.py @@ -297,7 +297,7 @@ class Authenticator(LoggingConfigurable): self._init_deprecated_methods() def _init_deprecated_methods(self): - # TODO: properly handle deprecated signature *and* name + # handles deprecated signature *and* name # with correct subclass override priority! for old_name, new_name in ( ('check_whitelist', 'check_allowed'), From 946ed844c5be53f03dbab741318537ffca5bcbfc Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Thu, 25 Jun 2020 19:41:46 +1200 Subject: [PATCH 085/156] Update jupyterhub/handlers/base.py Co-authored-by: Min RK --- jupyterhub/handlers/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyterhub/handlers/base.py b/jupyterhub/handlers/base.py index b08a6ac7..75d0697d 100644 --- a/jupyterhub/handlers/base.py +++ b/jupyterhub/handlers/base.py @@ -678,7 +678,7 @@ class BaseHandler(RequestHandler): a list with "next" (to avoid redirect-loops) :rtype (str) """ - if not exclude: + if exclude is None: exclude = ['next'] if self.request.query: query_string = [ From ef4455bb677de153695b687e0dab5d4424c58742 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Wed, 24 Jun 2020 16:01:08 +1200 Subject: [PATCH 086/156] Closes #2182 display hamburger menu only if user variable is present (in responsive mode) --- share/jupyterhub/templates/page.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/share/jupyterhub/templates/page.html b/share/jupyterhub/templates/page.html index bc701ce4..5953ac1c 100644 --- a/share/jupyterhub/templates/page.html +++ b/share/jupyterhub/templates/page.html @@ -106,12 +106,14 @@ {% endblock %} + {% if user %} + {% endif %}
    -
    -
    - JupyterHub {{ server_version }} -
    -
    {% call modal('Delete User', btn_class='btn-danger delete-button') %} Are you sure you want to delete user USER? @@ -175,6 +170,14 @@ {% endblock %} +{% block footer %} +
    +
    + JupyterHub {{ server_version }} +
    +
    +{% endblock %} + {% block script %} {{ super() }}