hazza/jupyterhub
1
0
Fork 0
mirror of https://github.com/jupyterhub/jupyterhub.git synced 2025-10-15 22:13:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #886 from yuvipanda/spawner-docs

Browse Source
This commit is contained in:
Min RK
2016-11-30 13:48:05 +01:00
parent ba3a8f2e76 2152a94156
commit d5a6e2b2ac
2 changed files with 502 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

329
jupyterhub/auth.py
View File

@@ -21,88 +21,117 @@ from .handlers.login import LoginHandler
from .utils import url_path_join from .utils import url_path_join
from .traitlets import Command from .traitlets import Command
class Authenticator(LoggingConfigurable):
"""A class for authentication.
The primary API is one method, `authenticate`, a tornado coroutine
for authenticating users.
"""
db = Any()
admin_users = Set(
help="""set of usernames of admin users
If unspecified, only the user that launches the server will be admin. class Authenticator(LoggingConfigurable):
"""Base class for implementing an authentication provider for JupyterHub"""
db = Any()
admin_users = Set(
help="""
Set of users that will have admin rights on this JupyterHub.
Admin users have extra privilages:
- Use the admin panel to see list of users logged in
- Add / remove users in some authenticators
- Restart / halt the hub
- Start / stop users' single-user servers
- Can access each individual users' single-user server (if configured)
Admin access should be treated the same way root access is.
Defaults to an empty set, in which case no user has admin access.
""" """
).tag(config=True) ).tag(config=True)
whitelist = Set( whitelist = Set(
help="""Username whitelist. help="""
Whitelist of usernames that are allowed to log in.
Use this to restrict which users can login.
If empty, allow any user to attempt login. Use this with supported authenticators to restrict which users can log in. This is an
additional whitelist that further restricts users, beyond whatever restrictions the
authenticator has in place.
If empty, does not perform any additional restriction.
""" """
).tag(config=True) ).tag(config=True)
custom_html = Unicode('',
help="""HTML login form for custom handlers. custom_html = Unicode(
Override in form-based custom authenticators help="""
that don't use username+password, HTML form to be overridden by authenticators if they want a custom authentication form.
or need custom branding.
Defaults to an empty string, which shows the default username/password form.
""" """
) )
login_service = Unicode('',
help="""Name of the login service for external login_service = Unicode(
login services (e.g. 'GitHub'). help="""
Name of the login service that this authenticator is providing using to authenticate users.
Example: GitHub, MediaWiki, Google, etc.
Setting this value replaces the login form with a "Login with <login_service>" button.
Any authenticator that redirects to an external service (e.g. using OAuth) should set this.
""" """
) )
username_pattern = Unicode( username_pattern = Unicode(
help="""Regular expression pattern for validating usernames. help="""
Regular expression pattern that all valid usernames must match.
If not defined: allow any username.
If a username does not match the pattern specified here, authentication will not be attempted.
If not set, allow any username.
""" """
).tag(config=True) ).tag(config=True)
@observe('username_pattern') @observe('username_pattern')
def _username_pattern_changed(self, change): def _username_pattern_changed(self, change):
if not change['new']: if not change['new']:
self.username_regex = None self.username_regex = None
self.username_regex = re.compile(change['new']) self.username_regex = re.compile(change['new'])
username_regex = Any() username_regex = Any(
help="""
Compiled regex kept in sync with `username_pattern`
"""
)
def validate_username(self, username): def validate_username(self, username):
"""Validate a (normalized) username. """Validate a normalized username
Return True if username is valid, False otherwise. Return True if username is valid, False otherwise.
""" """
if not self.username_regex: if not self.username_regex:
return True return True
return bool(self.username_regex.match(username)) return bool(self.username_regex.match(username))
username_map = Dict( username_map = Dict(
help="""Dictionary mapping authenticator usernames to JupyterHub users. help="""Dictionary mapping authenticator usernames to JupyterHub users.
Can be used to map OAuth service names to local users, for instance. Primarily used to normalize OAuth user names to local users.
Used in normalize_username.
""" """
).tag(config=True) ).tag(config=True)
def normalize_username(self, username): def normalize_username(self, username):
"""Normalize a username. """Normalize the given username and return it
Override in subclasses if usernames should have some normalization. Override in subclasses if usernames need different normalization rules.
Default: cast to lowercase, lookup in username_map.
The default attempts to lowercase the username and apply `username_map` if it is
set.
""" """
username = username.lower() username = username.lower()
username = self.username_map.get(username, username) username = self.username_map.get(username, username)
return username return username
def check_whitelist(self, username): def check_whitelist(self, username):
"""Check a username against our whitelist. """Check if a username is allowed to authenticate based on whitelist configuration
Return True if username is allowed, False otherwise. Return True if username is allowed, False otherwise.
No whitelist means any username should be allowed. No whitelist means any username is allowed.
Names are normalized *before* being checked against the whitelist. Names are normalized *before* being checked against the whitelist.
""" """
if not self.whitelist: if not self.whitelist:
@@ -112,18 +141,21 @@ class Authenticator(LoggingConfigurable):
@gen.coroutine @gen.coroutine
def get_authenticated_user(self, handler, data): def get_authenticated_user(self, handler, data):
"""This is the outer API for authenticating a user. """Authenticate the user who is attempting to log in
Returns normalized username if successful, None otherwise.
This calls `authenticate`, which should be overridden in subclasses, This calls `authenticate`, which should be overridden in subclasses,
normalizes the username if any normalization should be done, normalizes the username if any normalization should be done,
and then validates the name in the whitelist. and then validates the name in the whitelist.
This is the outer API for authenticating a user.
Subclasses should not need to override this method. Subclasses should not need to override this method.
The various stages can be overridden separately: The various stages can be overridden separately:
- `authenticate` turns formdata into a username
- authenticate turns formdata into a username - `normalize_username` normalizes the username
- normalize_username normalizes the username - `check_whitelist` checks against the user whitelist
- check_whitelist checks against the user whitelist
""" """
username = yield self.authenticate(handler, data) username = yield self.authenticate(handler, data)
if username is None: if username is None:
@@ -139,47 +171,53 @@ class Authenticator(LoggingConfigurable):
else: else:
self.log.warning("User %r not in whitelist.", username) self.log.warning("User %r not in whitelist.", username)
return return
@gen.coroutine @gen.coroutine
def authenticate(self, handler, data): def authenticate(self, handler, data):
"""Authenticate a user with login form data. """Authenticate a user with login form data
This must be a tornado gen.coroutine. This must be a tornado gen.coroutine.
It must return the username on successful authentication, It must return the username on successful authentication,
and return None on failed authentication. and return None on failed authentication.
Checking the whitelist is handled separately by the caller. Checking the whitelist is handled separately by the caller.
Args: Args:
handler (tornado.web.RequestHandler): the current request handler handler (tornado.web.RequestHandler): the current request handler
data (dict): The formdata of the login form. data (dict): The formdata of the login form.
The default form has 'username' and 'password' fields. The default form has 'username' and 'password' fields.
Return: Returns:
str: the username of the authenticated user username (str or None): The username of the authenticated user,
None: Authentication failed or None if Authentication failed
""" """
def pre_spawn_start(self, user, spawner): def pre_spawn_start(self, user, spawner):
"""Hook called before spawning a user's server. """Hook called before spawning a user's server
Can be used to do auth-related startup, e.g. opening PAM sessions. Can be used to do auth-related startup, e.g. opening PAM sessions.
""" """
def post_spawn_stop(self, user, spawner): def post_spawn_stop(self, user, spawner):
"""Hook called after stopping a user container. """Hook called after stopping a user container
Can be used to do auth-related cleanup, e.g. closing PAM sessions. Can be used to do auth-related cleanup, e.g. closing PAM sessions.
""" """
def add_user(self, user): def add_user(self, user):
"""Add a new user """Hook called when a user is added to JupyterHub
This is called:
- When a user first authenticates
- When the hub restarts, for all users.
By default, this just adds the user to the whitelist. By default, this just adds the user to the whitelist.
Subclasses may do more extensive things, Subclasses may do more extensive things, such as adding actual unix users,
such as adding actual unix users,
but they should call super to ensure the whitelist is updated. but they should call super to ensure the whitelist is updated.
Note that this should be idempotent, since it is called whenever the hub restarts
for all users.
Args: Args:
user (User): The User wrapper object user (User): The User wrapper object
""" """
@@ -187,97 +225,108 @@ class Authenticator(LoggingConfigurable):
raise ValueError("Invalid username: %s" % user.name) raise ValueError("Invalid username: %s" % user.name)
if self.whitelist: if self.whitelist:
self.whitelist.add(user.name) self.whitelist.add(user.name)
def delete_user(self, user): def delete_user(self, user):
"""Triggered when a user is deleted. """Hook called when a user is deleted
Removes the user from the whitelist. Removes the user from the whitelist.
Subclasses should call super to ensure the whitelist is updated. Subclasses should call super to ensure the whitelist is updated.
Args: Args:
user (User): The User wrapper object user (User): The User wrapper object
""" """
self.whitelist.discard(user.name) self.whitelist.discard(user.name)
def login_url(self, base_url): def login_url(self, base_url):
"""Override to register a custom login handler """Override this when registering a custom login handler
Generally used in combination with get_handlers. Generally used by authenticators that do not use simple form based authentication.
The subclass overriding this is responsible for making sure there is a handler
available to handle the URL returned from this method, using the `get_handlers`
method.
Args: Args:
base_url (str): the base URL of the Hub (e.g. /hub/) base_url (str): the base URL of the Hub (e.g. /hub/)
Returns: Returns:
str: The login URL, e.g. '/hub/login' str: The login URL, e.g. '/hub/login'
""" """
return url_path_join(base_url, 'login') return url_path_join(base_url, 'login')
def logout_url(self, base_url): def logout_url(self, base_url):
"""Override to register a custom logout handler. """Override when registering a custom logout handler
Generally used in combination with get_handlers. The subclass overriding this is responsible for making sure there is a handler
available to handle the URL returned from this method, using the `get_handlers`
method.
Args: Args:
base_url (str): the base URL of the Hub (e.g. /hub/) base_url (str): the base URL of the Hub (e.g. /hub/)
Returns: Returns:
str: The logout URL, e.g. '/hub/logout' str: The logout URL, e.g. '/hub/logout'
""" """
return url_path_join(base_url, 'logout') return url_path_join(base_url, 'logout')
def get_handlers(self, app): def get_handlers(self, app):
"""Return any custom handlers the authenticator needs to register """Return any custom handlers the authenticator needs to register
(e.g. for OAuth). Used in conjugation with `login_url` and `logout_url`.
Args: Args:
app (JupyterHub Application): app (JupyterHub Application):
the application object, in case it needs to be accessed for info. the application object, in case it needs to be accessed for info.
Returns: Returns:
list: list of ``('/url', Handler)`` tuples passed to tornado. handlers (list):
list of ``('/url', Handler)`` tuples passed to tornado.
The Hub prefix is added to any URLs. The Hub prefix is added to any URLs.
""" """
return [ return [
('/login', LoginHandler), ('/login', LoginHandler),
] ]
class LocalAuthenticator(Authenticator): class LocalAuthenticator(Authenticator):
"""Base class for Authenticators that work with local Linux/UNIX users """Base class for Authenticators that work with local Linux/UNIX users
Checks for local users, and can attempt to create them if they exist. Checks for local users, and can attempt to create them if they exist.
""" """
create_system_users = Bool(False, create_system_users = Bool(False,
help="""If a user is added that doesn't exist on the system, help="""
should I try to create the system user? If set to True, will attempt to create local system users if they do not exist already.
Supports Linux and BSD variants only.
""" """
).tag(config=True) ).tag(config=True)
add_user_cmd = Command( add_user_cmd = Command(
help="""The command to use for creating users as a list of strings. help="""
The command to use for creating users as a list of strings
For each element in the list, the string USERNAME will be replaced with For each element in the list, the string USERNAME will be replaced with
the user's username. The username will also be appended as the final argument. the user's username. The username will also be appended as the final argument.
For Linux, the default value is: For Linux, the default value is:
['adduser', '-q', '--gecos', '""', '--disabled-password'] ['adduser', '-q', '--gecos', '""', '--disabled-password']
To specify a custom home directory, set this to: To specify a custom home directory, set this to:
['adduser', '-q', '--gecos', '""', '--home', '/customhome/USERNAME', '--disabled-password'] ['adduser', '-q', '--gecos', '""', '--home', '/customhome/USERNAME', '--disabled-password']
This will run the command: This will run the command:
adduser -q --gecos "" --home /customhome/river --disabled-password river adduser -q --gecos "" --home /customhome/river --disabled-password river
when the user 'river' is created. when the user 'river' is created.
""" """
).tag(config=True) ).tag(config=True)
@default('add_user_cmd') @default('add_user_cmd')
def _add_user_cmd_default(self): def _add_user_cmd_default(self):
"""Guess the most likely-to-work adduser command for each platform"""
if sys.platform == 'darwin': if sys.platform == 'darwin':
raise ValueError("I don't know how to create users on OS X") raise ValueError("I don't know how to create users on OS X")
elif which('pw'): elif which('pw'):
@@ -288,10 +337,18 @@ class LocalAuthenticator(Authenticator):
return ['adduser', '-q', '--gecos', '""', '--disabled-password'] return ['adduser', '-q', '--gecos', '""', '--disabled-password']
group_whitelist = Set( group_whitelist = Set(
help="Automatically whitelist anyone in this group.", help="""
Whitelist all users from this UNIX group.
This makes the username whitelist ineffective.
"""
).tag(config=True) ).tag(config=True)
@observe('group_whitelist') @observe('group_whitelist')
def _group_whitelist_changed(self, change): def _group_whitelist_changed(self, change):
"""
Log a warning if both group_whitelist and user whitelist are set.
"""
if self.whitelist: if self.whitelist:
self.log.warning( self.log.warning(
"Ignoring username whitelist because group whitelist supplied!" "Ignoring username whitelist because group whitelist supplied!"
@@ -304,6 +361,9 @@ class LocalAuthenticator(Authenticator):
return super().check_whitelist(username) return super().check_whitelist(username)
def check_group_whitelist(self, username): def check_group_whitelist(self, username):
"""
If group_whitelist is configured, check if authenticating user is part of group.
"""
if not self.group_whitelist: if not self.group_whitelist:
return False return False
for grnam in self.group_whitelist: for grnam in self.group_whitelist:
@@ -318,9 +378,9 @@ class LocalAuthenticator(Authenticator):
@gen.coroutine @gen.coroutine
def add_user(self, user): def add_user(self, user):
"""Add a new user """Hook called whenever a new user is added
If self.create_system_users, the user will attempt to be created. If self.create_system_users, the user will attempt to be created if it doesn't exist.
""" """
user_exists = yield gen.maybe_future(self.system_user_exists(user)) user_exists = yield gen.maybe_future(self.system_user_exists(user))
if not user_exists: if not user_exists:
@@ -328,9 +388,9 @@ class LocalAuthenticator(Authenticator):
yield gen.maybe_future(self.add_system_user(user)) yield gen.maybe_future(self.add_system_user(user))
else: else:
raise KeyError("User %s does not exist." % user.name) raise KeyError("User %s does not exist." % user.name)
yield gen.maybe_future(super().add_user(user)) yield gen.maybe_future(super().add_user(user))
@staticmethod @staticmethod
def system_user_exists(user): def system_user_exists(user):
"""Check if the user exists on the system""" """Check if the user exists on the system"""
@@ -342,7 +402,10 @@ class LocalAuthenticator(Authenticator):
return True return True
def add_system_user(self, user): def add_system_user(self, user):
"""Create a new Linux/UNIX user on the system. Works on FreeBSD and Linux, at least.""" """Create a new local UNIX user on the system.
Tested to work on FreeBSD and Linux, at least.
"""
name = user.name name = user.name
cmd = [ arg.replace('USERNAME', name) for arg in self.add_user_cmd ] + [name] cmd = [ arg.replace('USERNAME', name) for arg in self.add_user_cmd ] + [name]
self.log.info("Creating user: %s", ' '.join(map(pipes.quote, cmd))) self.log.info("Creating user: %s", ' '.join(map(pipes.quote, cmd)))
@@ -354,30 +417,37 @@ class LocalAuthenticator(Authenticator):
class PAMAuthenticator(LocalAuthenticator): class PAMAuthenticator(LocalAuthenticator):
"""Authenticate local Linux/UNIX users with PAM""" """Authenticate local UNIX users with PAM"""
encoding = Unicode('utf8', encoding = Unicode('utf8',
help="""The encoding to use for PAM""" help="""
The text encoding to use when communicating with PAM
"""
).tag(config=True) ).tag(config=True)
service = Unicode('login', service = Unicode('login',
help="""The PAM service to use for authentication.""" help="""
The name of the PAM service to use for authentication
"""
).tag(config=True) ).tag(config=True)
open_sessions = Bool(True, open_sessions = Bool(True,
help="""Whether to open PAM sessions when spawners are started. help="""
Whether to open a new PAM session when spawners are started.
This may trigger things like mounting shared filsystems, This may trigger things like mounting shared filsystems,
loading credentials, etc. depending on system configuration, loading credentials, etc. depending on system configuration,
but it does not always work. but it does not always work.
It can be disabled with:: If any errors are encountered when opening/closing PAM sessions,
this is automatically set to False.
c.PAMAuthenticator.open_sessions = False
""" """
).tag(config=True) ).tag(config=True)
@gen.coroutine @gen.coroutine
def authenticate(self, handler, data): def authenticate(self, handler, data):
"""Authenticate with PAM, and return the username if login is successful. """Authenticate with PAM, and return the username if login is successful.
Return None otherwise. Return None otherwise.
""" """
username = data['username'] username = data['username']
@@ -390,9 +460,9 @@ class PAMAuthenticator(LocalAuthenticator):
self.log.warning("PAM Authentication failed: %s", e) self.log.warning("PAM Authentication failed: %s", e)
else: else:
return username return username
def pre_spawn_start(self, user, spawner): def pre_spawn_start(self, user, spawner):
"""Open PAM session for user""" """Open PAM session for user if so configured"""
if not self.open_sessions: if not self.open_sessions:
return return
try: try:
@@ -401,9 +471,9 @@ class PAMAuthenticator(LocalAuthenticator):
self.log.warning("Failed to open PAM session for %s: %s", user.name, e) self.log.warning("Failed to open PAM session for %s: %s", user.name, e)
self.log.warning("Disabling PAM sessions from now on.") self.log.warning("Disabling PAM sessions from now on.")
self.open_sessions = False self.open_sessions = False
def post_spawn_stop(self, user, spawner): def post_spawn_stop(self, user, spawner):
"""Close PAM session for user""" """Close PAM session for user if we were configured to opened one"""
if not self.open_sessions: if not self.open_sessions:
return return
try: try:
@@ -412,4 +482,3 @@ class PAMAuthenticator(LocalAuthenticator):
self.log.warning("Failed to close PAM session for %s: %s", user.name, e) self.log.warning("Failed to close PAM session for %s: %s", user.name, e)
self.log.warning("Disabling PAM sessions from now on.") self.log.warning("Disabling PAM sessions from now on.")
self.open_sessions = False self.open_sessions = False

461
jupyterhub/spawner.py
View File

@@ -1,4 +1,6 @@
"""Class for spawning single-user notebook servers.""" """
Contains base Spawner class & default implementation
"""
# Copyright (c) Jupyter Development Team. # Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License. # Distributed under the terms of the Modified BSD License.
@@ -27,32 +29,51 @@ from traitlets import (
from .traitlets import Command, ByteSpecification from .traitlets import Command, ByteSpecification
from .utils import random_port from .utils import random_port
class Spawner(LoggingConfigurable): class Spawner(LoggingConfigurable):
"""Base class for spawning single-user notebook servers. """Base class for spawning single-user notebook servers.
Subclass this, and override the following methods: Subclass this, and override the following methods:
- load_state - load_state
- get_state - get_state
- start - start
- stop - stop
- poll - poll
As JupyterHub supports multiple users, an instance of the Spawner subclass
is created for each user. If there are 20 JupyterHub users, there will be 20
instances of the subclass.
""" """
db = Any() db = Any()
user = Any() user = Any()
hub = Any() hub = Any()
authenticator = Any() authenticator = Any()
api_token = Unicode() api_token = Unicode()
ip = Unicode('127.0.0.1', ip = Unicode('127.0.0.1',
help="The IP address (or hostname) the single-user server should listen on" help="""
The IP address (or hostname) the single-user server should listen on.
The JupyterHub proxy implementation should be able to send packets to this interface.
"""
).tag(config=True) ).tag(config=True)
port = Integer(0, port = Integer(0,
help="The port for single-user servers to listen on. New in version 0.7." help="""
The port for single-user servers to listen on.
Defaults to `0`, which uses a randomly allocated port number each time.
New in version 0.7.
"""
) )
start_timeout = Integer(60, start_timeout = Integer(60,
help="""Timeout (in seconds) before giving up on the spawner. help="""
Timeout (in seconds) before giving up on starting of single-user server.
This is the timeout for start to return, not the timeout for the server to respond. This is the timeout for start to return, not the timeout for the server to respond.
Callers of spawner.start will assume that startup has failed if it takes longer than this. Callers of spawner.start will assume that startup has failed if it takes longer than this.
start should return when the server process is started and its location is known. start should return when the server process is started and its location is known.
@@ -60,7 +81,8 @@ class Spawner(LoggingConfigurable):
).tag(config=True) ).tag(config=True)
http_timeout = Integer(30, http_timeout = Integer(30,
help="""Timeout (in seconds) before giving up on a spawned HTTP server help="""
Timeout (in seconds) before giving up on a spawned HTTP server
Once a server has successfully been spawned, this is the amount of time Once a server has successfully been spawned, this is the amount of time
we wait before assuming that the server is unable to accept we wait before assuming that the server is unable to accept
@@ -69,21 +91,30 @@ class Spawner(LoggingConfigurable):
).tag(config=True) ).tag(config=True)
poll_interval = Integer(30, poll_interval = Integer(30,
help="""Interval (in seconds) on which to poll the spawner.""" help="""
Interval (in seconds) on which to poll the spawner for single-user server's status.
At every poll interval, each spawner's `.poll` method is called, which checks
if the single-user server is still running. If it isn't running, then JupyterHub modifies
its own state accordingly and removes appropriate routes from the configurable proxy.
"""
).tag(config=True) ).tag(config=True)
_callbacks = List() _callbacks = List()
_poll_callback = Any() _poll_callback = Any()
debug = Bool(False, debug = Bool(False,
help="Enable debug-logging of the single-user server" help="Enable debug-logging of the single-user server"
).tag(config=True) ).tag(config=True)
options_form = Unicode("", help=""" options_form = Unicode(
help="""
An HTML form for options a user can specify on launching their server. An HTML form for options a user can specify on launching their server.
The surrounding `<form>` element and the submit button are already provided. The surrounding `<form>` element and the submit button are already provided.
For example: For example:
Set your key: Set your key:
<input name="key" val="default_key"></input> <input name="key" val="default_key"></input>
<br> <br>
@@ -92,6 +123,8 @@ class Spawner(LoggingConfigurable):
<option value="A">The letter A</option> <option value="A">The letter A</option>
<option value="B">The letter B</option> <option value="B">The letter B</option>
</select> </select>
The data from this form submission will be passed on to your spawner in `self.user_options`
""").tag(config=True) """).tag(config=True)
def options_from_form(self, form_data): def options_from_form(self, form_data):
@@ -107,9 +140,15 @@ class Spawner(LoggingConfigurable):
prior to `Spawner.start`. prior to `Spawner.start`.
""" """
return form_data return form_data
user_options = Dict(help="This is where form-specified options ultimately end up.") user_options = Dict(
help="""
Dict of user specified options for the user's spawned instance of a single-user server.
These user options are usually provided by the `options_form` displayed to the user when they start
their server.
""")
env_keep = List([ env_keep = List([
'PATH', 'PATH',
'PYTHONPATH', 'PYTHONPATH',
@@ -119,50 +158,93 @@ class Spawner(LoggingConfigurable):
'LANG', 'LANG',
'LC_ALL', 'LC_ALL',
], ],
help="Whitelist of environment variables for the subprocess to inherit" help="""
Whitelist of environment variables for the single-user server to inherit from the JupyterHub process.
This whitelist is used to ensure that sensitive information in the JupyterHub process's environment
(such as `CONFIGPROXY_AUTH_TOKEN`) is not passed to the single-user server's process.
"""
).tag(config=True) ).tag(config=True)
env = Dict(help="""Deprecated: use Spawner.get_env or Spawner.environment env = Dict(help="""Deprecated: use Spawner.get_env or Spawner.environment
- extend Spawner.get_env for adding required env in Spawner subclasses - extend Spawner.get_env for adding required env in Spawner subclasses
- Spawner.environment for config-specified env - Spawner.environment for config-specified env
""") """)
environment = Dict( environment = Dict(
help="""Environment variables to load for the Spawner. help="""
Extra environment variables to set for the single-user server's process.
Value could be a string or a callable. If it is a callable, it will Environment variables that end up in the single-user server's process come from 3 sources:
be called with one parameter, which will be the instance of the spawner - This `environment` configurable
in use. It should quickly (without doing much blocking operations) return - The JupyterHub process' environment variables that are whitelisted in `env_keep`
a string that will be used as the value for the environment variable. - Variables to establish contact between the single-user notebook and the hub (such as JPY_API_TOKEN)
The `enviornment` configurable should be set by JupyterHub administrators to add
installation specific environment variables. It is a dict where the key is the name of the environment
variable, and the value can be a string or a callable. If it is a callable, it will be called
with one parameter (the spawner instance), and should return a string fairly quickly (no blocking
operations please!).
Note that the spawner class' interface is not guaranteed to be exactly same across upgrades,
so if you are using the callable take care to verify it continues to work after upgrades!
""" """
).tag(config=True) ).tag(config=True)
cmd = Command(['jupyterhub-singleuser'], cmd = Command(['jupyterhub-singleuser'],
help="""The command used for starting notebooks.""" help="""
The command used for starting the single-user server.
Provide either a string or a list containing the path to the startup script command. Extra arguments,
other than this path, should be provided via `args`.
This is usually set if you want to start the single-user server in a different python
environment (with virtualenv/conda) than JupyterHub itself.
Some spawners allow shell-style expansion here, allowing you to use environment variables.
Most, including the default, do not. Consult the documentation for your spawner to verify!
"""
).tag(config=True) ).tag(config=True)
args = List(Unicode(), args = List(Unicode(),
help="""Extra arguments to be passed to the single-user server""" help="""
).tag(config=True) Extra arguments to be passed to the single-user server.
notebook_dir = Unicode('', Some spawners allow shell-style expansion here, allowing you to use environment variables here.
help="""The notebook directory for the single-user server Most, including the default, do not. Consult the documentation for your spawner to verify!
`~` will be expanded to the user's home directory
`{username}` will be expanded to the user's username
""" """
).tag(config=True) ).tag(config=True)
default_url = Unicode('', notebook_dir = Unicode(
help="""The default URL for the single-user server. help="""
Path to the notebook directory for the single-user server.
Can be used in conjunction with --notebook-dir=/ to enable The user sees a file listing of this directory when the notebook interface is started. The
full filesystem traversal, while preserving user's homedir as current interface does not easily allow browsing beyond the subdirectories in this directory's
landing page for notebook tree.
`{username}` will be expanded to the user's username `~` will be expanded to the home directory of the user, and {username} will be replaced
with the name of the user.
Note that this does *not* prevent users from accessing files outside of this path! They
can do so with many other means.
""" """
).tag(config=True) ).tag(config=True)
default_url = Unicode(
help="""
The URL the single-user server should start in.
`{username}` will be expanded to the user's username
Example uses:
- You can set `notebook_dir` to `/` and `default_url` to `/home/{username}` to allow people to
navigate the whole filesystem from their notebook, but still start in their home directory.
- You can set this to `/lab` to have JupyterLab start by default, rather than Jupyter Notebook.
"""
).tag(config=True)
@validate('notebook_dir', 'default_url') @validate('notebook_dir', 'default_url')
def _deprecate_percent_u(self, proposal): def _deprecate_percent_u(self, proposal):
print(proposal) print(proposal)
@@ -174,17 +256,20 @@ class Spawner(LoggingConfigurable):
v = v.replace('%U', '{username}') v = v.replace('%U', '{username}')
self.log.warning("Converting %r to %r", proposal['value'], v) self.log.warning("Converting %r to %r", proposal['value'], v)
return v return v
disable_user_config = Bool(False, disable_user_config = Bool(False,
help="""Disable per-user configuration of single-user servers. help="""
Disable per-user configuration of single-user servers.
This prevents any config in users' $HOME directories
from having an effect on their server. When starting the user's single-user server, any config file found in the user's $HOME directory
will be ignored.
Note: a user could circumvent this if the user modifies their Python environment, such as when
they have their own conda environments / virtualenvs / containers.
""" """
).tag(config=True) ).tag(config=True)
mem_limit = ByteSpecification( mem_limit = ByteSpecification(None,
None,
help=""" help="""
Maximum number of bytes a single-user notebook server is allowed to use. Maximum number of bytes a single-user notebook server is allowed to use.
@@ -203,8 +288,7 @@ class Spawner(LoggingConfigurable):
""" """
).tag(config=True) ).tag(config=True)
cpu_limit = Float( cpu_limit = Float(None,
None,
allow_none=True, allow_none=True,
help=""" help="""
Maximum number of cpu-cores a single-user notebook server is allowed to use. Maximum number of cpu-cores a single-user notebook server is allowed to use.
@@ -220,8 +304,7 @@ class Spawner(LoggingConfigurable):
""" """
).tag(config=True) ).tag(config=True)
mem_guarantee = ByteSpecification( mem_guarantee = ByteSpecification(None,
None,
help=""" help="""
Minimum number of bytes a single-user notebook server is guaranteed to have available. Minimum number of bytes a single-user notebook server is guaranteed to have available.
@@ -235,8 +318,7 @@ class Spawner(LoggingConfigurable):
""" """
).tag(config=True) ).tag(config=True)
cpu_guarantee = Float( cpu_guarantee = Float(None,
None,
allow_none=True, allow_none=True,
help=""" help="""
Minimum number of cpu-cores a single-user notebook server is guaranteed to have available. Minimum number of cpu-cores a single-user notebook server is guaranteed to have available.
@@ -252,59 +334,62 @@ class Spawner(LoggingConfigurable):
super(Spawner, self).__init__(**kwargs) super(Spawner, self).__init__(**kwargs)
if self.user.state: if self.user.state:
self.load_state(self.user.state) self.load_state(self.user.state)
def load_state(self, state): def load_state(self, state):
"""load state from the database """Restore state of spawner from database.
This is the extensible part of state. Called for each user's spawner after the hub process restarts.
Override in a subclass if there is state to load. `state` is a dict that'll contain the value returned by `get_state` of
Should call `super`. the spawner, or {} if the spawner hasn't persisted any state yet.
See Also Override in subclasses to restore any extra state that is needed to track
-------- the single-user server for that user. Subclasses should call super().
get_state, clear_state
""" """
pass pass
def get_state(self): def get_state(self):
"""store the state necessary for load_state """Save state of spawner into database.
A black box of extra state for custom spawners. A black box of extra state for custom spawners. The returned value of this is
Subclasses should call `super`. passed to `load_state`.
Subclasses should call `super().get_state()`, augment the state returned from
there, and return that state.
Returns Returns
------- -------
state: dict state: dict
a JSONable dict of state a JSONable dict of state
""" """
state = {} state = {}
return state return state
def clear_state(self): def clear_state(self):
"""clear any state that should be cleared when the process stops """Clear any state that should be cleared when the single-user server stops.
State that should be preserved across server instances should not be cleared. State that should be preserved across single-user server instances should not be cleared.
Subclasses should call super, to ensure that state is properly cleared. Subclasses should call super, to ensure that state is properly cleared.
""" """
self.api_token = '' self.api_token = ''
def get_env(self): def get_env(self):
"""Return the environment dict to use for the Spawner. """Return the environment dict to use for the Spawner.
This applies things like `env_keep`, anything defined in `Spawner.environment`, This applies things like `env_keep`, anything defined in `Spawner.environment`,
and adds the API token to the env. and adds the API token to the env.
When overriding in subclasses, subclasses must call `super().get_env()`, extend the
returned dict and return it.
Use this to access the env in Spawner.start to allow extension in subclasses. Use this to access the env in Spawner.start to allow extension in subclasses.
""" """
env = {} env = {}
if self.env: if self.env:
warnings.warn("Spawner.env is deprecated, found %s" % self.env, DeprecationWarning) warnings.warn("Spawner.env is deprecated, found %s" % self.env, DeprecationWarning)
env.update(self.env) env.update(self.env)
for key in self.env_keep: for key in self.env_keep:
if key in os.environ: if key in os.environ:
env[key] = os.environ[key] env[key] = os.environ[key]
@@ -375,7 +460,10 @@ class Spawner(LoggingConfigurable):
return s.format(**self.template_namespace()) return s.format(**self.template_namespace())
def get_args(self): def get_args(self):
"""Return the arguments to be passed after self.cmd""" """Return the arguments to be passed after self.cmd
Doesn't expect shell expansion to happen.
"""
args = [ args = [
'--user="%s"' % self.user.name, '--user="%s"' % self.user.name,
'--cookie-name="%s"' % self.user.server.cookie_name, '--cookie-name="%s"' % self.user.server.cookie_name,
@@ -406,33 +494,37 @@ class Spawner(LoggingConfigurable):
args.append('--disable-user-config') args.append('--disable-user-config')
args.extend(self.args) args.extend(self.args)
return args return args
@gen.coroutine @gen.coroutine
def start(self): def start(self):
"""Start the single-user server """Start the single-user server
Returns: Returns:
(str, int): the (ip, port) where the Hub can connect to the server.
(ip, port): the ip, port where the Hub can connect to the server.
.. versionchanged:: 0.7 .. versionchanged:: 0.7
Return ip, port instead of setting on self.user.server directly. Return ip, port instead of setting on self.user.server directly.
""" """
raise NotImplementedError("Override in subclass. Must be a Tornado gen.coroutine.") raise NotImplementedError("Override in subclass. Must be a Tornado gen.coroutine.")
@gen.coroutine @gen.coroutine
def stop(self, now=False): def stop(self, now=False):
"""Stop the single-user process""" """Stop the single-user server
If `now` is set to `False`, do not wait for the server to stop. Otherwise, wait for
the server to stop before returning.
Must be a Tornado coroutine.
"""
raise NotImplementedError("Override in subclass. Must be a Tornado gen.coroutine.") raise NotImplementedError("Override in subclass. Must be a Tornado gen.coroutine.")
@gen.coroutine @gen.coroutine
def poll(self): def poll(self):
"""Check if the single-user process is running """Check if the single-user process is running
returns: Returns:
None if single-user process is running.
None, if single-user process is running. Integer exit status (0 if unknown), if it is not running.
Exit status (0 if unknown), if it is not running.
State transitions, behavior, and return response: State transitions, behavior, and return response:
@@ -452,29 +544,24 @@ class Spawner(LoggingConfigurable):
""" """
raise NotImplementedError("Override in subclass. Must be a Tornado gen.coroutine.") raise NotImplementedError("Override in subclass. Must be a Tornado gen.coroutine.")
def add_poll_callback(self, callback, *args, **kwargs): def add_poll_callback(self, callback, *args, **kwargs):
"""add a callback to fire when the subprocess stops """Add a callback to fire when the single-user server stops"""
as noticed by periodic poll_and_notify()
"""
if args or kwargs: if args or kwargs:
cb = callback cb = callback
callback = lambda : cb(*args, **kwargs) callback = lambda : cb(*args, **kwargs)
self._callbacks.append(callback) self._callbacks.append(callback)
def stop_polling(self): def stop_polling(self):
"""stop the periodic poll""" """Stop polling for single-user server's running state"""
if self._poll_callback: if self._poll_callback:
self._poll_callback.stop() self._poll_callback.stop()
self._poll_callback = None self._poll_callback = None
def start_polling(self): def start_polling(self):
"""Start polling periodically """Start polling periodically for single-user server's running state.
callbacks registered via `add_poll_callback` will fire Callbacks registered via `add_poll_callback` will fire if/when the server stops.
if/when the process stops.
Explicit termination via the stop method will not trigger the callbacks. Explicit termination via the stop method will not trigger the callbacks.
""" """
if self.poll_interval <= 0: if self.poll_interval <= 0:
@@ -482,9 +569,9 @@ class Spawner(LoggingConfigurable):
return return
else: else:
self.log.debug("Polling subprocess every %is", self.poll_interval) self.log.debug("Polling subprocess every %is", self.poll_interval)
self.stop_polling() self.stop_polling()
self._poll_callback = PeriodicCallback( self._poll_callback = PeriodicCallback(
self.poll_and_notify, self.poll_and_notify,
1e3 * self.poll_interval 1e3 * self.poll_interval
@@ -493,27 +580,25 @@ class Spawner(LoggingConfigurable):
@gen.coroutine @gen.coroutine
def poll_and_notify(self): def poll_and_notify(self):
"""Used as a callback to periodically poll the process, """Used as a callback to periodically poll the process and notify any watchers"""
and notify any watchers
"""
status = yield self.poll() status = yield self.poll()
if status is None: if status is None:
# still running, nothing to do here # still running, nothing to do here
return return
self.stop_polling() self.stop_polling()
for callback in self._callbacks: for callback in self._callbacks:
try: try:
yield gen.maybe_future(callback()) yield gen.maybe_future(callback())
except Exception: except Exception:
self.log.exception("Unhandled error in poll callback for %s", self) self.log.exception("Unhandled error in poll callback for %s", self)
return status return status
death_interval = Float(0.1) death_interval = Float(0.1)
@gen.coroutine @gen.coroutine
def wait_for_death(self, timeout=10): def wait_for_death(self, timeout=10):
"""wait for the process to die, up to timeout seconds""" """Wait for the single-user server to die, up to timeout seconds"""
for i in range(int(timeout / self.death_interval)): for i in range(int(timeout / self.death_interval)):
status = yield self.poll() status = yield self.poll()
if status is not None: if status is not None:
@@ -521,8 +606,12 @@ class Spawner(LoggingConfigurable):
else: else:
yield gen.sleep(self.death_interval) yield gen.sleep(self.death_interval)
def _try_setcwd(path): def _try_setcwd(path):
"""Try to set CWD, walking up and ultimately falling back to a temp dir""" """Try to set CWD to path, walking up until a valid directory is found.
If no valid directory is found, a temp directory is created and cwd is set to that.
"""
while path != '/': while path != '/':
try: try:
os.chdir(path) os.chdir(path)
@@ -538,15 +627,24 @@ def _try_setcwd(path):
def set_user_setuid(username): def set_user_setuid(username):
"""return a preexec_fn for setting the user (via setuid) of a spawned process""" """Return a preexec_fn for spawning a single-user server as a particular user.
Returned preexec_fn will set uid/gid, and attempt to chdir to the target user's
home directory.
"""
user = pwd.getpwnam(username) user = pwd.getpwnam(username)
uid = user.pw_uid uid = user.pw_uid
gid = user.pw_gid gid = user.pw_gid
home = user.pw_dir home = user.pw_dir
gids = [ g.gr_gid for g in grp.getgrall() if username in g.gr_mem ] gids = [ g.gr_gid for g in grp.getgrall() if username in g.gr_mem ]
def preexec(): def preexec():
# set the user and group """Set uid/gid of current process
Executed after fork but before exec by python.
Also try to chdir to the user's home directory.
"""
os.setgid(gid) os.setgid(gid)
try: try:
os.setgroups(gids) os.setgroups(gids)
@@ -556,53 +654,92 @@ def set_user_setuid(username):
# start in the user's home dir # start in the user's home dir
_try_setcwd(home) _try_setcwd(home)
return preexec return preexec
class LocalProcessSpawner(Spawner): class LocalProcessSpawner(Spawner):
"""A Spawner that just uses Popen to start local processes as users. """
A Spawner that uses `subprocess.Popen` to start single-user servers as local processes.
Requires users to exist on the local system.
Requires local UNIX users matching the authenticated users to exist.
Does not work on Windows.
This is the default spawner for JupyterHub. This is the default spawner for JupyterHub.
""" """
INTERRUPT_TIMEOUT = Integer(10, INTERRUPT_TIMEOUT = Integer(10,
help="Seconds to wait for process to halt after SIGINT before proceeding to SIGTERM" help="""
Seconds to wait for single-user server process to halt after SIGINT.
If the process has not exited cleanly after this many seconds, a SIGTERM is sent.
"""
).tag(config=True) ).tag(config=True)
TERM_TIMEOUT = Integer(5, TERM_TIMEOUT = Integer(5,
help="Seconds to wait for process to halt after SIGTERM before proceeding to SIGKILL" help="""
Seconds to wait for single-user server process to halt after SIGTERM.
If the process does not exit cleanly after this many seconds of SIGTERM, a SIGKILL is sent.
"""
).tag(config=True) ).tag(config=True)
KILL_TIMEOUT = Integer(5, KILL_TIMEOUT = Integer(5,
help="Seconds to wait for process to halt after SIGKILL before giving up" help="""
Seconds to wait for process to halt after SIGKILL before giving up.
If the process does not exit cleanly after this many seconds of SIGKILL, it becomes a zombie
process. The hub process will log a warning and then give up.
"""
).tag(config=True) ).tag(config=True)
proc = Instance(Popen, allow_none=True) proc = Instance(Popen,
pid = Integer(0) allow_none=True,
help="""
The process representing the single-user server process spawned for current user.
Is None if no process has been spawned yet.
""")
pid = Integer(0,
help="""
The process id (pid) of the single-user server process spawned for current user.
"""
)
def make_preexec_fn(self, name): def make_preexec_fn(self, name):
"""
Return a function that can be used to set the user id of the spawned process to user with name `name`
This function can be safely passed to `preexec_fn` of `Popen`
"""
return set_user_setuid(name) return set_user_setuid(name)
def load_state(self, state): def load_state(self, state):
"""load pid from state""" """Restore state about spawned single-user server after a hub restart.
Local processes only need the process id.
"""
super(LocalProcessSpawner, self).load_state(state) super(LocalProcessSpawner, self).load_state(state)
if 'pid' in state: if 'pid' in state:
self.pid = state['pid'] self.pid = state['pid']
def get_state(self): def get_state(self):
"""add pid to state""" """Save state that is needed to restore this spawner instance after a hub restore.
Local processes only need the process id.
"""
state = super(LocalProcessSpawner, self).get_state() state = super(LocalProcessSpawner, self).get_state()
if self.pid: if self.pid:
state['pid'] = self.pid state['pid'] = self.pid
return state return state
def clear_state(self): def clear_state(self):
"""clear pid state""" """Clear stored state about this spawner (pid)"""
super(LocalProcessSpawner, self).clear_state() super(LocalProcessSpawner, self).clear_state()
self.pid = 0 self.pid = 0
def user_env(self, env): def user_env(self, env):
"""Augment environment of spawned process with user specific env variables."""
env['USER'] = self.user.name env['USER'] = self.user.name
home = pwd.getpwnam(self.user.name).pw_dir home = pwd.getpwnam(self.user.name).pw_dir
shell = pwd.getpwnam(self.user.name).pw_shell shell = pwd.getpwnam(self.user.name).pw_shell
@@ -613,16 +750,16 @@ class LocalProcessSpawner(Spawner):
if shell: if shell:
env['SHELL'] = shell env['SHELL'] = shell
return env return env
def get_env(self): def get_env(self):
"""Add user environment variables""" """Get the complete set of environment variables to be set in the spawned process."""
env = super().get_env() env = super().get_env()
env = self.user_env(env) env = self.user_env(env)
return env return env
@gen.coroutine @gen.coroutine
def start(self): def start(self):
"""Start the process""" """Start the single-user server."""
self.port = random_port() self.port = random_port()
cmd = [] cmd = []
env = self.get_env() env = self.get_env()
@@ -659,7 +796,11 @@ class LocalProcessSpawner(Spawner):
@gen.coroutine @gen.coroutine
def poll(self): def poll(self):
"""Poll the process""" """Poll the spawned process to see if it is still running.
If the process is still running, we return None. If it is not running,
we return the exit code of the process if we have access to it, or 0 otherwise.
"""
# if we started the process, poll with Popen # if we started the process, poll with Popen
if self.proc is not None: if self.proc is not None:
status = self.proc.poll() status = self.proc.poll()
@@ -667,15 +808,14 @@ class LocalProcessSpawner(Spawner):
# clear state if the process is done # clear state if the process is done
self.clear_state() self.clear_state()
return status return status
# if we resumed from stored state, # if we resumed from stored state,
# we don't have the Popen handle anymore, so rely on self.pid # we don't have the Popen handle anymore, so rely on self.pid
if not self.pid: if not self.pid:
# no pid, not running # no pid, not running
self.clear_state() self.clear_state()
return 0 return 0
# send signal 0 to check if PID exists # send signal 0 to check if PID exists
# this doesn't work on Windows, but that's okay because we don't support Windows. # this doesn't work on Windows, but that's okay because we don't support Windows.
alive = yield self._signal(0) alive = yield self._signal(0)
@@ -684,10 +824,15 @@ class LocalProcessSpawner(Spawner):
return 0 return 0
else: else:
return None return None
@gen.coroutine @gen.coroutine
def _signal(self, sig): def _signal(self, sig):
"""simple implementation of signal, which we can use when we are using setuid (we are root)""" """Send given signal to a single-user server's process.
Returns True if the process still exists, False otherwise.
The hub process is assumed to have enough privileges to do this (e.g. root).
"""
try: try:
os.kill(self.pid, sig) os.kill(self.pid, sig)
except OSError as e: except OSError as e:
@@ -696,12 +841,13 @@ class LocalProcessSpawner(Spawner):
else: else:
raise raise
return True # process exists return True # process exists
@gen.coroutine @gen.coroutine
def stop(self, now=False): def stop(self, now=False):
"""stop the subprocess """Stop the single-user server process for the current user.
if `now`, skip waiting for clean shutdown If `now` is set to True, do not wait for the process to die.
Otherwise, it'll wait.
""" """
if not now: if not now:
status = yield self.poll() status = yield self.poll()
@@ -710,7 +856,7 @@ class LocalProcessSpawner(Spawner):
self.log.debug("Interrupting %i", self.pid) self.log.debug("Interrupting %i", self.pid)
yield self._signal(signal.SIGINT) yield self._signal(signal.SIGINT)
yield self.wait_for_death(self.INTERRUPT_TIMEOUT) yield self.wait_for_death(self.INTERRUPT_TIMEOUT)
# clean shutdown failed, use TERM # clean shutdown failed, use TERM
status = yield self.poll() status = yield self.poll()
if status is not None: if status is not None:
@@ -718,7 +864,7 @@ class LocalProcessSpawner(Spawner):
self.log.debug("Terminating %i", self.pid) self.log.debug("Terminating %i", self.pid)
yield self._signal(signal.SIGTERM) yield self._signal(signal.SIGTERM)
yield self.wait_for_death(self.TERM_TIMEOUT) yield self.wait_for_death(self.TERM_TIMEOUT)
# TERM failed, use KILL # TERM failed, use KILL
status = yield self.poll() status = yield self.poll()
if status is not None: if status is not None:
@@ -731,4 +877,3 @@ class LocalProcessSpawner(Spawner):
if status is None: if status is None:
# it all failed, zombie process # it all failed, zombie process
self.log.warning("Process %i never died", self.pid) self.log.warning("Process %i never died", self.pid)