hazza/jupyterhub
1
0
Fork 0
mirror of https://github.com/jupyterhub/jupyterhub.git synced 2025-10-14 21:43:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow specifying statsd host/port/prefix info

Browse Source 
Currently only passes it through to CHP. This is needed
for the cases when JupyterHub spawns and maintains CHP.
This commit is contained in:
YuviPanda
2016-03-30 18:59:32 -07:00
parent 2bbfd75f4d
commit c6d05d0840
1 changed files with 136 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

252
jupyterhub/app.py
View File

@@ -100,22 +100,22 @@ class NewToken(Application):
"""Generate and print a new API token"""
name = 'jupyterhub-token'
description = """Generate and return new API token for a user.
Usage:
jupyterhub token [username]
"""
examples = """
$> jupyterhub token kaylee
ab01cd23ef45
"""
name = Unicode(getuser())
aliases = token_aliases
classes = []
def parse_command_line(self, argv=None):
super().parse_command_line(argv=argv)
if not self.extra_args:
@@ -124,7 +124,7 @@ class NewToken(Application):
print("Must specify exactly one username", file=sys.stderr)
self.exit(1)
self.name = self.extra_args[0]
def start(self):
hub = JupyterHub(parent=self)
hub.load_config_file(hub.config_file)
@@ -143,39 +143,39 @@ class JupyterHub(Application):
"""An Application for starting a Multi-User Jupyter Notebook server."""
name = 'jupyterhub'
version = jupyterhub.__version__
description = """Start a multi-user Jupyter Notebook server
Spawns a configurable-http-proxy and multi-user Hub,
which authenticates users and spawns single-user Notebook servers
on behalf of users.
"""
examples = """
generate default config file:
jupyterhub --generate-config -f /etc/jupyterhub/jupyterhub.py
spawn the server on 10.0.1.2:443 with https:
jupyterhub --ip 10.0.1.2 --port 443 --ssl-key my_ssl.key --ssl-cert my_ssl.cert
"""
aliases = Dict(aliases)
flags = Dict(flags)
subcommands = {
'token': (NewToken, "Generate an API token for a user")
}
classes = List([
Spawner,
LocalProcessSpawner,
Authenticator,
PAMAuthenticator,
])
config_file = Unicode('jupyterhub_config.py',
help="The config file to load",
).tag(config=True)
@@ -201,7 +201,7 @@ class JupyterHub(Application):
proxy_check_interval = Integer(30,
help="Interval (in seconds) at which to check if the proxy is running."
).tag(config=True)
data_files_path = Unicode(DATA_FILES_PATH,
help="The location of jupyterhub data files (e.g. /usr/local/share/jupyter/hub)"
).tag(config=True)
@@ -213,7 +213,7 @@ class JupyterHub(Application):
@default('template_paths')
def _template_paths_default(self):
return [os.path.join(self.data_files_path, 'templates')]
confirm_no_ssl = Bool(False,
help="""Confirm that JupyterHub should be run without SSL.
This is **NOT RECOMMENDED** unless SSL termination is being handled by another layer.
@@ -221,20 +221,20 @@ class JupyterHub(Application):
).tag(config=True)
ssl_key = Unicode('',
help="""Path to SSL key file for the public facing interface of the proxy
Use with ssl_cert
"""
).tag(config=True)
ssl_cert = Unicode('',
help="""Path to SSL certificate file for the public facing interface of the proxy
Use with ssl_key
"""
).tag(config=True)
ip = Unicode('',
help="The public facing ip of the whole application (the proxy)"
).tag(config=True)
subdomain_host = Unicode('',
help="""Run single-user servers on subdomains of this host.
@@ -254,7 +254,7 @@ class JupyterHub(Application):
# host should include '://'
# if not specified, assume https: You have to be really explicit about HTTP!
self.subdomain_host = 'https://' + new
port = Integer(8000,
help="The public facing port of the proxy"
).tag(config=True)
@@ -268,14 +268,14 @@ class JupyterHub(Application):
@default('logo_file')
def _logo_file_default(self):
return os.path.join(self.data_files_path, 'static', 'images', 'jupyter.png')
jinja_environment_options = Dict(
help="Supply extra arguments that will be passed to Jinja environment."
).tag(config=True)
proxy_cmd = Command('configurable-http-proxy',
help="""The command to start the http proxy.
Only override if configurable-http-proxy is not on your PATH
"""
).tag(config=True)
@@ -288,7 +288,7 @@ class JupyterHub(Application):
Loaded from the CONFIGPROXY_AUTH_TOKEN env variable by default.
"""
).tag(config=True)
@default('proxy_auth_token')
def _proxy_auth_token_default(self):
token = os.environ.get('CONFIGPROXY_AUTH_TOKEN', None)
@@ -301,18 +301,18 @@ class JupyterHub(Application):
]))
token = orm.new_token()
return token
proxy_api_ip = Unicode('127.0.0.1',
help="The ip for the proxy API handlers"
).tag(config=True)
proxy_api_port = Integer(
help="The port for the proxy API handlers"
).tag(config=True)
@default('proxy_api_port')
def _proxy_api_port_default(self):
return self.port + 1
hub_port = Integer(8081,
help="The port for this process"
).tag(config=True)
@@ -322,18 +322,18 @@ class JupyterHub(Application):
hub_prefix = URLPrefix('/hub/',
help="The prefix for the hub server. Must not be '/'"
).tag(config=True)
@default('hub_prefix')
def _hub_prefix_default(self):
return url_path_join(self.base_url, '/hub/')
@observe('hub_prefix')
def _hub_prefix_changed(self, name, old, new):
if new == '/':
raise TraitError("'/' is not a valid hub prefix")
if not new.startswith(self.base_url):
self.hub_prefix = url_path_join(self.base_url, new)
cookie_secret = Bytes(
help="""The cookie secret to use to encrypt cookies.
@@ -343,18 +343,18 @@ class JupyterHub(Application):
config=True,
env='JPY_COOKIE_SECRET',
)
cookie_secret_file = Unicode('jupyterhub_cookie_secret',
help="""File in which to store the cookie secret."""
).tag(config=True)
authenticator_class = Type(PAMAuthenticator, Authenticator,
help="""Class for authenticating users.
This should be a class with the following form:
- constructor takes one kwarg: `config`, the IPython config object.
- is a tornado.gen.coroutine
- returns username on success, None on failure
- takes two arguments: (handler, data),
@@ -362,7 +362,7 @@ class JupyterHub(Application):
and `data` is the POST form data from the login page.
"""
).tag(config=True)
authenticator = Instance(Authenticator)
@default('authenticator')
@@ -372,11 +372,11 @@ class JupyterHub(Application):
# class for spawning single-user servers
spawner_class = Type(LocalProcessSpawner, Spawner,
help="""The class to use for spawning single-user servers.
Should be a subclass of Spawner.
"""
).tag(config=True)
db_url = Unicode('sqlite:///jupyterhub.sqlite',
help="url for the database. e.g. `sqlite:///jupyterhub.sqlite`"
).tag(config=True)
@@ -401,65 +401,79 @@ class JupyterHub(Application):
help="log all database transactions. This has A LOT of output"
).tag(config=True)
session_factory = Any()
users = Instance(UserDict)
@default('users')
def _users_default(self):
assert self.tornado_settings
return UserDict(db_factory=lambda : self.db, settings=self.tornado_settings)
admin_access = Bool(False,
help="""Grant admin users permission to access single-user servers.
Users should be properly informed if this is enabled.
"""
).tag(config=True)
admin_users = Set(
help="""DEPRECATED, use Authenticator.admin_users instead."""
).tag(config=True)
tornado_settings = Dict(
help="Extra settings overrides to pass to the tornado application."
).tag(config=True)
cleanup_servers = Bool(True,
help="""Whether to shutdown single-user servers when the Hub shuts down.
Disable if you want to be able to teardown the Hub while leaving the single-user servers running.
If both this and cleanup_proxy are False, sending SIGINT to the Hub will
only shutdown the Hub, leaving everything else running.
The Hub should be able to resume from database state.
"""
).tag(config=True)
cleanup_proxy = Bool(True,
help="""Whether to shutdown the proxy when the Hub shuts down.
Disable if you want to be able to teardown the Hub while leaving the proxy running.
Only valid if the proxy was starting by the Hub process.
If both this and cleanup_servers are False, sending SIGINT to the Hub will
only shutdown the Hub, leaving everything else running.
The Hub should be able to resume from database state.
"""
).tag(config=True)
statsd_host = Unicode(
help="Host to send statds metrics to"
).tag(config=True)
statsd_port = Int(
8125,
help="Port on which to send statsd metrics about the hub"
).tag(config=True)
statsd_prefix = Unicode(
'jupyterhub',
help="Prefix to use for all metrics sent by jupyterhub to statsd"
)
handlers = List()
_log_formatter_cls = CoroutineLogFormatter
http_server = None
proxy_process = None
io_loop = None
@default('log_level')
def _log_level_default(self):
return logging.INFO
@default('log_datefmt')
def _log_datefmt_default(self):
"""Exclude date from default date format"""
@@ -514,7 +528,7 @@ class JupyterHub(Application):
raise TraitError("The hub and proxy API cannot both listen on port %i" % self.hub_port)
if self.proxy_api_port == self.port:
raise TraitError("The proxy's public and API ports cannot both be %i" % self.port)
@staticmethod
def add_url_prefix(prefix, handlers):
"""add a url prefix to handlers"""
@@ -523,7 +537,7 @@ class JupyterHub(Application):
lis[0] = url_path_join(prefix, tup[0])
handlers[i] = tuple(lis)
return handlers
def init_handlers(self):
h = []
# load handlers from the authenticator
@@ -531,7 +545,7 @@ class JupyterHub(Application):
# set default handlers
h.extend(handlers.default_handlers)
h.extend(apihandlers.default_handlers)
h.append((r'/logo', LogoHandler, {'path': self.logo_file}))
self.handlers = self.add_url_prefix(self.hub_prefix, h)
# some extra handlers, outside hub_prefix
@@ -545,7 +559,7 @@ class JupyterHub(Application):
(r"(?!%s).*" % self.hub_prefix, handlers.PrefixRedirectHandler),
(r'(.*)', handlers.Template404),
])
def _check_db_path(self, path):
"""More informative log messages for failed filesystem access"""
path = os.path.abspath(path)
@@ -557,7 +571,7 @@ class JupyterHub(Application):
self.log.error("%s cannot create files in %s", user, parent)
if os.path.exists(path) and not os.access(path, os.W_OK):
self.log.error("%s cannot edit %s", user, path)
def init_secrets(self):
trait_name = 'cookie_secret'
trait = self.traits()[trait_name]
@@ -589,7 +603,7 @@ class JupyterHub(Application):
secret_from = 'new'
self.log.debug("Generating new %s", trait_name)
secret = os.urandom(SECRET_BYTES)
if secret_file and secret_from == 'new':
# if we generated a new secret, store it in the secret_file
self.log.info("Writing %s to %s", trait_name, secret_file)
@@ -602,7 +616,7 @@ class JupyterHub(Application):
self.log.warning("Failed to set permissions on %s", secret_file)
# store the loaded trait value
self.cookie_secret = secret
# thread-local storage of db objects
_local = Instance(threading.local, ())
@property
@@ -610,7 +624,7 @@ class JupyterHub(Application):
if not hasattr(self._local, 'db'):
self._local.db = scoped_session(self.session_factory)()
return self._local.db
@property
def hub(self):
if not getattr(self._local, 'hub', None):
@@ -620,13 +634,13 @@ class JupyterHub(Application):
if self.subdomain_host and self._local.hub:
self._local.hub.host = self.subdomain_host
return self._local.hub
@hub.setter
def hub(self, hub):
self._local.hub = hub
if hub and self.subdomain_host:
hub.host = self.subdomain_host
@property
def proxy(self):
if not getattr(self._local, 'proxy', None):
@@ -636,11 +650,11 @@ class JupyterHub(Application):
if p:
p.auth_token = self.proxy_auth_token
return self._local.proxy
@proxy.setter
def proxy(self, proxy):
self._local.proxy = proxy
def init_db(self):
"""Create the database connection"""
self.log.debug("Connecting to db: %s", self.db_url)
@@ -659,7 +673,7 @@ class JupyterHub(Application):
if self.db_url.startswith('sqlite:///'):
self._check_db_path(self.db_url.split(':///', 1)[1])
self.exit(1)
def init_hub(self):
"""Load the Hub config into the database"""
self.hub = self.db.query(orm.Hub).first()
@@ -684,12 +698,12 @@ class JupyterHub(Application):
" This should be the public domain[:port] of the Hub.")
self.db.commit()
@gen.coroutine
def init_users(self):
"""Load users into and from the database"""
db = self.db
if self.admin_users and not self.authenticator.admin_users:
self.log.warning(
"\nJupyterHub.admin_users is deprecated."
@@ -704,11 +718,11 @@ class JupyterHub(Application):
for username in admin_users:
if not self.authenticator.validate_username(username):
raise ValueError("username %r is not valid" % username)
if not admin_users:
self.log.warning("No admin users, admin interface will be unavailable.")
self.log.warning("Add any administrative users to `c.Authenticator.admin_users` in config.")
new_users = []
for name in admin_users:
@@ -759,11 +773,11 @@ class JupyterHub(Application):
# From this point on, any user changes should be done simultaneously
# to the whitelist set and user db, unless the whitelist is empty (all users allowed).
@gen.coroutine
def init_spawners(self):
db = self.db
user_summaries = ['']
def _user_summary(user):
parts = ['{0: >8}'.format(user.name)]
@@ -772,7 +786,7 @@ class JupyterHub(Application):
if user.server:
parts.append('running at %s' % user.server)
return ' '.join(parts)
@gen.coroutine
def user_stopped(user):
status = yield user.spawner.poll()
@@ -781,7 +795,7 @@ class JupyterHub(Application):
)
yield self.proxy.delete_user(user)
yield user.stop()
for orm_user in db.query(orm.User):
self.users[orm_user.id] = user = User(orm_user, self.tornado_settings)
if not user.state:
@@ -827,7 +841,7 @@ class JupyterHub(Application):
self.proxy.api_server.port = self.proxy_api_port
self.proxy.api_server.base_url = '/api/routes/'
self.db.commit()
@gen.coroutine
def start_proxy(self):
"""Actually start the configurable-http-proxy"""
@@ -867,7 +881,13 @@ class JupyterHub(Application):
cmd.extend(['--ssl-key', self.ssl_key])
if self.ssl_cert:
cmd.extend(['--ssl-cert', self.ssl_cert])
# Require SSL to be used or `--no-ssl` to confirm no SSL on
if self.statsd_host:
cmd.extend([
'--statsd-host', self.statsd_host,
'--statsd-port', self.statsd_port,
'--statsd-prefix', self.statsd_prefix + '.chp'
])
# Require SSL to be used or `--no-ssl` to confirm no SSL on
if ' --ssl' not in ' '.join(cmd):
if self.confirm_no_ssl:
self.log.warning("Running JupyterHub without SSL."
@@ -897,7 +917,7 @@ class JupyterHub(Application):
# py2-compatible `raise e from None`
e.__cause__ = None
raise e
for server in (self.proxy.public_server, self.proxy.api_server):
for i in range(10):
_check()
@@ -909,7 +929,7 @@ class JupyterHub(Application):
break
yield server.wait_up(1)
self.log.debug("Proxy started and appears to be up")
@gen.coroutine
def check_proxy(self):
if self.proxy_process.poll() is None:
@@ -921,7 +941,7 @@ class JupyterHub(Application):
self.log.info("Setting up routes on new proxy")
yield self.proxy.add_all_users(self.users)
self.log.info("New proxy back up, and good to go")
def init_tornado_settings(self):
"""Set up the tornado settings dict."""
base_url = self.hub.server.base_url
@@ -933,10 +953,10 @@ class JupyterHub(Application):
loader=FileSystemLoader(self.template_paths),
**jinja_options
)
login_url = self.authenticator.login_url(base_url)
logout_url = self.authenticator.logout_url(base_url)
# if running from git, disable caching of require.js
# otherwise cache based on server start time
parent = os.path.dirname(os.path.dirname(jupyterhub.__file__))
@@ -944,7 +964,7 @@ class JupyterHub(Application):
version_hash = ''
else:
version_hash=datetime.now().strftime("%Y%m%d%H%M%S"),
subdomain_host = self.subdomain_host
domain = urlparse(subdomain_host).hostname
settings = dict(
@@ -977,18 +997,18 @@ class JupyterHub(Application):
self.tornado_settings = settings
# constructing users requires access to tornado_settings
self.tornado_settings['users'] = self.users
def init_tornado_application(self):
"""Instantiate the tornado Application object"""
self.tornado_application = web.Application(self.handlers, **self.tornado_settings)
def write_pid_file(self):
pid = os.getpid()
if self.pid_file:
self.log.debug("Writing PID %i to %s", pid, self.pid_file)
with open(self.pid_file, 'w') as f:
f.write('%i' % pid)
@gen.coroutine
@catch_config_error
def initialize(self, *args, **kwargs):
@@ -1017,11 +1037,11 @@ class JupyterHub(Application):
yield self.init_spawners()
self.init_handlers()
self.init_tornado_application()
@gen.coroutine
def cleanup(self):
"""Shutdown our various subprocesses and cleanup runtime files."""
futures = []
if self.cleanup_servers:
self.log.info("Cleaning up single-user servers...")
@@ -1031,7 +1051,7 @@ class JupyterHub(Application):
futures.append(user.stop())
else:
self.log.info("Leaving single-user servers running")
# clean up proxy while SUS are shutting down
if self.cleanup_proxy:
if self.proxy_process:
@@ -1045,24 +1065,24 @@ class JupyterHub(Application):
self.log.info("I didn't start the proxy, I can't clean it up")
else:
self.log.info("Leaving proxy running")
# wait for the requests to stop finish:
for f in futures:
try:
yield f
except Exception as e:
self.log.error("Failed to stop user: %s", e)
self.db.commit()
if self.pid_file and os.path.exists(self.pid_file):
self.log.info("Cleaning up PID file %s", self.pid_file)
os.remove(self.pid_file)
# finally stop the loop once we are all cleaned up
self.log.info("...done")
def write_config_file(self):
"""Write our default config to a .py config file"""
if os.path.exists(self.config_file) and not self.answer_yes:
@@ -1080,14 +1100,14 @@ class JupyterHub(Application):
answer = ask()
if answer.startswith('n'):
return
config_text = self.generate_config_file()
if isinstance(config_text, bytes):
config_text = config_text.decode('utf8')
print("Writing default config to: %s" % self.config_file)
with open(self.config_file, mode='w') as f:
f.write(config_text)
@gen.coroutine
def update_last_activity(self):
"""Update User.last_activity timestamps from the proxy"""
@@ -1108,22 +1128,22 @@ class JupyterHub(Application):
self.db.commit()
yield self.proxy.check_routes(self.users, routes)
@gen.coroutine
def start(self):
"""Start the whole thing"""
self.io_loop = loop = IOLoop.current()
if self.subapp:
self.subapp.start()
loop.stop()
return
if self.generate_config:
self.write_config_file()
loop.stop()
return
# start the webserver
self.http_server = tornado.httpserver.HTTPServer(self.tornado_application, xheaders=True)
try:
@@ -1133,7 +1153,7 @@ class JupyterHub(Application):
raise
else:
self.log.info("Hub API listening on %s", self.hub.server.bind_url)
# start the proxy
try:
yield self.start_proxy()
@@ -1141,16 +1161,16 @@ class JupyterHub(Application):
self.log.critical("Failed to start proxy", exc_info=True)
self.exit(1)
return
loop.add_callback(self.proxy.add_all_users, self.users)
if self.proxy_process:
# only check / restart the proxy if we started it in the first place.
# this means a restarted Hub cannot restart a Proxy that its
# predecessor started.
pc = PeriodicCallback(self.check_proxy, 1e3 * self.proxy_check_interval)
pc.start()
if self.last_activity_interval:
pc = PeriodicCallback(self.update_last_activity, 1e3 * self.last_activity_interval)
pc.start()
@@ -1162,12 +1182,12 @@ class JupyterHub(Application):
def init_signal(self):
signal.signal(signal.SIGTERM, self.sigterm)
def sigterm(self, signum, frame):
self.log.critical("Received SIGTERM, shutting down")
self.io_loop.stop()
self.atexit()
_atexit_ran = False
def atexit(self):
"""atexit callback"""
@@ -1179,8 +1199,8 @@ class JupyterHub(Application):
loop = IOLoop()
loop.make_current()
loop.run_sync(self.cleanup)
def stop(self):
if not self.io_loop:
return
@@ -1190,7 +1210,7 @@ class JupyterHub(Application):
else:
self.http_server.stop()
self.io_loop.add_callback(self.io_loop.stop)
@gen.coroutine
def launch_instance_async(self, argv=None):
try:
@@ -1199,7 +1219,7 @@ class JupyterHub(Application):
except Exception as e:
self.log.exception("")
self.exit(1)
@classmethod
def launch_instance(cls, argv=None):
self = cls.instance()