From 826f07e37bde9bcdfc8ab719142029ad51da16db Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 6 Jan 2015 13:02:18 -0800 Subject: [PATCH 1/9] allow creating new API tokens on the command line with jupyterhub token [username] --- jupyterhub/app.py | 41 +++++++++++++++++++++++++++++++++++- jupyterhub/tests/test_app.py | 8 +++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 377afa8d..9b9d8540 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -79,6 +79,38 @@ flags = { SECRET_BYTES = 2048 # the number of bytes to use when generating new secrets +class NewToken(Application): + """Generate and print a new API token""" + name = 'jupyterhub-token' + description = """Generate and return new API token for a user.""" + + examples = """ + $> jupyterhub token myuser + ab01cd23ef45 + """ + + name = Unicode() + aliases = {} + flags = {} + + def parse_command_line(self, argv=None): + super().parse_command_line(argv=argv) + if len(self.extra_args) != 1: + print("Must specify exactly one username", file=sys.stderr) + self.exit(1) + self.name = self.extra_args[0] + + def start(self): + hub = JupyterHub(parent=self) + hub.init_db() + user = orm.User.find(hub.db, self.name) + if user is None: + print("No such user: %s" % self.name) + self.exit(1) + token = user.new_api_token() + print(token) + + class JupyterHub(Application): """An Application for starting a Multi-User Jupyter Notebook server.""" name = 'jupyterhub' @@ -104,6 +136,10 @@ class JupyterHub(Application): aliases = Dict(aliases) flags = Dict(flags) + subcommands = { + 'token': (NewToken, "Generate an API token for a user") + } + classes = List([ Spawner, LocalProcessSpawner, @@ -688,7 +724,7 @@ class JupyterHub(Application): @catch_config_error def initialize(self, *args, **kwargs): super().initialize(*args, **kwargs) - if self.generate_config: + if self.generate_config or self.subapp: return self.load_config_file(self.config_file) self.init_logging() @@ -793,6 +829,9 @@ class JupyterHub(Application): def start(self): """Start the whole thing""" + if self.subapp: + return self.subapp.start() + if self.generate_config: self.write_config_file() return diff --git a/jupyterhub/tests/test_app.py b/jupyterhub/tests/test_app.py index e4cfc167..ff26adcf 100644 --- a/jupyterhub/tests/test_app.py +++ b/jupyterhub/tests/test_app.py @@ -1,7 +1,9 @@ """Test the JupyterHub entry point""" import os +import re import sys +from getpass import getuser from subprocess import check_output from tempfile import NamedTemporaryFile @@ -10,6 +12,12 @@ def test_help_all(): assert '--ip' in out assert '--JupyterHub.ip' in out +def test_token_app(): + cmd = [sys.executable, '-m', 'jupyterhub', 'token'] + out = check_output(cmd + ['--help-all']).decode('utf8', 'replace') + out = check_output(cmd + [getuser()]).decode('utf8', 'replace').strip() + assert re.match(r'^[a-z0-9]+$', out) + def test_generate_config(): with NamedTemporaryFile(prefix='jupyterhub_config', suffix='.py') as tf: cfg_file = tf.name From 0abe4653ca6d6b441fae99fe5ce32e8213f3575b Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 6 Jan 2015 13:02:30 -0800 Subject: [PATCH 2/9] default to whoami for creating tokens --- jupyterhub/app.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 9b9d8540..5b240662 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -82,20 +82,27 @@ SECRET_BYTES = 2048 # the number of bytes to use when generating new secrets class NewToken(Application): """Generate and print a new API token""" name = 'jupyterhub-token' - description = """Generate and return new API token for a user.""" + description = """Generate and return new API token for a user. + + Usage: + + jupyterhub token [username] + """ examples = """ - $> jupyterhub token myuser + $> jupyterhub token kaylee ab01cd23ef45 """ - name = Unicode() + name = Unicode(getuser()) aliases = {} flags = {} def parse_command_line(self, argv=None): super().parse_command_line(argv=argv) - if len(self.extra_args) != 1: + if not self.extra_args: + return + if len(self.extra_args) > 1: print("Must specify exactly one username", file=sys.stderr) self.exit(1) self.name = self.extra_args[0] From e066ac1863e11ae84f80e38c83ca05413ade5e72 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 6 Jan 2015 15:25:04 -0800 Subject: [PATCH 3/9] add last_activity to user API model --- jupyterhub/apihandlers/users.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jupyterhub/apihandlers/users.py b/jupyterhub/apihandlers/users.py index 0464474a..e6066513 100644 --- a/jupyterhub/apihandlers/users.py +++ b/jupyterhub/apihandlers/users.py @@ -18,6 +18,7 @@ class BaseUserHandler(APIHandler): 'name': user.name, 'admin': user.admin, 'server': user.server.base_url if user.server else None, + 'last_activity': user.last_activity.isoformat(), } _model_types = { From 213b3e2afbf8f1549b5c41358d07b50ee1817e63 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 6 Jan 2015 15:25:32 -0800 Subject: [PATCH 4/9] don't track every login as activity only server activity should be tracked --- jupyterhub/handlers/base.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jupyterhub/handlers/base.py b/jupyterhub/handlers/base.py index db544d4c..3c401a54 100644 --- a/jupyterhub/handlers/base.py +++ b/jupyterhub/handlers/base.py @@ -79,9 +79,7 @@ class BaseHandler(RequestHandler): if orm_token is None: return None else: - user = orm_token.user - user.last_activity = datetime.utcnow() - return user + return orm_token.user def _user_for_cookie(self, cookie_name, cookie_value=None): """Get the User for a given cookie, if there is one""" From 00b7f47324ab47714fcdda30e998a9f7aaefb2b2 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 6 Jan 2015 15:29:20 -0800 Subject: [PATCH 5/9] reduce default last-activity interval to 5 minutes from 10 minutes --- jupyterhub/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 5b240662..17641894 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -168,7 +168,7 @@ class JupyterHub(Application): Useful for daemonizing jupyterhub. """ ) - last_activity_interval = Integer(600, config=True, + last_activity_interval = Integer(300, config=True, help="Interval (in seconds) at which to update last-activity timestamps." ) proxy_check_interval = Integer(30, config=True, From e59ba5417314053cbed4f6776b9e3130fae2a24a Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 6 Jan 2015 15:39:47 -0800 Subject: [PATCH 6/9] add example script for culling idle servers use case for `jupyterhub token` --- examples/cull-idle/cull_idle_servers.py | 84 +++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 examples/cull-idle/cull_idle_servers.py diff --git a/examples/cull-idle/cull_idle_servers.py b/examples/cull-idle/cull_idle_servers.py new file mode 100644 index 00000000..82ac4b78 --- /dev/null +++ b/examples/cull-idle/cull_idle_servers.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +"""script to monitor and cull idle single-user servers + +Caveats: + +last_activity is not updated with high frequency, +so cull timeout should be greater than the sum of: + +- single-user websocket ping interval (default: 30s) +- JupyterHub.last_activity_interval (default: 5 minutes) + +Generate an API token and store it in `JPY_API_TOKEN`: + + export JPY_API_TOKEN=`jupyterhub token` + python cull_idle_servers.py [--timeout=900] [--url=http://127.0.0.1:8081/hub] +""" + +import datetime +import json +import os + +from dateutil.parser import parse as parse_date + +from tornado.gen import coroutine +from tornado.log import app_log +from tornado.httpclient import AsyncHTTPClient, HTTPRequest +from tornado.ioloop import IOLoop, PeriodicCallback +from tornado.options import define, options, parse_command_line + + +@coroutine +def cull_idle(url, api_token, timeout): + """cull idle single-user servers""" + auth_header = { + 'Authorization': 'token %s' % api_token + } + req = HTTPRequest(url=url + '/api/users', + headers=auth_header, + ) + now = datetime.datetime.utcnow() + cull_limit = now - datetime.timedelta(seconds=timeout) + client = AsyncHTTPClient() + resp = yield client.fetch(req) + users = json.loads(resp.body.decode('utf8', 'replace')) + futures = [] + for user in users: + last_activity = parse_date(user['last_activity']) + if user['server'] and last_activity < cull_limit: + app_log.info("Culling %s (inactive since %s)", user['name'], last_activity) + req = HTTPRequest(url=url + '/api/users/%s/server' % user['name'], + method='DELETE', + headers=auth_header, + ) + futures.append((user['name'], client.fetch(req))) + elif user['server'] and last_activity > cull_limit: + app_log.debug("Not culling %s (active since %s)", user['name'], last_activity) + + for (name, f) in futures: + yield f + app_log.debug("Finished culling %s", name) + +if __name__ == '__main__': + define('url', default='http://127.0.0.1:8081/hub', help="The JupyterHub API URL") + define('timeout', default=600, help="The idle timeout (in seconds)") + define('cull_every', default=0, help="The interval (in seconds) for checking for idle servers to cull") + + parse_command_line() + if not options.cull_every: + options.cull_every = options.timeout // 2 + + api_token = os.environ['JPY_API_TOKEN'] + + loop = IOLoop.current() + cull = lambda : cull_idle(options.url, api_token, options.timeout) + # run once before scheduling periodic call + loop.run_sync(cull) + # schedule periodic cull + pc = PeriodicCallback(cull, 1e3 * options.cull_every) + pc.start() + try: + loop.start() + except KeyboardInterrupt: + pass + \ No newline at end of file From 6766c662d2ebe6a5081a649eab5c8b877b4ffac1 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 6 Jan 2015 15:50:53 -0800 Subject: [PATCH 7/9] init users when creating tokens --- jupyterhub/app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 17641894..e0be4cd1 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -95,8 +95,6 @@ class NewToken(Application): """ name = Unicode(getuser()) - aliases = {} - flags = {} def parse_command_line(self, argv=None): super().parse_command_line(argv=argv) @@ -110,6 +108,7 @@ class NewToken(Application): def start(self): hub = JupyterHub(parent=self) hub.init_db() + hub.init_users() user = orm.User.find(hub.db, self.name) if user is None: print("No such user: %s" % self.name) From 1d324e794e91934d86613f9c9fa9828f0d70fb53 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 6 Jan 2015 15:51:04 -0800 Subject: [PATCH 8/9] run token test in temporary directory --- jupyterhub/tests/test_app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jupyterhub/tests/test_app.py b/jupyterhub/tests/test_app.py index ff26adcf..f4f59d13 100644 --- a/jupyterhub/tests/test_app.py +++ b/jupyterhub/tests/test_app.py @@ -5,7 +5,7 @@ import re import sys from getpass import getuser from subprocess import check_output -from tempfile import NamedTemporaryFile +from tempfile import NamedTemporaryFile, TemporaryDirectory def test_help_all(): out = check_output([sys.executable, '-m', 'jupyterhub', '--help-all']).decode('utf8', 'replace') @@ -15,7 +15,8 @@ def test_help_all(): def test_token_app(): cmd = [sys.executable, '-m', 'jupyterhub', 'token'] out = check_output(cmd + ['--help-all']).decode('utf8', 'replace') - out = check_output(cmd + [getuser()]).decode('utf8', 'replace').strip() + with TemporaryDirectory() as td: + out = check_output(cmd + [getuser()], cwd=td).decode('utf8', 'replace').strip() assert re.match(r'^[a-z0-9]+$', out) def test_generate_config(): From 7a0979aa4fc3c08e68263144278cff937a368fd4 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 7 Jan 2015 15:10:34 -0800 Subject: [PATCH 9/9] don't compare last_activity in user models --- jupyterhub/tests/test_api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jupyterhub/tests/test_api.py b/jupyterhub/tests/test_api.py index fd2a7863..2b3d1718 100644 --- a/jupyterhub/tests/test_api.py +++ b/jupyterhub/tests/test_api.py @@ -95,7 +95,11 @@ def test_get_users(app): db = app.db r = api_request(app, 'users') assert r.status_code == 200 - assert sorted(r.json(), key=lambda d: d['name']) == [ + + users = sorted(r.json(), key=lambda d: d['name']) + for u in users: + u.pop('last_activity') + assert users == [ { 'name': 'admin', 'admin': True,