diff --git a/.flake8 b/.flake8 index 48e1d856..ba05051a 100644 --- a/.flake8 +++ b/.flake8 @@ -4,9 +4,10 @@ # W: style warnings # C: complexity # F401: module imported but unused +# F403: import * # F811: redefinition of unused `name` from line `N` # F841: local variable assigned but never used -ignore = E, C, W, F401, F811, F841 +ignore = E, C, W, F401, F403, F811, F841 exclude = .cache, diff --git a/jupyterhub/apihandlers/base.py b/jupyterhub/apihandlers/base.py index a07e71bc..72c21702 100644 --- a/jupyterhub/apihandlers/base.py +++ b/jupyterhub/apihandlers/base.py @@ -32,7 +32,7 @@ class APIHandler(BaseHandler): self.log.warning("Blocking API request with no referer") return False - host_path = url_path_join(host, self.hub.server.base_url) + host_path = url_path_join(host, self.hub.base_url) referer_path = referer.split('://', 1)[-1] if not (referer_path + '/').startswith(host_path): self.log.warning("Blocking Cross Origin API request. Referer: %s, Host: %s", diff --git a/jupyterhub/apihandlers/proxy.py b/jupyterhub/apihandlers/proxy.py index a2b705f3..177f855d 100644 --- a/jupyterhub/apihandlers/proxy.py +++ b/jupyterhub/apihandlers/proxy.py @@ -4,6 +4,7 @@ # Distributed under the terms of the Modified BSD License. import json +from urllib.parse import urlparse from tornado import gen, web @@ -21,7 +22,7 @@ class ProxyAPIHandler(APIHandler): This is the same as fetching the routing table directly from the proxy, but without clients needing to maintain separate """ - routes = yield self.proxy.get_routes() + routes = yield self.proxy.get_all_routes() self.write(json.dumps(routes)) @admin_only @@ -48,17 +49,11 @@ class ProxyAPIHandler(APIHandler): if not isinstance(model, dict): raise web.HTTPError(400, "Request body must be JSON dict") - server = self.proxy.api_server - if 'ip' in model: - server.ip = model['ip'] - if 'port' in model: - server.port = model['port'] - if 'protocol' in model: - server.proto = model['protocol'] + if 'api_url' in model: + self.proxy.api_url = model['api_url'] if 'auth_token' in model: self.proxy.auth_token = model['auth_token'] - self.db.commit() - self.log.info("Updated proxy at %s", server.bind_url) + self.log.info("Updated proxy at %s", self.proxy) yield self.proxy.check_routes(self.users, self.services) diff --git a/jupyterhub/apihandlers/users.py b/jupyterhub/apihandlers/users.py index 78af7c97..4433dde1 100644 --- a/jupyterhub/apihandlers/users.py +++ b/jupyterhub/apihandlers/users.py @@ -224,7 +224,7 @@ class UserCreateNamedServerAPIHandler(APIHandler): def post(self, name): user = self.find_user(name) if user is None: - raise HTTPError(404, "No such user %r" % name) + raise web.HTTPError(404, "No such user %r" % name) if user.running: # include notify, so that a server that died is noticed immediately state = yield user.spawner.poll_and_notify() diff --git a/jupyterhub/app.py b/jupyterhub/app.py index b6b89293..62f74b4b 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -30,7 +30,6 @@ from sqlalchemy.orm import scoped_session import tornado.httpserver import tornado.options -from tornado.httpclient import HTTPError from tornado.ioloop import IOLoop, PeriodicCallback from tornado.log import app_log, access_log, gen_log from tornado import gen, web @@ -54,6 +53,7 @@ from .user import User, UserDict from .oauth.store import make_provider from ._data import DATA_FILES_PATH from .log import CoroutineLogFormatter, log_request +from .proxy import Proxy, ConfigurableHTTPProxy from .traitlets import URLPrefix, Command from .utils import ( url_path_join, @@ -62,6 +62,7 @@ from .utils import ( # classes for config from .auth import Authenticator, PAMAuthenticator from .spawner import Spawner, LocalProcessSpawner +from .objects import Hub # For faking stats from .emptyclass import EmptyClass @@ -140,7 +141,6 @@ class NewToken(Application): hub = JupyterHub(parent=self) hub.load_config_file(hub.config_file) hub.init_db() - hub.hub = hub.db.query(orm.Hub).first() hub.init_users() user = orm.User.find(hub.db, self.name) if user is None: @@ -349,45 +349,45 @@ class JupyterHub(Application): help="Supply extra arguments that will be passed to Jinja environment." ).tag(config=True) - proxy_cmd = Command('configurable-http-proxy', - help="""The command to start the http proxy. + proxy_class = Type(ConfigurableHTTPProxy, Proxy, + help="""Select the Proxy API implementation.""" + ).tag(config=True) - Only override if configurable-http-proxy is not on your PATH - """ + proxy_cmd = Command([], config=True, + help="DEPRECATED. Use ConfigurableHTTPProxy.command", ).tag(config=True) + debug_proxy = Bool(False, - help="show debug output in configurable-http-proxy" + help="DEPRECATED: Use ConfigurableHTTPProxy.debug", ).tag(config=True) proxy_auth_token = Unicode( - help="""The Proxy Auth token. - - Loaded from the CONFIGPROXY_AUTH_TOKEN env variable by default. - """ + help="DEPRECATED: Use ConfigurableHTTPProxy.auth_token" ).tag(config=True) - @default('proxy_auth_token') - def _proxy_auth_token_default(self): - token = os.environ.get('CONFIGPROXY_AUTH_TOKEN', None) - if not token: - self.log.warning('\n'.join([ - "", - "Generating CONFIGPROXY_AUTH_TOKEN. Restarting the Hub will require restarting the proxy.", - "Set CONFIGPROXY_AUTH_TOKEN env or JupyterHub.proxy_auth_token config to avoid this message.", - "", - ])) - token = orm.new_token() - return token + _proxy_config_map = { + 'proxy_cmd': 'command', + 'debug_proxy': 'debug', + 'proxy_auth_token': 'auth_token', + } + @observe(*_proxy_config_map) + def _deprecated_proxy_config(self, change): + dest = self._proxy_config_map[change.name] + self.log.warning("JupyterHub.%s is deprecated in JupyterHub 0.8, use ConfigurableHTTPProxy.%s", change.name, dest) + self.config.ConfigurableHTTPProxy[dest] = change.new - proxy_api_ip = Unicode('127.0.0.1', - help="The ip for the proxy API handlers" + proxy_api_ip = Unicode( + help="DEPRECATED: Use ConfigurableHTTPProxy.api_url" ).tag(config=True) proxy_api_port = Integer( - help="The port for the proxy API handlers" + help="DEPRECATED: Use ConfigurableHTTPProxy.api_url" ).tag(config=True) - - @default('proxy_api_port') - def _proxy_api_port_default(self): - return self.port + 1 + @observe('proxy_api_port', 'proxy_api_ip') + def _deprecated_proxy_api(self, change): + self.log.warning("JupyterHub.%s is deprecated in JupyterHub 0.8, use ConfigurableHTTPProxy.api_url", change.name) + self.config.ConfigurableHTTPProxy.api_url = 'http://{}:{}'.format( + self.proxy_api_ip or '127.0.0.1', + self.proxy_api_port or self.port + 1, + ) hub_port = Integer(8081, help="The port for this process" @@ -682,10 +682,6 @@ class JupyterHub(Application): def init_ports(self): if self.hub_port == self.port: raise TraitError("The hub and proxy cannot both listen on port %i" % self.port) - if self.hub_port == self.proxy_api_port: - raise TraitError("The hub and proxy API cannot both listen on port %i" % self.hub_port) - if self.proxy_api_port == self.port: - raise TraitError("The proxy's public and API ports cannot both be %i" % self.port) @staticmethod def add_url_prefix(prefix, handlers): @@ -805,36 +801,6 @@ class JupyterHub(Application): self._local.db = scoped_session(self.session_factory)() return self._local.db - @property - def hub(self): - if not getattr(self._local, 'hub', None): - q = self.db.query(orm.Hub) - assert q.count() <= 1 - self._local.hub = q.first() - if self.subdomain_host and self._local.hub: - self._local.hub.host = self.subdomain_host - return self._local.hub - - @hub.setter - def hub(self, hub): - self._local.hub = hub - if hub and self.subdomain_host: - hub.host = self.subdomain_host - - @property - def proxy(self): - if not getattr(self._local, 'proxy', None): - q = self.db.query(orm.Proxy) - assert q.count() <= 1 - p = self._local.proxy = q.first() - if p: - p.auth_token = self.proxy_auth_token - return self._local.proxy - - @proxy.setter - def proxy(self, proxy): - self._local.proxy = proxy - def init_db(self): """Create the database connection""" self.log.debug("Connecting to db: %s", self.db_url) @@ -861,28 +827,14 @@ class JupyterHub(Application): def init_hub(self): """Load the Hub config into the database""" - self.hub = self.db.query(orm.Hub).first() - if self.hub is None: - self.hub = orm.Hub( - server=orm.Server( - ip=self.hub_ip, - port=self.hub_port, - base_url=self.hub_prefix, - cookie_name='jupyter-hub-token', - ) - ) - self.db.add(self.hub) - else: - server = self.hub.server - server.ip = self.hub_ip - server.port = self.hub_port - server.base_url = self.hub_prefix - if self.subdomain_host: - if not self.subdomain_host: - raise ValueError("Must specify subdomain_host when using subdomains." - " This should be the public domain[:port] of the Hub.") - - self.db.commit() + self.hub = Hub( + ip=self.hub_ip, + port=self.hub_port, + base_url=self.hub_prefix, + cookie_name='jupyter-hub-token', + public_host=self.subdomain_host, + ) + print(self.hub) @gen.coroutine def init_users(self): @@ -1182,124 +1134,28 @@ class JupyterHub(Application): ) def init_proxy(self): - """Load the Proxy config into the database""" - self.proxy = self.db.query(orm.Proxy).first() - if self.proxy is None: - self.proxy = orm.Proxy( - public_server=orm.Server(), - api_server=orm.Server(), - ) - self.db.add(self.proxy) - self.db.commit() - self.proxy.auth_token = self.proxy_auth_token # not persisted - self.proxy.log = self.log - self.proxy.public_server.ip = self.ip - self.proxy.public_server.port = self.port - self.proxy.public_server.base_url = self.base_url - self.proxy.api_server.ip = self.proxy_api_ip - self.proxy.api_server.port = self.proxy_api_port - self.proxy.api_server.base_url = '/api/routes/' - self.db.commit() - - @gen.coroutine - def start_proxy(self): - """Actually start the configurable-http-proxy""" - # check for proxy - if self.proxy.public_server.is_up() or self.proxy.api_server.is_up(): - # check for *authenticated* access to the proxy (auth token can change) - try: - routes = yield self.proxy.get_routes() - except (HTTPError, OSError, socket.error) as e: - if isinstance(e, HTTPError) and e.code == 403: - msg = "Did CONFIGPROXY_AUTH_TOKEN change?" - else: - msg = "Is something else using %s?" % self.proxy.public_server.bind_url - self.log.error("Proxy appears to be running at %s, but I can't access it (%s)\n%s", - self.proxy.public_server.bind_url, e, msg) - self.exit(1) - return - else: - self.log.info("Proxy already running at: %s", self.proxy.public_server.bind_url) - yield self.proxy.check_routes(self.users, self._service_map, routes) - self.proxy_process = None - return - - env = os.environ.copy() - env['CONFIGPROXY_AUTH_TOKEN'] = self.proxy.auth_token - cmd = self.proxy_cmd + [ - '--ip', self.proxy.public_server.ip, - '--port', str(self.proxy.public_server.port), - '--api-ip', self.proxy.api_server.ip, - '--api-port', str(self.proxy.api_server.port), - '--default-target', self.hub.server.host, - '--error-target', url_path_join(self.hub.server.url, 'error'), - ] - if self.subdomain_host: - cmd.append('--host-routing') - if self.debug_proxy: - cmd.extend(['--log-level', 'debug']) - if self.ssl_key: - cmd.extend(['--ssl-key', self.ssl_key]) - if self.ssl_cert: - cmd.extend(['--ssl-cert', self.ssl_cert]) - if self.statsd_host: - cmd.extend([ - '--statsd-host', self.statsd_host, - '--statsd-port', str(self.statsd_port), - '--statsd-prefix', self.statsd_prefix + '.chp' - ]) - # Warn if SSL is not used - if ' --ssl' not in ' '.join(cmd): - self.log.warning("Running JupyterHub without SSL." - " I hope there is SSL termination happening somewhere else...") - self.log.info("Starting proxy @ %s", self.proxy.public_server.bind_url) - self.log.debug("Proxy cmd: %s", cmd) - try: - self.proxy_process = Popen(cmd, env=env, start_new_session=True) - except FileNotFoundError as e: - self.log.error( - "Failed to find proxy %r\n" - "The proxy can be installed with `npm install -g configurable-http-proxy`" - % self.proxy_cmd - ) - self.exit(1) - - def _check(): - status = self.proxy_process.poll() - if status is not None: - e = RuntimeError("Proxy failed to start with exit code %i" % status) - # py2-compatible `raise e from None` - e.__cause__ = None - raise e - - for server in (self.proxy.public_server, self.proxy.api_server): - for i in range(10): - _check() - try: - yield server.wait_up(1) - except TimeoutError: - continue - else: - break - yield server.wait_up(1) - self.log.debug("Proxy started and appears to be up") - - @gen.coroutine - def check_proxy(self): - if self.proxy_process.poll() is None: - return - self.log.error("Proxy stopped with exit code %r", - 'unknown' if self.proxy_process is None else self.proxy_process.poll() + """Load the Proxy config""" + # FIXME: handle deprecated config here + public_url = 'http{s}://{ip}:{port}{base_url}'.format( + s='s' if self.ssl_cert else '', + ip=self.ip, + port=self.port, + base_url=self.base_url, + ) + self.proxy = self.proxy_class( + db=self.db, + public_url=public_url, + parent=self, + app=self, + log=self.log, + hub=self.hub, + ssl_cert=self.ssl_cert, + ssl_key=self.ssl_key, ) - yield self.start_proxy() - self.log.info("Setting up routes on new proxy") - yield self.proxy.add_all_users(self.users) - yield self.proxy.add_all_services(self.services) - self.log.info("New proxy back up, and good to go") def init_tornado_settings(self): """Set up the tornado settings dict.""" - base_url = self.hub.server.base_url + base_url = self.hub.base_url jinja_options = dict( autoescape=True, ) @@ -1337,7 +1193,7 @@ class JupyterHub(Application): login_url=login_url, logout_url=logout_url, static_path=os.path.join(self.data_files_path, 'static'), - static_url_prefix=url_path_join(self.hub.server.base_url, 'static/'), + static_url_prefix=url_path_join(self.hub.base_url, 'static/'), static_handler_class=CacheControlStaticFilesHandler, template_path=self.template_paths, jinja2_env=jinja_env, @@ -1422,13 +1278,8 @@ class JupyterHub(Application): # clean up proxy while single-user servers are shutting down if self.cleanup_proxy: - if self.proxy_process: - self.log.info("Cleaning up proxy[%i]...", self.proxy_process.pid) - if self.proxy_process.poll() is None: - try: - self.proxy_process.terminate() - except Exception as e: - self.log.error("Failed to terminate proxy process: %s", e) + if self.proxy.should_start: + yield gen.maybe_future(self.proxy.stop()) else: self.log.info("I didn't start the proxy, I can't clean it up") else: @@ -1484,26 +1335,30 @@ class JupyterHub(Application): @gen.coroutine def update_last_activity(self): """Update User.last_activity timestamps from the proxy""" - routes = yield self.proxy.get_routes() + routes = yield self.proxy.get_all_routes() users_count = 0 active_users_count = 0 for prefix, route in routes.items(): - if 'user' not in route: + route_data = route['data'] + if 'user' not in route_data: # not a user route, ignore it continue - user = orm.User.find(self.db, route['user']) + users_count += 1 + if 'last_activity' not in route_data: + # no last activity data (possibly proxy other than CHP) + continue + user = orm.User.find(self.db, route_data['user']) if user is None: self.log.warning("Found no user for route: %s", route) continue try: - dt = datetime.strptime(route['last_activity'], ISO8601_ms) + dt = datetime.strptime(route_data['last_activity'], ISO8601_ms) except Exception: - dt = datetime.strptime(route['last_activity'], ISO8601_s) + dt = datetime.strptime(route_data['last_activity'], ISO8601_s) user.last_activity = max(user.last_activity, dt) # FIXME: Make this configurable duration. 30 minutes for now! if (datetime.now() - user.last_activity).total_seconds() < 30 * 60: active_users_count += 1 - users_count += 1 self.statsd.gauge('users.running', users_count) self.statsd.gauge('users.active', active_users_count) @@ -1530,17 +1385,20 @@ class JupyterHub(Application): try: self.http_server.listen(self.hub_port, address=self.hub_ip) except Exception: - self.log.error("Failed to bind hub to %s", self.hub.server.bind_url) + self.log.error("Failed to bind hub to %s", self.hub.bind_url) raise else: - self.log.info("Hub API listening on %s", self.hub.server.bind_url) + self.log.info("Hub API listening on %s", self.hub.bind_url) # start the proxy - try: - yield self.start_proxy() - except Exception as e: - self.log.critical("Failed to start proxy", exc_info=True) - self.exit(1) + if self.proxy.should_start: + try: + yield self.proxy.start() + except Exception as e: + self.log.critical("Failed to start proxy", exc_info=True) + self.exit(1) + else: + self.log.info("Not starting proxy") # start the service(s) for service_name, service in self._service_map.items(): @@ -1574,12 +1432,6 @@ class JupyterHub(Application): loop.add_callback(self.proxy.add_all_users, self.users) loop.add_callback(self.proxy.add_all_services, self._service_map) - if self.proxy_process: - # only check / restart the proxy if we started it in the first place. - # this means a restarted Hub cannot restart a Proxy that its - # predecessor started. - pc = PeriodicCallback(self.check_proxy, 1e3 * self.proxy_check_interval) - pc.start() if self.service_check_interval and any(s.url for s in self._service_map.values()): pc = PeriodicCallback(self.check_services_health, 1e3 * self.service_check_interval) @@ -1589,7 +1441,7 @@ class JupyterHub(Application): pc = PeriodicCallback(self.update_last_activity, 1e3 * self.last_activity_interval) pc.start() - self.log.info("JupyterHub is now running at %s", self.proxy.public_server.url) + self.log.info("JupyterHub is now running at %s", self.proxy.public_url) # register cleanup on both TERM and INT atexit.register(self.atexit) self.init_signal() diff --git a/jupyterhub/dbutil.py b/jupyterhub/dbutil.py index e2ee2b8e..f104eadd 100644 --- a/jupyterhub/dbutil.py +++ b/jupyterhub/dbutil.py @@ -89,5 +89,4 @@ def _alembic(*args): if __name__ == '__main__': - import sys _alembic(*sys.argv[1:]) diff --git a/jupyterhub/handlers/base.py b/jupyterhub/handlers/base.py index 97c3255a..7288d2bf 100644 --- a/jupyterhub/handlers/base.py +++ b/jupyterhub/handlers/base.py @@ -17,8 +17,9 @@ from tornado.web import RequestHandler from tornado import gen, web from .. import orm -from ..user import User +from ..objects import Server from ..spawner import LocalProcessSpawner +from ..user import User from ..utils import url_path_join # pattern for the authentication token header @@ -103,7 +104,7 @@ class BaseHandler(RequestHandler): @property def csp_report_uri(self): return self.settings.get('csp_report_uri', - url_path_join(self.hub.server.base_url, 'security/csp-report') + url_path_join(self.hub.base_url, 'security/csp-report') ) @property @@ -184,7 +185,7 @@ class BaseHandler(RequestHandler): max_age_days=self.cookie_max_age_days, ) def clear(): - self.clear_cookie(cookie_name, path=self.hub.server.base_url) + self.clear_cookie(cookie_name, path=self.hub.base_url) if cookie_id is None: if self.get_cookie(cookie_name): @@ -208,7 +209,7 @@ class BaseHandler(RequestHandler): def get_current_user_cookie(self): """get_current_user from a cookie token""" - return self._user_for_cookie(self.hub.server.cookie_name) + return self._user_for_cookie(self.hub.cookie_name) def get_current_user(self): """get current username""" @@ -245,9 +246,7 @@ class BaseHandler(RequestHandler): kwargs = {} if self.subdomain_host: kwargs['domain'] = self.domain - if user and user.server: - self.clear_cookie(user.server.cookie_name, path=user.server.base_url, **kwargs) - self.clear_cookie(self.hub.server.cookie_name, path=self.hub.server.base_url, **kwargs) + self.clear_cookie(self.hub.cookie_name, path=self.hub.base_url, **kwargs) self.clear_cookie('jupyterhub-services', path=url_path_join(self.base_url, 'services')) def _set_user_cookie(self, user, server): @@ -436,7 +435,7 @@ class BaseHandler(RequestHandler): def template_namespace(self): user = self.get_current_user() return dict( - base_url=self.hub.server.base_url, + base_url=self.hub.base_url, prefix=self.base_url, user=user, login_url=self.settings['login_url'], @@ -502,7 +501,7 @@ class PrefixRedirectHandler(BaseHandler): else: path = self.request.path self.redirect(url_path_join( - self.hub.server.base_url, path, + self.hub.base_url, path, ), permanent=False) @@ -528,12 +527,12 @@ class UserSpawnHandler(BaseHandler): port = host_info.port if not port: port = 443 if host_info.scheme == 'https' else 80 - if port != self.proxy.public_server.port and port == self.hub.server.port: + if port != Server.from_url(self.proxy.public_url).port and port == self.hub.port: self.log.warning(""" Detected possible direct connection to Hub's private ip: %s, bypassing proxy. This will result in a redirect loop. Make sure to connect to the proxied public URL %s - """, self.request.full_url(), self.proxy.public_server.url) + """, self.request.full_url(), self.proxy.public_url) # logged in as correct user, spawn the server if current_user.spawner: @@ -548,14 +547,14 @@ class UserSpawnHandler(BaseHandler): status = yield current_user.spawner.poll() if status is not None: if current_user.spawner.options_form: - self.redirect(url_concat(url_path_join(self.hub.server.base_url, 'spawn'), + self.redirect(url_concat(url_path_join(self.hub.base_url, 'spawn'), {'next': self.request.uri})) return else: yield self.spawn_single_user(current_user) # set login cookie anew self.set_login_cookie(current_user) - without_prefix = self.request.uri[len(self.hub.server.base_url):] + without_prefix = self.request.uri[len(self.hub.base_url):] target = url_path_join(self.base_url, without_prefix) if self.subdomain_host: target = current_user.host + target diff --git a/jupyterhub/handlers/login.py b/jupyterhub/handlers/login.py index 7ea257a0..d7aff8c0 100644 --- a/jupyterhub/handlers/login.py +++ b/jupyterhub/handlers/login.py @@ -57,7 +57,7 @@ class LoginHandler(BaseHandler): if user.running: next_url = user.url else: - next_url = self.hub.server.base_url + next_url = self.hub.base_url # set new login cookie # because single-user cookie may have been cleared or incorrect self.set_login_cookie(self.get_current_user()) @@ -101,7 +101,7 @@ class LoginHandler(BaseHandler): next_url = self.get_argument('next', default='') if not next_url.startswith('/'): next_url = '' - next_url = next_url or self.hub.server.base_url + next_url = next_url or self.hub.base_url self.redirect(next_url) self.log.info("User logged in: %s", username) else: diff --git a/jupyterhub/handlers/pages.py b/jupyterhub/handlers/pages.py index 3e857943..dbd3a434 100644 --- a/jupyterhub/handlers/pages.py +++ b/jupyterhub/handlers/pages.py @@ -37,7 +37,7 @@ class RootHandler(BaseHandler): # The next request will be handled by UserSpawnHandler, # ultimately redirecting to the logged-in user's server. without_prefix = next_url[len(self.base_url):] - next_url = url_path_join(self.hub.server.base_url, without_prefix) + next_url = url_path_join(self.hub.base_url, without_prefix) self.log.warning("Redirecting %s to %s. For sharing public links, use /user-redirect/", self.request.uri, next_url, ) @@ -50,7 +50,7 @@ class RootHandler(BaseHandler): self.log.debug("User is running: %s", url) self.set_login_cookie(user) # set cookie else: - url = url_path_join(self.hub.server.base_url, 'home') + url = url_path_join(self.hub.base_url, 'home') self.log.debug("User is not running: %s", url) else: url = self.settings['login_url'] @@ -215,7 +215,7 @@ class ProxyErrorHandler(BaseHandler): status_message = responses.get(status_code, 'Unknown HTTP Error') # build template namespace - hub_home = url_path_join(self.hub.server.base_url, 'home') + hub_home = url_path_join(self.hub.base_url, 'home') message_html = '' if status_code == 503: message_html = ' '.join([ diff --git a/jupyterhub/objects.py b/jupyterhub/objects.py new file mode 100644 index 00000000..b4019aaf --- /dev/null +++ b/jupyterhub/objects.py @@ -0,0 +1,134 @@ +"""Some general objects for use in JupyterHub""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +from urllib.parse import urlparse + +from tornado import gen + +from traitlets import ( + HasTraits, Instance, Integer, Unicode, + default, observe, +) +from . import orm +from .utils import ( + url_path_join, can_connect, wait_for_server, + wait_for_http_server, random_port, +) + +class Server(HasTraits): + """An object representing an HTTP endpoint. + + *Some* of these reside in the database (user servers), + but others (Hub, proxy) are in-memory only. + """ + orm_server = Instance(orm.Server, allow_none=True) + + ip = Unicode() + proto = Unicode('http') + port = Integer() + base_url = Unicode('/') + cookie_name = Unicode('') + + @classmethod + def from_url(cls, url): + """Create a Server from a given URL""" + urlinfo = urlparse(url) + proto = urlinfo.scheme + ip = urlinfo.hostname or '' + port = urlinfo.port + if not port: + if proto == 'https': + port = 443 + else: + port = 80 + return cls(proto=proto, ip=ip, port=port, base_url=urlinfo.path) + + @default('port') + def _default_port(self): + return random_port() + + @observe('orm_server') + def _orm_server_changed(self, change): + """When we get an orm_server, get attributes from there.""" + obj = change.new + self.proto = obj.proto + self.ip = obj.ip + self.port = obj.port + self.base_url = obj.base_url + self.cookie_name = obj.cookie_name + + # setter to pass through to the database + @observe('ip', 'proto', 'port', 'base_url', 'cookie_name') + def _change(self, change): + if self.orm_server: + setattr(self.orm_server, change.name, change.new) + + @property + def host(self): + ip = self.ip + if ip in {'', '0.0.0.0'}: + # when listening on all interfaces, connect to localhost + ip = '127.0.0.1' + return "{proto}://{ip}:{port}".format( + proto=self.proto, + ip=ip, + port=self.port, + ) + + @property + def url(self): + return "{host}{uri}".format( + host=self.host, + uri=self.base_url, + ) + + @property + def bind_url(self): + """representation of URL used for binding + + Never used in APIs, only logging, + since it can be non-connectable value, such as '', meaning all interfaces. + """ + if self.ip in {'', '0.0.0.0'}: + return self.url.replace('127.0.0.1', self.ip or '*', 1) + return self.url + + @gen.coroutine + def wait_up(self, timeout=10, http=False): + """Wait for this server to come up""" + if http: + yield wait_for_http_server(self.url, timeout=timeout) + else: + yield wait_for_server(self.ip or '127.0.0.1', self.port, timeout=timeout) + + def is_up(self): + """Is the server accepting connections?""" + return can_connect(self.ip or '127.0.0.1', self.port) + + +class Hub(Server): + """Bring it all together at the hub. + + The Hub is a server, plus its API path suffix + + the api_url is the full URL plus the api_path suffix on the end + of the server base_url. + """ + + @property + def server(self): + """backward-compat""" + return self + public_host = Unicode() + + @property + def api_url(self): + """return the full API url (with proto://host...)""" + return url_path_join(self.url, 'api') + + def __repr__(self): + return "<%s %s:%s>" % ( + self.__class__.__name__, self.server.ip, self.server.port, + ) diff --git a/jupyterhub/orm.py b/jupyterhub/orm.py index d6a0ada4..7ddc4f91 100644 --- a/jupyterhub/orm.py +++ b/jupyterhub/orm.py @@ -79,251 +79,6 @@ class Server(Base): def __repr__(self): return "" % (self.ip, self.port) - @property - def host(self): - ip = self.ip - if ip in {'', '0.0.0.0'}: - # when listening on all interfaces, connect to localhost - ip = '127.0.0.1' - return "{proto}://{ip}:{port}".format( - proto=self.proto, - ip=ip, - port=self.port, - ) - - @property - def url(self): - return "{host}{uri}".format( - host=self.host, - uri=self.base_url, - ) - - @property - def bind_url(self): - """representation of URL used for binding - - Never used in APIs, only logging, - since it can be non-connectable value, such as '', meaning all interfaces. - """ - if self.ip in {'', '0.0.0.0'}: - return self.url.replace('127.0.0.1', self.ip or '*', 1) - return self.url - - @gen.coroutine - def wait_up(self, timeout=10, http=False): - """Wait for this server to come up""" - if http: - yield wait_for_http_server(self.url, timeout=timeout) - else: - yield wait_for_server(self.ip or '127.0.0.1', self.port, timeout=timeout) - - def is_up(self): - """Is the server accepting connections?""" - return can_connect(self.ip or '127.0.0.1', self.port) - - -class Proxy(Base): - """A configurable-http-proxy instance. - - A proxy consists of the API server info and the public-facing server info, - plus an auth token for configuring the proxy table. - """ - __tablename__ = 'proxies' - id = Column(Integer, primary_key=True) - auth_token = None - _public_server_id = Column(Integer, ForeignKey('servers.id')) - public_server = relationship(Server, primaryjoin=_public_server_id == Server.id) - _api_server_id = Column(Integer, ForeignKey('servers.id')) - api_server = relationship(Server, primaryjoin=_api_server_id == Server.id) - - def __repr__(self): - if self.public_server: - return "<%s %s:%s>" % ( - self.__class__.__name__, self.public_server.ip, self.public_server.port, - ) - else: - return "<%s [unconfigured]>" % self.__class__.__name__ - - def api_request(self, path, method='GET', body=None, client=None): - """Make an authenticated API request of the proxy""" - client = client or AsyncHTTPClient() - url = url_path_join(self.api_server.url, path) - - if isinstance(body, dict): - body = json.dumps(body) - self.log.debug("Fetching %s %s", method, url) - req = HTTPRequest(url, - method=method, - headers={'Authorization': 'token {}'.format(self.auth_token)}, - body=body, - ) - - return client.fetch(req) - - @gen.coroutine - def add_service(self, service, client=None): - """Add a service's server to the proxy table.""" - if not service.server: - raise RuntimeError( - "Service %s does not have an http endpoint to add to the proxy.", service.name) - - self.log.info("Adding service %s to proxy %s => %s", - service.name, service.proxy_path, service.server.host, - ) - - yield self.api_request(service.proxy_path, - method='POST', - body=dict( - target=service.server.host, - service=service.name, - ), - client=client, - ) - - @gen.coroutine - def delete_service(self, service, client=None): - """Remove a service's server from the proxy table.""" - self.log.info("Removing service %s from proxy", service.name) - yield self.api_request(service.proxy_path, - method='DELETE', - client=client, - ) - - # FIX-ME - # we need to add a reference to a specific server - @gen.coroutine - def add_user(self, user, client=None): - """Add a user's server to the proxy table.""" - self.log.info("Adding user %s to proxy %s => %s", - user.name, user.proxy_path, user.server.host, - ) - - if user.spawn_pending: - raise RuntimeError( - "User %s's spawn is pending, shouldn't be added to the proxy yet!", user.name) - - yield self.api_request(user.proxy_path, - method='POST', - body=dict( - target=user.server.host, - user=user.name, - ), - client=client, - ) - - @gen.coroutine - def delete_user(self, user, client=None): - """Remove a user's server from the proxy table.""" - self.log.info("Removing user %s from proxy", user.name) - yield self.api_request(user.proxy_path, - method='DELETE', - client=client, - ) - - @gen.coroutine - def add_all_services(self, service_dict): - """Update the proxy table from the database. - - Used when loading up a new proxy. - """ - db = inspect(self).session - futures = [] - for orm_service in db.query(Service): - service = service_dict[orm_service.name] - if service.server: - futures.append(self.add_service(service)) - # wait after submitting them all - for f in futures: - yield f - - @gen.coroutine - def add_all_users(self, user_dict): - """Update the proxy table from the database. - - Used when loading up a new proxy. - """ - db = inspect(self).session - futures = [] - for orm_user in db.query(User): - user = user_dict[orm_user] - if user.running: - futures.append(self.add_user(user)) - # wait after submitting them all - for f in futures: - yield f - - @gen.coroutine - def get_routes(self, client=None): - """Fetch the proxy's routes""" - resp = yield self.api_request('', client=client) - return json.loads(resp.body.decode('utf8', 'replace')) - - # FIX-ME - # we need to add a reference to a specific server - @gen.coroutine - def check_routes(self, user_dict, service_dict, routes=None): - """Check that all users are properly routed on the proxy""" - if not routes: - routes = yield self.get_routes() - - user_routes = { r['user'] for r in routes.values() if 'user' in r } - futures = [] - db = inspect(self).session - for orm_user in db.query(User): - user = user_dict[orm_user] - if user.running: - if user.name not in user_routes: - self.log.warning("Adding missing route for %s (%s)", user.name, user.server) - futures.append(self.add_user(user)) - else: - # User not running, make sure it's not in the table - if user.name in user_routes: - self.log.warning("Removing route for not running %s", user.name) - futures.append(self.delete_user(user)) - - # check service routes - service_routes = { r['service'] for r in routes.values() if 'service' in r } - for orm_service in db.query(Service).filter(Service.server != None): - service = service_dict[orm_service.name] - if service.server is None: - # This should never be True, but seems to be on rare occasion. - # catch filter bug, either in sqlalchemy or my understanding of its behavior - self.log.error("Service %s has no server, but wasn't filtered out.", service) - continue - if service.name not in service_routes: - self.log.warning("Adding missing route for %s (%s)", service.name, service.server) - futures.append(self.add_service(service)) - for f in futures: - yield f - - -class Hub(Base): - """Bring it all together at the hub. - - The Hub is a server, plus its API path suffix - - the api_url is the full URL plus the api_path suffix on the end - of the server base_url. - """ - __tablename__ = 'hubs' - id = Column(Integer, primary_key=True) - _server_id = Column(Integer, ForeignKey('servers.id')) - server = relationship(Server, primaryjoin=_server_id == Server.id) - host = '' - - @property - def api_url(self): - """return the full API url (with proto://host...)""" - return url_path_join(self.server.url, 'api') - - def __repr__(self): - if self.server: - return "<%s %s:%s>" % ( - self.__class__.__name__, self.server.ip, self.server.port, - ) - else: - return "<%s [unconfigured]>" % self.__class__.__name__ - # user:group many:many mapping table user_group_map = Table('user_group_map', Base.metadata, @@ -393,23 +148,14 @@ class User(Base): # group mapping groups = relationship('Group', secondary='user_group_map', back_populates='users') - @property - def server(self): - """Returns the first element of servers. - Returns None if the list is empty. - """ - if len(self.servers) == 0: - return None - else: - return self.servers[0] - def __repr__(self): - if self.server: + if self.servers: + server = self.servers[0] return "<{cls}({name}@{ip}:{port})>".format( cls=self.__class__.__name__, name=self.name, - ip=self.server.ip, - port=self.server.port, + ip=server.ip, + port=server.port, ) else: return "<{cls}({name} [unconfigured])>".format( diff --git a/jupyterhub/proxy.py b/jupyterhub/proxy.py new file mode 100644 index 00000000..48d4f1b3 --- /dev/null +++ b/jupyterhub/proxy.py @@ -0,0 +1,416 @@ +"""API for JupyterHub's proxy.""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +import json +import os +from subprocess import Popen +import time + +from tornado import gen +from tornado.httpclient import AsyncHTTPClient, HTTPRequest +from tornado.ioloop import PeriodicCallback + + +from traitlets import ( + Any, Bool, Instance, Integer, Unicode, + default, +) +from jupyterhub.traitlets import Command + +from traitlets.config import LoggingConfigurable +from .objects import Server +from .orm import Service, User +from . import utils +from .utils import url_path_join + + +class Proxy(LoggingConfigurable): + """Base class for configurable proxies that JupyterHub can use.""" + + db = Any() + app = Any() + hub = Any() + public_url = Unicode() + ssl_key = Unicode() + ssl_cert = Unicode() + + should_start = Bool(True, config=True, + help="""Should the Hub start the proxy. + + If True, the Hub will start the proxy and stop it. + Set to False if the proxy is managed externally, + such as by systemd, docker, or another service manager. + """) + + def start(self): + """Start the proxy. + + Will be called during startup if should_start is True. + """ + + def stop(self): + """Stop the proxy. + + Will be called during teardown if should_start is True. + """ + + @gen.coroutine + def add_route(self, routespec, target, data): + """Add a route to the proxy. + + Args: + routespec (str): A specification for which this route will be matched. + Could be either a url_prefix or a fqdn. + target (str): A URL that will be the target of this route. + data (dict): A JSONable dict that will be associated with this route, and will + be returned when retrieving information about this route. + + Will raise an appropriate Exception (FIXME: find what?) if the route could + not be added. + + The proxy implementation should also have a way to associate the fact that a + route came from JupyterHub. + """ + pass + + @gen.coroutine + def delete_route(self, routespec): + """Delete a route with a given routespec if it exists.""" + pass + + @gen.coroutine + def get_route(self, routespec): + """Return the route info for a given routespec. + + Args: + routespec (str): The route specification that was used to add this routespec + + Returns: + result (dict): with the following keys: + `routespec`: The normalized route specification passed in to add_route + `target`: The target for this route + `data`: The arbitrary data that was passed in by JupyterHub when adding this + route. + None: if there are no routes matching the given routespec + """ + pass + + @gen.coroutine + def get_all_routes(self): + """Fetch and return all the routes associated by JupyterHub from the + proxy. + + Should return a dictionary of routes, where the keys are + routespecs and each value is the dict that would be returned by + `get_route(routespec)`. + """ + pass + + # Most basic implementers must only implement above methods + + @gen.coroutine + def add_service(self, service, client=None): + """Add a service's server to the proxy table.""" + if not service.server: + raise RuntimeError( + "Service %s does not have an http endpoint to add to the proxy.", service.name) + + self.log.info("Adding service %s to proxy %s => %s", + service.name, service.proxy_path, service.server.host, + ) + + yield self.add_route( + service.proxy_path, + service.server.host, + {'service': service.name} + ) + + @gen.coroutine + def delete_service(self, service, client=None): + """Remove a service's server from the proxy table.""" + self.log.info("Removing service %s from proxy", service.name) + yield self.delete_route(service.proxy_path) + + @gen.coroutine + def add_user(self, user, client=None): + """Add a user's server to the proxy table.""" + self.log.info("Adding user %s to proxy %s => %s", + user.name, user.proxy_path, user.server.host, + ) + + if user.spawn_pending: + raise RuntimeError( + "User %s's spawn is pending, shouldn't be added to the proxy yet!", user.name) + + yield self.add_route( + user.proxy_path, + user.server.host, + {'user': user.name} + ) + + @gen.coroutine + def delete_user(self, user): + """Remove a user's server from the proxy table.""" + self.log.info("Removing user %s from proxy", user.name) + yield self.delete_route(user.proxy_path) + + @gen.coroutine + def add_all_services(self, service_dict): + """Update the proxy table from the database. + + Used when loading up a new proxy. + """ + db = self.db + futures = [] + for orm_service in db.query(Service): + service = service_dict[orm_service.name] + if service.server: + futures.append(self.add_service(service)) + # wait after submitting them all + for f in futures: + yield f + + @gen.coroutine + def add_all_users(self, user_dict): + """Update the proxy table from the database. + + Used when loading up a new proxy. + """ + db = self.db + futures = [] + for orm_user in db.query(User): + user = user_dict[orm_user] + if user.running: + futures.append(self.add_user(user)) + # wait after submitting them all + for f in futures: + yield f + + @gen.coroutine + def check_routes(self, user_dict, service_dict, routes=None): + """Check that all users are properly routed on the proxy.""" + if not routes: + routes = yield self.get_all_routes() + + user_routes = {r['data']['user'] for r in routes.values() if 'user' in r['data']} + futures = [] + db = self.db + for orm_user in db.query(User): + user = user_dict[orm_user] + if user.running: + if user.name not in user_routes: + self.log.warning( + "Adding missing route for %s (%s)", user.name, user.server) + futures.append(self.add_user(user)) + else: + # User not running, make sure it's not in the table + if user.name in user_routes: + self.log.warning( + "Removing route for not running %s", user.name) + futures.append(self.delete_user(user)) + + # check service routes + service_routes = {r['data']['service'] + for r in routes.values() if 'service' in r['data']} + for orm_service in db.query(Service).filter( + Service.server is not None): + service = service_dict[orm_service.name] + if service.server is None: + # This should never be True, but seems to be on rare occasion. + # catch filter bug, either in sqlalchemy or my understanding of + # its behavior + self.log.error( + "Service %s has no server, but wasn't filtered out.", service) + continue + if service.name not in service_routes: + self.log.warning("Adding missing route for %s (%s)", + service.name, service.server) + futures.append(self.add_service(service)) + for f in futures: + yield f + + @gen.coroutine + def restore_routes(self): + self.log.info("Setting up routes on new proxy") + yield self.add_all_users(self.app.users) + yield self.add_all_services(self.app.services) + self.log.info("New proxy back up, and good to go") + + +class ConfigurableHTTPProxy(Proxy): + """Proxy implementation for the default configurable-http-proxy.""" + + proxy_process = Any() + client = Instance(AsyncHTTPClient, ()) + + debug = Bool(False, help="Add debug-level logging to the Proxy", config=True) + auth_token = Unicode( + help="""The Proxy Auth token. + + Loaded from the CONFIGPROXY_AUTH_TOKEN env variable by default. + """, + ).tag(config=True) + check_running_interval = Integer(5, config=True) + + @default('auth_token') + def _auth_token_default(self): + token = os.environ.get('CONFIGPROXY_AUTH_TOKEN', None) + if not token: + self.log.warning('\n'.join([ + "", + "Generating CONFIGPROXY_AUTH_TOKEN. Restarting the Hub will require restarting the proxy.", + "Set CONFIGPROXY_AUTH_TOKEN env or JupyterHub.proxy_auth_token config to avoid this message.", + "", + ])) + token = utils.new_token() + return token + + api_url = Unicode('http://127.0.0.1:8001', config=True, + help="""The ip (or hostname) of the proxy's API endpoint""" + ) + command = Command('configurable-http-proxy', config=True, + help="""The command to start the proxy""" + ) + + @gen.coroutine + def start(self): + public_server = Server.from_url(self.public_url) + api_server = Server.from_url(self.api_url) + env = os.environ.copy() + env['CONFIGPROXY_AUTH_TOKEN'] = self.auth_token + cmd = self.command + [ + '--ip', public_server.ip, + '--port', str(public_server.port), + '--api-ip', api_server.ip, + '--api-port', str(api_server.port), + '--default-target', self.hub.host, + '--error-target', url_path_join(self.hub.url, 'error'), + ] + if self.app.subdomain_host: + cmd.append('--host-routing') + if self.debug: + cmd.extend(['--log-level', 'debug']) + if self.ssl_key: + cmd.extend(['--ssl-key', self.ssl_key]) + if self.ssl_cert: + cmd.extend(['--ssl-cert', self.ssl_cert]) + if self.app.statsd_host: + cmd.extend([ + '--statsd-host', self.app.statsd_host, + '--statsd-port', str(self.app.statsd_port), + '--statsd-prefix', self.app.statsd_prefix + '.chp' + ]) + # Warn if SSL is not used + if ' --ssl' not in ' '.join(cmd): + self.log.warning("Running JupyterHub without SSL." + " I hope there is SSL termination happening somewhere else...") + self.log.info("Starting proxy @ %s", public_server.bind_url) + self.log.debug("Proxy cmd: %s", cmd) + try: + self.proxy_process = Popen(cmd, env=env, start_new_session=True) + except FileNotFoundError as e: + self.log.error( + "Failed to find proxy %r\n" + "The proxy can be installed with `npm install -g configurable-http-proxy`" + % self.cmd + ) + self.exit(1) + + def _check_process(): + status = self.proxy_process.poll() + if status is not None: + e = RuntimeError( + "Proxy failed to start with exit code %i" % status) + # py2-compatible `raise e from None` + e.__cause__ = None + raise e + + for server in (public_server, api_server): + for i in range(10): + _check_process() + try: + yield server.wait_up(1) + except TimeoutError: + continue + else: + break + yield server.wait_up(1) + time.sleep(1) + _check_process() + self.log.debug("Proxy started and appears to be up") + pc = PeriodicCallback(self.check_running, 1e3 * self.check_running_interval) + pc.start() + + def stop(self): + self.log.info("Cleaning up proxy[%i]...", self.proxy_process.pid) + if self.proxy_process.poll() is None: + try: + self.proxy_process.terminate() + except Exception as e: + self.log.error("Failed to terminate proxy process: %s", e) + + @gen.coroutine + def check_running(self): + """Check if the proxy is still running""" + if self.proxy_process.poll() is None: + return + self.log.error("Proxy stopped with exit code %r", + 'unknown' if self.proxy_process is None else self.proxy_process.poll() + ) + yield self.start() + yield self.restore_routes() + + def api_request(self, path, method='GET', body=None, client=None): + """Make an authenticated API request of the proxy.""" + client = client or AsyncHTTPClient() + url = url_path_join(self.api_url, 'api/routes', path) + + if isinstance(body, dict): + body = json.dumps(body) + self.log.debug("Proxy: Fetching %s %s", method, url) + req = HTTPRequest(url, + method=method, + headers={'Authorization': 'token {}'.format( + self.auth_token)}, + body=body, + ) + + return client.fetch(req) + + def add_route(self, routespec, target, data=None): + body = data or {} + body['target'] = target + return self.api_request(routespec, + method='POST', + body=body, + ) + + def delete_route(self, routespec): + return self.api_request(routespec, method='DELETE') + + def _reformat_routespec(self, routespec, chp_data): + """Reformat CHP data format to JupyterHub's proxy API.""" + target = chp_data.pop('target') + return { + 'routespec': routespec, + 'target': target, + 'data': chp_data, + } + + @gen.coroutine + def get_route(self, routespec): + chp_data = yield self.api_request(routespec, method='DELETE') + return self._reformat_routespec(routespec, chp_data) + + @gen.coroutine + def get_all_routes(self, client=None): + """Fetch the proxy's routes.""" + resp = yield self.api_request('', client=client) + chp_routes = json.loads(resp.body.decode('utf8', 'replace')) + all_routes = {} + for routespec, chp_data in chp_routes.items(): + all_routes[routespec] = self._reformat_routespec( + routespec, chp_data) + return all_routes diff --git a/jupyterhub/services/service.py b/jupyterhub/services/service.py index 916f8e15..1e5479a6 100644 --- a/jupyterhub/services/service.py +++ b/jupyterhub/services/service.py @@ -52,6 +52,7 @@ from traitlets import ( from traitlets.config import LoggingConfigurable from .. import orm +from ..objects import Server from ..traitlets import Command from ..spawner import LocalProcessSpawner, set_user_setuid from ..utils import url_path_join @@ -60,7 +61,7 @@ class _MockUser(HasTraits): name = Unicode() server = Instance(orm.Server, allow_none=True) state = Dict() - service = Instance(__module__ + '.Service') + service = Instance(__name__ + '.Service') host = Unicode() @property @@ -221,7 +222,10 @@ class Service(LoggingConfigurable): @property def server(self): - return self.orm.server + if self.orm.server: + return Server(orm_server=self.orm.server) + else: + return None @property def prefix(self): diff --git a/jupyterhub/singleuser.py b/jupyterhub/singleuser.py index 14df66f2..a77fc4ac 100755 --- a/jupyterhub/singleuser.py +++ b/jupyterhub/singleuser.py @@ -228,12 +228,6 @@ class SingleUserNotebookApp(NotebookApp): value = value + '/' return value - @default('cookie_name') - def _cookie_name_default(self): - if os.environ.get('JUPYTERHUB_SERVICE_NAME'): - # if I'm a service, use the services cookie name - return 'jupyterhub-services' - @default('port') def _port_default(self): if os.environ.get('JUPYTERHUB_SERVICE_URL'): diff --git a/jupyterhub/spawner.py b/jupyterhub/spawner.py index 4b2bfb3c..5515e493 100644 --- a/jupyterhub/spawner.py +++ b/jupyterhub/spawner.py @@ -431,7 +431,7 @@ class Spawner(LoggingConfigurable): env['JUPYTERHUB_ADMIN_ACCESS'] = '1' # OAuth settings env['JUPYTERHUB_CLIENT_ID'] = self.oauth_client_id - env['JUPYTERHUB_HOST'] = self.hub.host + env['JUPYTERHUB_HOST'] = self.hub.public_host env['JUPYTERHUB_OAUTH_CALLBACK_URL'] = \ url_path_join(self.user.url, 'oauth_callback') @@ -496,8 +496,8 @@ class Spawner(LoggingConfigurable): args = [ '--user="%s"' % self.user.name, '--base-url="%s"' % self.user.server.base_url, - '--hub-host="%s"' % self.hub.host, - '--hub-prefix="%s"' % self.hub.server.base_url, + '--hub-host="%s"' % self.hub.public_host, + '--hub-prefix="%s"' % self.hub.base_url, '--hub-api-url="%s"' % self.hub.api_url, ] if self.ip: diff --git a/jupyterhub/tests/conftest.py b/jupyterhub/tests/conftest.py index b214ac3d..66377426 100644 --- a/jupyterhub/tests/conftest.py +++ b/jupyterhub/tests/conftest.py @@ -9,7 +9,7 @@ from subprocess import TimeoutExpired import time from unittest import mock from pytest import fixture, raises -from tornado import ioloop +from tornado import ioloop, gen from .. import orm from ..utils import random_port @@ -32,11 +32,7 @@ def db(): name=getuser(), ) user.servers.append(orm.Server()) - hub = orm.Hub( - server=orm.Server(), - ) _db.add(user) - _db.add(hub) _db.commit() return _db @@ -85,10 +81,14 @@ def _mockservice(request, app, url=False): with mock.patch.object(jupyterhub.services.service, '_ServiceSpawner', MockServiceSpawner): app.services = [spec] app.init_services() - app.io_loop.add_callback(app.proxy.add_all_services, app._service_map) assert name in app._service_map service = app._service_map[name] - app.io_loop.add_callback(service.start) + @gen.coroutine + def start(): + # wait for proxy to be updated before starting the service + yield app.proxy.add_all_services(app._service_map) + service.start() + app.io_loop.add_callback(start) def cleanup(): service.stop() app.services[:] = [] @@ -100,6 +100,8 @@ def _mockservice(request, app, url=False): # ensure process finishes starting with raises(TimeoutExpired): service.proc.wait(1) + if url: + ioloop.IOLoop().run_sync(service.server.wait_up) return service diff --git a/jupyterhub/tests/mocking.py b/jupyterhub/tests/mocking.py index 9045357a..f8e4628a 100644 --- a/jupyterhub/tests/mocking.py +++ b/jupyterhub/tests/mocking.py @@ -4,7 +4,6 @@ import os import sys from tempfile import NamedTemporaryFile import threading - from unittest import mock import requests @@ -18,6 +17,7 @@ from traitlets import default from ..app import JupyterHub from ..auth import PAMAuthenticator from .. import orm +from ..objects import Server from ..spawner import LocalProcessSpawner from ..singleuser import SingleUserNotebookApp from ..utils import random_port, url_path_join @@ -165,7 +165,7 @@ class MockHub(JupyterHub): self.db.add(user) self.db.commit() yield super(MockHub, self).start() - yield self.hub.server.wait_up(http=True) + yield self.hub.wait_up(http=True) self.io_loop.add_callback(evt.set) def _start(): @@ -207,7 +207,7 @@ def public_host(app): if app.subdomain_host: return app.subdomain_host else: - return app.proxy.public_server.host + return Server.from_url(app.proxy.public_url).host def public_url(app, user_or_service=None, path=''): @@ -220,7 +220,7 @@ def public_url(app, user_or_service=None, path=''): prefix = user_or_service.server.base_url else: host = public_host(app) - prefix = app.proxy.public_server.base_url + prefix = Server.from_url(app.proxy.public_url).base_url if path: return host + url_path_join(prefix, path) else: @@ -246,7 +246,8 @@ class StubSingleUserSpawner(MockSpawner): _thread = None @gen.coroutine def start(self): - self.user.server.port = random_port() + ip = self.user.server.ip + port = self.user.server.port = random_port() env = self.get_env() args = self.get_args() evt = threading.Event() @@ -267,6 +268,7 @@ class StubSingleUserSpawner(MockSpawner): self._thread.start() ready = evt.wait(timeout=3) assert ready + return (ip, port) @gen.coroutine def stop(self): diff --git a/jupyterhub/tests/test_api.py b/jupyterhub/tests/test_api.py index 512f81d6..0f0d1a64 100644 --- a/jupyterhub/tests/test_api.py +++ b/jupyterhub/tests/test_api.py @@ -83,7 +83,7 @@ def auth_header(db, name): @check_db_locks def api_request(app, *api_path, **kwargs): """Make an API request""" - base_url = app.hub.server.url + base_url = app.hub.url headers = kwargs.setdefault('headers', {}) if 'Authorization' not in headers: @@ -94,7 +94,7 @@ def api_request(app, *api_path, **kwargs): f = getattr(requests, method) resp = f(url, **kwargs) assert "frame-ancestors 'self'" in resp.headers['Content-Security-Policy'] - assert ujoin(app.hub.server.base_url, "security/csp-report") in resp.headers['Content-Security-Policy'] + assert ujoin(app.hub.base_url, "security/csp-report") in resp.headers['Content-Security-Policy'] assert 'http' not in resp.headers['Content-Security-Policy'] return resp @@ -132,7 +132,7 @@ def test_auth_api(app): def test_referer_check(app, io_loop): - url = ujoin(public_host(app), app.hub.server.base_url) + url = ujoin(public_host(app), app.hub.base_url) host = urlparse(url).netloc user = find_user(app.db, 'admin') if user is None: @@ -779,7 +779,7 @@ def test_get_service(app, mockservice_url): def test_root_api(app): - base_url = app.hub.server.url + base_url = app.hub.url url = ujoin(base_url, 'api') r = requests.get(url) r.raise_for_status() diff --git a/jupyterhub/tests/test_orm.py b/jupyterhub/tests/test_orm.py index 531e3523..d873cef2 100644 --- a/jupyterhub/tests/test_orm.py +++ b/jupyterhub/tests/test_orm.py @@ -7,6 +7,7 @@ import pytest from tornado import gen from .. import orm +from .. import objects from ..user import User from .mocking import MockSpawner @@ -20,6 +21,9 @@ def test_server(db): assert server.proto == 'http' assert isinstance(server.port, int) assert isinstance(server.cookie_name, str) + + # test wrapper + server = objects.Server(orm_server=server) assert server.host == 'http://127.0.0.1:%i' % server.port assert server.url == server.host + '/' assert server.bind_url == 'http://*:%i/' % server.port @@ -28,45 +32,10 @@ def test_server(db): assert server.url == server.host + '/' -def test_proxy(db): - proxy = orm.Proxy( - auth_token='abc-123', - public_server=orm.Server( - ip='192.168.1.1', - port=8000, - ), - api_server=orm.Server( - ip='127.0.0.1', - port=8001, - ), - ) - db.add(proxy) - db.commit() - assert proxy.public_server.ip == '192.168.1.1' - assert proxy.api_server.ip == '127.0.0.1' - assert proxy.auth_token == 'abc-123' - - -def test_hub(db): - hub = orm.Hub( - server=orm.Server( - ip = '1.2.3.4', - port = 1234, - base_url='/hubtest/', - ), - - ) - db.add(hub) - db.commit() - assert hub.server.ip == '1.2.3.4' - assert hub.server.port == 1234 - assert hub.api_url == 'http://1.2.3.4:1234/hubtest/api' - - def test_user(db): - user = orm.User(name='kaylee', + user = User(orm.User(name='kaylee', state={'pid': 4234}, - ) + )) server = orm.Server() user.servers.append(server) db.add(user) diff --git a/jupyterhub/tests/test_pages.py b/jupyterhub/tests/test_pages.py index 0f0d3b3c..72f4b34e 100644 --- a/jupyterhub/tests/test_pages.py +++ b/jupyterhub/tests/test_pages.py @@ -16,7 +16,7 @@ from .test_api import api_request def get_page(path, app, hub=True, **kw): if hub: - prefix = app.hub.server.base_url + prefix = app.hub.base_url else: prefix = app.base_url base_url = ujoin(public_host(app), prefix) @@ -24,11 +24,11 @@ def get_page(path, app, hub=True, **kw): return requests.get(ujoin(base_url, path), **kw) def test_root_no_auth(app, io_loop): - print(app.hub.server.is_up()) - routes = io_loop.run_sync(app.proxy.get_routes) + print(app.hub.is_up()) + routes = io_loop.run_sync(app.proxy.get_all_routes) print(routes) print(app.hub.server) - url = ujoin(public_host(app), app.hub.server.base_url) + url = ujoin(public_host(app), app.hub.base_url) print(url) r = requests.get(url) r.raise_for_status() @@ -123,7 +123,7 @@ def test_spawn_page(app): def test_spawn_form(app, io_loop): with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): - base_url = ujoin(public_host(app), app.hub.server.base_url) + base_url = ujoin(public_host(app), app.hub.base_url) cookies = app.login_user('jones') orm_u = orm.User.find(app.db, 'jones') u = app.users[orm_u] @@ -145,7 +145,7 @@ def test_spawn_form(app, io_loop): def test_spawn_form_with_file(app, io_loop): with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}): - base_url = ujoin(public_host(app), app.hub.server.base_url) + base_url = ujoin(public_host(app), app.hub.base_url) cookies = app.login_user('jones') orm_u = orm.User.find(app.db, 'jones') u = app.users[orm_u] @@ -181,7 +181,7 @@ def test_user_redirect(app): assert path == ujoin(app.base_url, '/hub/login') query = urlparse(r.url).query assert query == urlencode({ - 'next': ujoin(app.hub.server.base_url, '/user-redirect/tree/top/') + 'next': ujoin(app.hub.base_url, '/user-redirect/tree/top/') }) r = get_page('/user-redirect/notebooks/test.ipynb', app, cookies=cookies) @@ -320,7 +320,7 @@ def test_login_no_whitelist_adds_user(app): def test_static_files(app): - base_url = ujoin(public_host(app), app.hub.server.base_url) + base_url = ujoin(public_host(app), app.hub.base_url) r = requests.get(ujoin(base_url, 'logo')) r.raise_for_status() assert r.headers['content-type'] == 'image/png' diff --git a/jupyterhub/tests/test_proxy.py b/jupyterhub/tests/test_proxy.py index f3cd60fa..befb0291 100644 --- a/jupyterhub/tests/test_proxy.py +++ b/jupyterhub/tests/test_proxy.py @@ -6,6 +6,8 @@ from queue import Queue from subprocess import Popen from urllib.parse import urlparse, unquote +from traitlets.config import Config + import pytest from .. import orm @@ -19,12 +21,12 @@ def test_external_proxy(request, io_loop): auth_token = 'secret!' proxy_ip = '127.0.0.1' proxy_port = 54321 + cfg = Config() + cfg.ConfigurableHTTPProxy.auth_token = auth_token + cfg.ConfigurableHTTPProxy.api_url = 'http://%s:%i' % (proxy_ip, proxy_port) + cfg.ConfigurableHTTPProxy.should_start = False - app = MockHub.instance( - proxy_api_ip=proxy_ip, - proxy_api_port=proxy_port, - proxy_auth_token=auth_token, - ) + app = MockHub.instance(config=cfg) def fin(): MockHub.clear_instance() @@ -35,7 +37,8 @@ def test_external_proxy(request, io_loop): # configures and starts proxy process env = os.environ.copy() env['CONFIGPROXY_AUTH_TOKEN'] = auth_token - cmd = app.proxy_cmd + [ + cmd = [ + 'configurable-http-proxy', '--ip', app.ip, '--port', str(app.port), '--api-ip', proxy_ip, @@ -57,10 +60,10 @@ def test_external_proxy(request, io_loop): wait_for_proxy() app.start([]) - assert app.proxy_process is None + assert app.proxy.proxy_process is None # test if api service has a root route '/' - routes = io_loop.run_sync(app.proxy.get_routes) + routes = io_loop.run_sync(app.proxy.get_all_routes) assert list(routes.keys()) == ['/'] # add user to the db and start a single user server @@ -70,7 +73,7 @@ def test_external_proxy(request, io_loop): r = api_request(app, 'users', name, 'server', method='post') r.raise_for_status() - routes = io_loop.run_sync(app.proxy.get_routes) + routes = io_loop.run_sync(app.proxy.get_all_routes) # sets the desired path result user_path = unquote(ujoin(app.base_url, 'user/river')) if app.subdomain_host: @@ -83,7 +86,8 @@ def test_external_proxy(request, io_loop): proxy = Popen(cmd, env=env) wait_for_proxy() - routes = io_loop.run_sync(app.proxy.get_routes) + routes = io_loop.run_sync(app.proxy.get_all_routes) + assert list(routes.keys()) == ['/'] # poke the server to update the proxy @@ -91,7 +95,7 @@ def test_external_proxy(request, io_loop): r.raise_for_status() # check that the routes are correct - routes = io_loop.run_sync(app.proxy.get_routes) + routes = io_loop.run_sync(app.proxy.get_all_routes) assert sorted(routes.keys()) == ['/', user_path] # teardown the proxy, and start a new one with different auth and port @@ -99,10 +103,10 @@ def test_external_proxy(request, io_loop): new_auth_token = 'different!' env['CONFIGPROXY_AUTH_TOKEN'] = new_auth_token proxy_port = 55432 - cmd = app.proxy_cmd + [ + cmd = ['configurable-http-proxy', '--ip', app.ip, '--port', str(app.port), - '--api-ip', app.proxy_api_ip, + '--api-ip', proxy_ip, '--api-port', str(proxy_port), '--default-target', 'http://%s:%i' % (app.hub_ip, app.hub_port), ] @@ -112,14 +116,13 @@ def test_external_proxy(request, io_loop): wait_for_proxy() # tell the hub where the new proxy is + new_api_url = 'http://{}:{}'.format(proxy_ip, proxy_port) r = api_request(app, 'proxy', method='patch', data=json.dumps({ - 'port': proxy_port, - 'protocol': 'http', - 'ip': app.ip, + 'api_url': new_api_url, 'auth_token': new_auth_token, })) r.raise_for_status() - assert app.proxy.api_server.port == proxy_port + assert app.proxy.api_url == new_api_url # get updated auth token from main thread def get_app_proxy_token(): @@ -131,7 +134,7 @@ def test_external_proxy(request, io_loop): app.proxy.auth_token = new_auth_token # check that the routes are correct - routes = io_loop.run_sync(app.proxy.get_routes) + routes = io_loop.run_sync(app.proxy.get_all_routes) assert sorted(routes.keys()) == ['/', user_path] @@ -152,18 +155,18 @@ def test_check_routes(app, io_loop, username, endpoints): # check a valid route exists for user test_user = app.users[username] - before = sorted(io_loop.run_sync(app.proxy.get_routes)) + before = sorted(io_loop.run_sync(app.proxy.get_all_routes)) assert unquote(test_user.proxy_path) in before # check if a route is removed when user deleted io_loop.run_sync(lambda: app.proxy.check_routes(app.users, app._service_map)) io_loop.run_sync(lambda: proxy.delete_user(test_user)) - during = sorted(io_loop.run_sync(app.proxy.get_routes)) + during = sorted(io_loop.run_sync(app.proxy.get_all_routes)) assert unquote(test_user.proxy_path) not in during # check if a route exists for user io_loop.run_sync(lambda: app.proxy.check_routes(app.users, app._service_map)) - after = sorted(io_loop.run_sync(app.proxy.get_routes)) + after = sorted(io_loop.run_sync(app.proxy.get_all_routes)) assert unquote(test_user.proxy_path) in after # check that before and after state are the same diff --git a/jupyterhub/tests/test_services.py b/jupyterhub/tests/test_services.py index 337f2cc3..f039c0df 100644 --- a/jupyterhub/tests/test_services.py +++ b/jupyterhub/tests/test_services.py @@ -25,7 +25,7 @@ def external_service(app, name='mockservice'): env = { 'JUPYTERHUB_API_TOKEN': hexlify(os.urandom(5)), 'JUPYTERHUB_SERVICE_NAME': name, - 'JUPYTERHUB_API_URL': url_path_join(app.hub.server.url, 'api/'), + 'JUPYTERHUB_API_URL': url_path_join(app.hub.url, 'api/'), 'JUPYTERHUB_SERVICE_URL': 'http://127.0.0.1:%i' % random_port(), } proc = Popen(mockservice_cmd, env=env) @@ -64,7 +64,7 @@ def test_managed_service(mockservice): def test_proxy_service(app, mockservice_url, io_loop): service = mockservice_url name = service.name - io_loop.run_sync(app.proxy.get_routes) + io_loop.run_sync(app.proxy.get_all_routes) url = public_url(app, service) + '/foo' r = requests.get(url, allow_redirects=False) path = '/services/{}/foo'.format(name) diff --git a/jupyterhub/tests/test_spawner.py b/jupyterhub/tests/test_spawner.py index c22539bc..6ca5581d 100644 --- a/jupyterhub/tests/test_spawner.py +++ b/jupyterhub/tests/test_spawner.py @@ -17,6 +17,7 @@ import requests from tornado import gen from ..user import User +from ..objects import Hub from .. import spawner as spawnermod from ..spawner import LocalProcessSpawner from .. import orm @@ -43,8 +44,8 @@ def setup(): def new_spawner(db, **kwargs): kwargs.setdefault('cmd', [sys.executable, '-c', _echo_sleep]) + kwargs.setdefault('hub', Hub()) kwargs.setdefault('user', User(db.query(orm.User).first(), {})) - kwargs.setdefault('hub', db.query(orm.Hub).first()) kwargs.setdefault('notebook_dir', os.getcwd()) kwargs.setdefault('default_url', '/user/{username}/lab') kwargs.setdefault('INTERRUPT_TIMEOUT', 1) diff --git a/jupyterhub/user.py b/jupyterhub/user.py index c0727d31..347230d7 100644 --- a/jupyterhub/user.py +++ b/jupyterhub/user.py @@ -9,9 +9,10 @@ from sqlalchemy import inspect from tornado import gen from tornado.log import app_log -from .utils import url_path_join, default_server_name, new_token +from .utils import url_path_join, default_server_name from . import orm +from .objects import Server from traitlets import HasTraits, Any, Dict, observe, default from .spawner import LocalProcessSpawner @@ -111,23 +112,20 @@ class User(HasTraits): def spawner_class(self): return self.settings.get('spawner_class', LocalProcessSpawner) - def __init__(self, orm_user, settings, **kwargs): + def __init__(self, orm_user, settings=None, **kwargs): self.orm_user = orm_user - self.settings = settings + self.settings = settings or {} super().__init__(**kwargs) - hub = self.db.query(orm.Hub).first() - self.allow_named_servers = self.settings.get('allow_named_servers', False) - self.cookie_name = '%s-%s' % (hub.server.cookie_name, quote(self.name, safe='')) self.base_url = url_path_join( self.settings.get('base_url', '/'), 'user', self.escaped_name) self.spawner = self.spawner_class( user=self, db=self.db, - hub=hub, + hub=self.settings.get('hub'), authenticator=self.authenticator, config=self.settings.get('config'), ) @@ -157,6 +155,13 @@ class User(HasTraits): if self.server is None: return False return True + + @property + def server(self): + if len(self.servers) == 0: + return None + else: + return Server(orm_server=self.servers[0]) @property def escaped_name(self): @@ -223,18 +228,17 @@ class User(HasTraits): server_name = '' base_url = self.base_url - server = orm.Server( - name = server_name, - cookie_name=self.cookie_name, + orm_server = orm.Server( + name=server_name, base_url=base_url, ) - self.servers.append(server) - db.add(self) - db.commit() + self.servers.append(orm_server) api_token = self.new_api_token() db.commit() + server = Server(orm_server=orm_server) + spawner = self.spawner # Passing server_name to the spawner spawner.server_name = server_name @@ -278,7 +282,7 @@ class User(HasTraits): ip_port = yield gen.with_timeout(timedelta(seconds=spawner.start_timeout), f) if ip_port: # get ip, port info from return value of start() - self.server.ip, self.server.port = ip_port + server.ip, server.port = ip_port else: # prior to 0.7, spawners had to store this info in user.server themselves. # Handle < 0.7 behavior with a warning, assuming info was stored in db by the Spawner. @@ -316,14 +320,14 @@ class User(HasTraits): db.commit() self.waiting_for_response = True try: - yield self.server.wait_up(http=True, timeout=spawner.http_timeout) + yield server.wait_up(http=True, timeout=spawner.http_timeout) except Exception as e: if isinstance(e, TimeoutError): self.log.warning( "{user}'s server never showed up at {url} " "after {http_timeout} seconds. Giving up".format( user=self.name, - url=self.server.url, + url=server.url, http_timeout=spawner.http_timeout, ) ) @@ -331,7 +335,7 @@ class User(HasTraits): else: e.reason = 'error' self.log.error("Unhandled error waiting for {user}'s server to show up at {url}: {error}".format( - user=self.name, url=self.server.url, error=e, + user=self.name, url=server.url, error=e, )) try: yield self.stop() diff --git a/jupyterhub/utils.py b/jupyterhub/utils.py index 82b58151..1c68bdfb 100644 --- a/jupyterhub/utils.py +++ b/jupyterhub/utils.py @@ -37,6 +37,8 @@ def can_connect(ip, port): Return True if we can connect, False otherwise. """ + if ip in {'', '0.0.0.0'}: + ip = '127.0.0.1' try: socket.create_connection((ip, port)) except socket.error as e: @@ -50,6 +52,8 @@ def can_connect(ip, port): @gen.coroutine def wait_for_server(ip, port, timeout=10): """Wait for any server to show up at ip:port.""" + if ip in {'', '0.0.0.0'}: + ip = '127.0.0.1' loop = ioloop.IOLoop.current() tic = loop.time() while loop.time() - tic < timeout: