hazza/jupyterhub
1
0
Fork 0
mirror of https://github.com/jupyterhub/jupyterhub.git synced 2025-10-18 07:23:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1098 from minrk/proxy

Browse Source 
implement proxy API
This commit is contained in:
Min RK
2017-05-22 14:17:41 -07:00
committed by GitHub
parent 7b96950a9c d548aa1e72
commit 62517d0c89
25 changed files with 756 additions and 631 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.flake8
View File

@@ -4,9 +4,10 @@
# W: style warnings
# C: complexity
# F401: module imported but unused
# F403: import *
# F811: redefinition of unused `name` from line `N`
# F841: local variable assigned but never used
ignore = E, C, W, F401, F811, F841
ignore = E, C, W, F401, F403, F811, F841
exclude =
.cache,

2
jupyterhub/apihandlers/base.py
View File

@@ -32,7 +32,7 @@ class APIHandler(BaseHandler):
self.log.warning("Blocking API request with no referer")
return False
host_path = url_path_join(host, self.hub.server.base_url)
host_path = url_path_join(host, self.hub.base_url)
referer_path = referer.split('://', 1)[-1]
if not (referer_path + '/').startswith(host_path):
self.log.warning("Blocking Cross Origin API request. Referer: %s, Host: %s",

15
jupyterhub/apihandlers/proxy.py
View File

@@ -4,6 +4,7 @@
# Distributed under the terms of the Modified BSD License.
import json
from urllib.parse import urlparse
from tornado import gen, web
@@ -21,7 +22,7 @@ class ProxyAPIHandler(APIHandler):
This is the same as fetching the routing table directly from the proxy,
but without clients needing to maintain separate
"""
routes = yield self.proxy.get_routes()
routes = yield self.proxy.get_all_routes()
self.write(json.dumps(routes))
@admin_only
@@ -48,17 +49,11 @@ class ProxyAPIHandler(APIHandler):
if not isinstance(model, dict):
raise web.HTTPError(400, "Request body must be JSON dict")
server = self.proxy.api_server
if 'ip' in model:
server.ip = model['ip']
if 'port' in model:
server.port = model['port']
if 'protocol' in model:
server.proto = model['protocol']
if 'api_url' in model:
self.proxy.api_url = model['api_url']
if 'auth_token' in model:
self.proxy.auth_token = model['auth_token']
self.db.commit()
self.log.info("Updated proxy at %s", server.bind_url)
self.log.info("Updated proxy at %s", self.proxy)
yield self.proxy.check_routes(self.users, self.services)

2
jupyterhub/apihandlers/users.py
View File

@@ -224,7 +224,7 @@ class UserCreateNamedServerAPIHandler(APIHandler):
def post(self, name):
user = self.find_user(name)
if user is None:
raise HTTPError(404, "No such user %r" % name)
raise web.HTTPError(404, "No such user %r" % name)
if user.running:
# include notify, so that a server that died is noticed immediately
state = yield user.spawner.poll_and_notify()

308
jupyterhub/app.py
View File

@@ -30,7 +30,6 @@ from sqlalchemy.orm import scoped_session
import tornado.httpserver
import tornado.options
from tornado.httpclient import HTTPError
from tornado.ioloop import IOLoop, PeriodicCallback
from tornado.log import app_log, access_log, gen_log
from tornado import gen, web
@@ -54,6 +53,7 @@ from .user import User, UserDict
from .oauth.store import make_provider
from ._data import DATA_FILES_PATH
from .log import CoroutineLogFormatter, log_request
from .proxy import Proxy, ConfigurableHTTPProxy
from .traitlets import URLPrefix, Command
from .utils import (
url_path_join,
@@ -62,6 +62,7 @@ from .utils import (
# classes for config
from .auth import Authenticator, PAMAuthenticator
from .spawner import Spawner, LocalProcessSpawner
from .objects import Hub
# For faking stats
from .emptyclass import EmptyClass
@@ -140,7 +141,6 @@ class NewToken(Application):
hub = JupyterHub(parent=self)
hub.load_config_file(hub.config_file)
hub.init_db()
hub.hub = hub.db.query(orm.Hub).first()
hub.init_users()
user = orm.User.find(hub.db, self.name)
if user is None:
@@ -349,45 +349,45 @@ class JupyterHub(Application):
help="Supply extra arguments that will be passed to Jinja environment."
).tag(config=True)
proxy_cmd = Command('configurable-http-proxy',
help="""The command to start the http proxy.
proxy_class = Type(ConfigurableHTTPProxy, Proxy,
help="""Select the Proxy API implementation."""
).tag(config=True)
Only override if configurable-http-proxy is not on your PATH
"""
proxy_cmd = Command([], config=True,
help="DEPRECATED. Use ConfigurableHTTPProxy.command",
).tag(config=True)
debug_proxy = Bool(False,
help="show debug output in configurable-http-proxy"
help="DEPRECATED: Use ConfigurableHTTPProxy.debug",
).tag(config=True)
proxy_auth_token = Unicode(
help="""The Proxy Auth token.
Loaded from the CONFIGPROXY_AUTH_TOKEN env variable by default.
"""
help="DEPRECATED: Use ConfigurableHTTPProxy.auth_token"
).tag(config=True)
@default('proxy_auth_token')
def _proxy_auth_token_default(self):
token = os.environ.get('CONFIGPROXY_AUTH_TOKEN', None)
if not token:
self.log.warning('\n'.join([
"",
"Generating CONFIGPROXY_AUTH_TOKEN. Restarting the Hub will require restarting the proxy.",
"Set CONFIGPROXY_AUTH_TOKEN env or JupyterHub.proxy_auth_token config to avoid this message.",
"",
]))
token = orm.new_token()
return token
_proxy_config_map = {
'proxy_cmd': 'command',
'debug_proxy': 'debug',
'proxy_auth_token': 'auth_token',
}
@observe(*_proxy_config_map)
def _deprecated_proxy_config(self, change):
dest = self._proxy_config_map[change.name]
self.log.warning("JupyterHub.%s is deprecated in JupyterHub 0.8, use ConfigurableHTTPProxy.%s", change.name, dest)
self.config.ConfigurableHTTPProxy[dest] = change.new
proxy_api_ip = Unicode('127.0.0.1',
help="The ip for the proxy API handlers"
proxy_api_ip = Unicode(
help="DEPRECATED: Use ConfigurableHTTPProxy.api_url"
).tag(config=True)
proxy_api_port = Integer(
help="The port for the proxy API handlers"
help="DEPRECATED: Use ConfigurableHTTPProxy.api_url"
).tag(config=True)
@default('proxy_api_port')
def _proxy_api_port_default(self):
return self.port + 1
@observe('proxy_api_port', 'proxy_api_ip')
def _deprecated_proxy_api(self, change):
self.log.warning("JupyterHub.%s is deprecated in JupyterHub 0.8, use ConfigurableHTTPProxy.api_url", change.name)
self.config.ConfigurableHTTPProxy.api_url = 'http://{}:{}'.format(
self.proxy_api_ip or '127.0.0.1',
self.proxy_api_port or self.port + 1,
)
hub_port = Integer(8081,
help="The port for this process"
@@ -682,10 +682,6 @@ class JupyterHub(Application):
def init_ports(self):
if self.hub_port == self.port:
raise TraitError("The hub and proxy cannot both listen on port %i" % self.port)
if self.hub_port == self.proxy_api_port:
raise TraitError("The hub and proxy API cannot both listen on port %i" % self.hub_port)
if self.proxy_api_port == self.port:
raise TraitError("The proxy's public and API ports cannot both be %i" % self.port)
@staticmethod
def add_url_prefix(prefix, handlers):
@@ -805,36 +801,6 @@ class JupyterHub(Application):
self._local.db = scoped_session(self.session_factory)()
return self._local.db
@property
def hub(self):
if not getattr(self._local, 'hub', None):
q = self.db.query(orm.Hub)
assert q.count() <= 1
self._local.hub = q.first()
if self.subdomain_host and self._local.hub:
self._local.hub.host = self.subdomain_host
return self._local.hub
@hub.setter
def hub(self, hub):
self._local.hub = hub
if hub and self.subdomain_host:
hub.host = self.subdomain_host
@property
def proxy(self):
if not getattr(self._local, 'proxy', None):
q = self.db.query(orm.Proxy)
assert q.count() <= 1
p = self._local.proxy = q.first()
if p:
p.auth_token = self.proxy_auth_token
return self._local.proxy
@proxy.setter
def proxy(self, proxy):
self._local.proxy = proxy
def init_db(self):
"""Create the database connection"""
self.log.debug("Connecting to db: %s", self.db_url)
@@ -861,28 +827,14 @@ class JupyterHub(Application):
def init_hub(self):
"""Load the Hub config into the database"""
self.hub = self.db.query(orm.Hub).first()
if self.hub is None:
self.hub = orm.Hub(
server=orm.Server(
ip=self.hub_ip,
port=self.hub_port,
base_url=self.hub_prefix,
cookie_name='jupyter-hub-token',
)
)
self.db.add(self.hub)
else:
server = self.hub.server
server.ip = self.hub_ip
server.port = self.hub_port
server.base_url = self.hub_prefix
if self.subdomain_host:
if not self.subdomain_host:
raise ValueError("Must specify subdomain_host when using subdomains."
" This should be the public domain[:port] of the Hub.")
self.db.commit()
self.hub = Hub(
ip=self.hub_ip,
port=self.hub_port,
base_url=self.hub_prefix,
cookie_name='jupyter-hub-token',
public_host=self.subdomain_host,
)
print(self.hub)
@gen.coroutine
def init_users(self):
@@ -1182,124 +1134,28 @@ class JupyterHub(Application):
)
def init_proxy(self):
"""Load the Proxy config into the database"""
self.proxy = self.db.query(orm.Proxy).first()
if self.proxy is None:
self.proxy = orm.Proxy(
public_server=orm.Server(),
api_server=orm.Server(),
)
self.db.add(self.proxy)
self.db.commit()
self.proxy.auth_token = self.proxy_auth_token # not persisted
self.proxy.log = self.log
self.proxy.public_server.ip = self.ip
self.proxy.public_server.port = self.port
self.proxy.public_server.base_url = self.base_url
self.proxy.api_server.ip = self.proxy_api_ip
self.proxy.api_server.port = self.proxy_api_port
self.proxy.api_server.base_url = '/api/routes/'
self.db.commit()
@gen.coroutine
def start_proxy(self):
"""Actually start the configurable-http-proxy"""
# check for proxy
if self.proxy.public_server.is_up() or self.proxy.api_server.is_up():
# check for *authenticated* access to the proxy (auth token can change)
try:
routes = yield self.proxy.get_routes()
except (HTTPError, OSError, socket.error) as e:
if isinstance(e, HTTPError) and e.code == 403:
msg = "Did CONFIGPROXY_AUTH_TOKEN change?"
else:
msg = "Is something else using %s?" % self.proxy.public_server.bind_url
self.log.error("Proxy appears to be running at %s, but I can't access it (%s)\n%s",
self.proxy.public_server.bind_url, e, msg)
self.exit(1)
return
else:
self.log.info("Proxy already running at: %s", self.proxy.public_server.bind_url)
yield self.proxy.check_routes(self.users, self._service_map, routes)
self.proxy_process = None
return
env = os.environ.copy()
env['CONFIGPROXY_AUTH_TOKEN'] = self.proxy.auth_token
cmd = self.proxy_cmd + [
'--ip', self.proxy.public_server.ip,
'--port', str(self.proxy.public_server.port),
'--api-ip', self.proxy.api_server.ip,
'--api-port', str(self.proxy.api_server.port),
'--default-target', self.hub.server.host,
'--error-target', url_path_join(self.hub.server.url, 'error'),
]
if self.subdomain_host:
cmd.append('--host-routing')
if self.debug_proxy:
cmd.extend(['--log-level', 'debug'])
if self.ssl_key:
cmd.extend(['--ssl-key', self.ssl_key])
if self.ssl_cert:
cmd.extend(['--ssl-cert', self.ssl_cert])
if self.statsd_host:
cmd.extend([
'--statsd-host', self.statsd_host,
'--statsd-port', str(self.statsd_port),
'--statsd-prefix', self.statsd_prefix + '.chp'
])
# Warn if SSL is not used
if ' --ssl' not in ' '.join(cmd):
self.log.warning("Running JupyterHub without SSL."
" I hope there is SSL termination happening somewhere else...")
self.log.info("Starting proxy @ %s", self.proxy.public_server.bind_url)
self.log.debug("Proxy cmd: %s", cmd)
try:
self.proxy_process = Popen(cmd, env=env, start_new_session=True)
except FileNotFoundError as e:
self.log.error(
"Failed to find proxy %r\n"
"The proxy can be installed with `npm install -g configurable-http-proxy`"
% self.proxy_cmd
)
self.exit(1)
def _check():
status = self.proxy_process.poll()
if status is not None:
e = RuntimeError("Proxy failed to start with exit code %i" % status)
# py2-compatible `raise e from None`
e.__cause__ = None
raise e
for server in (self.proxy.public_server, self.proxy.api_server):
for i in range(10):
_check()
try:
yield server.wait_up(1)
except TimeoutError:
continue
else:
break
yield server.wait_up(1)
self.log.debug("Proxy started and appears to be up")
@gen.coroutine
def check_proxy(self):
if self.proxy_process.poll() is None:
return
self.log.error("Proxy stopped with exit code %r",
'unknown' if self.proxy_process is None else self.proxy_process.poll()
"""Load the Proxy config"""
# FIXME: handle deprecated config here
public_url = 'http{s}://{ip}:{port}{base_url}'.format(
s='s' if self.ssl_cert else '',
ip=self.ip,
port=self.port,
base_url=self.base_url,
)
self.proxy = self.proxy_class(
db=self.db,
public_url=public_url,
parent=self,
app=self,
log=self.log,
hub=self.hub,
ssl_cert=self.ssl_cert,
ssl_key=self.ssl_key,
)
yield self.start_proxy()
self.log.info("Setting up routes on new proxy")
yield self.proxy.add_all_users(self.users)
yield self.proxy.add_all_services(self.services)
self.log.info("New proxy back up, and good to go")
def init_tornado_settings(self):
"""Set up the tornado settings dict."""
base_url = self.hub.server.base_url
base_url = self.hub.base_url
jinja_options = dict(
autoescape=True,
)
@@ -1337,7 +1193,7 @@ class JupyterHub(Application):
login_url=login_url,
logout_url=logout_url,
static_path=os.path.join(self.data_files_path, 'static'),
static_url_prefix=url_path_join(self.hub.server.base_url, 'static/'),
static_url_prefix=url_path_join(self.hub.base_url, 'static/'),
static_handler_class=CacheControlStaticFilesHandler,
template_path=self.template_paths,
jinja2_env=jinja_env,
@@ -1422,13 +1278,8 @@ class JupyterHub(Application):
# clean up proxy while single-user servers are shutting down
if self.cleanup_proxy:
if self.proxy_process:
self.log.info("Cleaning up proxy[%i]...", self.proxy_process.pid)
if self.proxy_process.poll() is None:
try:
self.proxy_process.terminate()
except Exception as e:
self.log.error("Failed to terminate proxy process: %s", e)
if self.proxy.should_start:
yield gen.maybe_future(self.proxy.stop())
else:
self.log.info("I didn't start the proxy, I can't clean it up")
else:
@@ -1484,26 +1335,30 @@ class JupyterHub(Application):
@gen.coroutine
def update_last_activity(self):
"""Update User.last_activity timestamps from the proxy"""
routes = yield self.proxy.get_routes()
routes = yield self.proxy.get_all_routes()
users_count = 0
active_users_count = 0
for prefix, route in routes.items():
if 'user' not in route:
route_data = route['data']
if 'user' not in route_data:
# not a user route, ignore it
continue
user = orm.User.find(self.db, route['user'])
users_count += 1
if 'last_activity' not in route_data:
# no last activity data (possibly proxy other than CHP)
continue
user = orm.User.find(self.db, route_data['user'])
if user is None:
self.log.warning("Found no user for route: %s", route)
continue
try:
dt = datetime.strptime(route['last_activity'], ISO8601_ms)
dt = datetime.strptime(route_data['last_activity'], ISO8601_ms)
except Exception:
dt = datetime.strptime(route['last_activity'], ISO8601_s)
dt = datetime.strptime(route_data['last_activity'], ISO8601_s)
user.last_activity = max(user.last_activity, dt)
# FIXME: Make this configurable duration. 30 minutes for now!
if (datetime.now() - user.last_activity).total_seconds() < 30 * 60:
active_users_count += 1
users_count += 1
self.statsd.gauge('users.running', users_count)
self.statsd.gauge('users.active', active_users_count)
@@ -1530,17 +1385,20 @@ class JupyterHub(Application):
try:
self.http_server.listen(self.hub_port, address=self.hub_ip)
except Exception:
self.log.error("Failed to bind hub to %s", self.hub.server.bind_url)
self.log.error("Failed to bind hub to %s", self.hub.bind_url)
raise
else:
self.log.info("Hub API listening on %s", self.hub.server.bind_url)
self.log.info("Hub API listening on %s", self.hub.bind_url)
# start the proxy
try:
yield self.start_proxy()
except Exception as e:
self.log.critical("Failed to start proxy", exc_info=True)
self.exit(1)
if self.proxy.should_start:
try:
yield self.proxy.start()
except Exception as e:
self.log.critical("Failed to start proxy", exc_info=True)
self.exit(1)
else:
self.log.info("Not starting proxy")
# start the service(s)
for service_name, service in self._service_map.items():
@@ -1574,12 +1432,6 @@ class JupyterHub(Application):
loop.add_callback(self.proxy.add_all_users, self.users)
loop.add_callback(self.proxy.add_all_services, self._service_map)
if self.proxy_process:
# only check / restart the proxy if we started it in the first place.
# this means a restarted Hub cannot restart a Proxy that its
# predecessor started.
pc = PeriodicCallback(self.check_proxy, 1e3 * self.proxy_check_interval)
pc.start()
if self.service_check_interval and any(s.url for s in self._service_map.values()):
pc = PeriodicCallback(self.check_services_health, 1e3 * self.service_check_interval)
@@ -1589,7 +1441,7 @@ class JupyterHub(Application):
pc = PeriodicCallback(self.update_last_activity, 1e3 * self.last_activity_interval)
pc.start()
self.log.info("JupyterHub is now running at %s", self.proxy.public_server.url)
self.log.info("JupyterHub is now running at %s", self.proxy.public_url)
# register cleanup on both TERM and INT
atexit.register(self.atexit)
self.init_signal()

1
jupyterhub/dbutil.py
View File

@@ -89,5 +89,4 @@ def _alembic(*args):
if __name__ == '__main__':
import sys
_alembic(*sys.argv[1:])

25
jupyterhub/handlers/base.py
View File

@@ -17,8 +17,9 @@ from tornado.web import RequestHandler
from tornado import gen, web
from .. import orm
from ..user import User
from ..objects import Server
from ..spawner import LocalProcessSpawner
from ..user import User
from ..utils import url_path_join
# pattern for the authentication token header
@@ -103,7 +104,7 @@ class BaseHandler(RequestHandler):
@property
def csp_report_uri(self):
return self.settings.get('csp_report_uri',
url_path_join(self.hub.server.base_url, 'security/csp-report')
url_path_join(self.hub.base_url, 'security/csp-report')
)
@property
@@ -184,7 +185,7 @@ class BaseHandler(RequestHandler):
max_age_days=self.cookie_max_age_days,
)
def clear():
self.clear_cookie(cookie_name, path=self.hub.server.base_url)
self.clear_cookie(cookie_name, path=self.hub.base_url)
if cookie_id is None:
if self.get_cookie(cookie_name):
@@ -208,7 +209,7 @@ class BaseHandler(RequestHandler):
def get_current_user_cookie(self):
"""get_current_user from a cookie token"""
return self._user_for_cookie(self.hub.server.cookie_name)
return self._user_for_cookie(self.hub.cookie_name)
def get_current_user(self):
"""get current username"""
@@ -245,9 +246,7 @@ class BaseHandler(RequestHandler):
kwargs = {}
if self.subdomain_host:
kwargs['domain'] = self.domain
if user and user.server:
self.clear_cookie(user.server.cookie_name, path=user.server.base_url, **kwargs)
self.clear_cookie(self.hub.server.cookie_name, path=self.hub.server.base_url, **kwargs)
self.clear_cookie(self.hub.cookie_name, path=self.hub.base_url, **kwargs)
self.clear_cookie('jupyterhub-services', path=url_path_join(self.base_url, 'services'))
def _set_user_cookie(self, user, server):
@@ -436,7 +435,7 @@ class BaseHandler(RequestHandler):
def template_namespace(self):
user = self.get_current_user()
return dict(
base_url=self.hub.server.base_url,
base_url=self.hub.base_url,
prefix=self.base_url,
user=user,
login_url=self.settings['login_url'],
@@ -502,7 +501,7 @@ class PrefixRedirectHandler(BaseHandler):
else:
path = self.request.path
self.redirect(url_path_join(
self.hub.server.base_url, path,
self.hub.base_url, path,
), permanent=False)
@@ -528,12 +527,12 @@ class UserSpawnHandler(BaseHandler):
port = host_info.port
if not port:
port = 443 if host_info.scheme == 'https' else 80
if port != self.proxy.public_server.port and port == self.hub.server.port:
if port != Server.from_url(self.proxy.public_url).port and port == self.hub.port:
self.log.warning("""
Detected possible direct connection to Hub's private ip: %s, bypassing proxy.
This will result in a redirect loop.
Make sure to connect to the proxied public URL %s
""", self.request.full_url(), self.proxy.public_server.url)
""", self.request.full_url(), self.proxy.public_url)
# logged in as correct user, spawn the server
if current_user.spawner:
@@ -548,14 +547,14 @@ class UserSpawnHandler(BaseHandler):
status = yield current_user.spawner.poll()
if status is not None:
if current_user.spawner.options_form:
self.redirect(url_concat(url_path_join(self.hub.server.base_url, 'spawn'),
self.redirect(url_concat(url_path_join(self.hub.base_url, 'spawn'),
{'next': self.request.uri}))
return
else:
yield self.spawn_single_user(current_user)
# set login cookie anew
self.set_login_cookie(current_user)
without_prefix = self.request.uri[len(self.hub.server.base_url):]
without_prefix = self.request.uri[len(self.hub.base_url):]
target = url_path_join(self.base_url, without_prefix)
if self.subdomain_host:
target = current_user.host + target

4
jupyterhub/handlers/login.py
View File

@@ -57,7 +57,7 @@ class LoginHandler(BaseHandler):
if user.running:
next_url = user.url
else:
next_url = self.hub.server.base_url
next_url = self.hub.base_url
# set new login cookie
# because single-user cookie may have been cleared or incorrect
self.set_login_cookie(self.get_current_user())
@@ -101,7 +101,7 @@ class LoginHandler(BaseHandler):
next_url = self.get_argument('next', default='')
if not next_url.startswith('/'):
next_url = ''
next_url = next_url or self.hub.server.base_url
next_url = next_url or self.hub.base_url
self.redirect(next_url)
self.log.info("User logged in: %s", username)
else:

6
jupyterhub/handlers/pages.py
View File

@@ -37,7 +37,7 @@ class RootHandler(BaseHandler):
# The next request will be handled by UserSpawnHandler,
# ultimately redirecting to the logged-in user's server.
without_prefix = next_url[len(self.base_url):]
next_url = url_path_join(self.hub.server.base_url, without_prefix)
next_url = url_path_join(self.hub.base_url, without_prefix)
self.log.warning("Redirecting %s to %s. For sharing public links, use /user-redirect/",
self.request.uri, next_url,
)
@@ -50,7 +50,7 @@ class RootHandler(BaseHandler):
self.log.debug("User is running: %s", url)
self.set_login_cookie(user) # set cookie
else:
url = url_path_join(self.hub.server.base_url, 'home')
url = url_path_join(self.hub.base_url, 'home')
self.log.debug("User is not running: %s", url)
else:
url = self.settings['login_url']
@@ -215,7 +215,7 @@ class ProxyErrorHandler(BaseHandler):
status_message = responses.get(status_code, 'Unknown HTTP Error')
# build template namespace
hub_home = url_path_join(self.hub.server.base_url, 'home')
hub_home = url_path_join(self.hub.base_url, 'home')
message_html = ''
if status_code == 503:
message_html = ' '.join([

134
jupyterhub/objects.py Normal file
View File

@@ -0,0 +1,134 @@
"""Some general objects for use in JupyterHub"""
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
from urllib.parse import urlparse
from tornado import gen
from traitlets import (
HasTraits, Instance, Integer, Unicode,
default, observe,
)
from . import orm
from .utils import (
url_path_join, can_connect, wait_for_server,
wait_for_http_server, random_port,
)
class Server(HasTraits):
"""An object representing an HTTP endpoint.
*Some* of these reside in the database (user servers),
but others (Hub, proxy) are in-memory only.
"""
orm_server = Instance(orm.Server, allow_none=True)
ip = Unicode()
proto = Unicode('http')
port = Integer()
base_url = Unicode('/')
cookie_name = Unicode('')
@classmethod
def from_url(cls, url):
"""Create a Server from a given URL"""
urlinfo = urlparse(url)
proto = urlinfo.scheme
ip = urlinfo.hostname or ''
port = urlinfo.port
if not port:
if proto == 'https':
port = 443
else:
port = 80
return cls(proto=proto, ip=ip, port=port, base_url=urlinfo.path)
@default('port')
def _default_port(self):
return random_port()
@observe('orm_server')
def _orm_server_changed(self, change):
"""When we get an orm_server, get attributes from there."""
obj = change.new
self.proto = obj.proto
self.ip = obj.ip
self.port = obj.port
self.base_url = obj.base_url
self.cookie_name = obj.cookie_name
# setter to pass through to the database
@observe('ip', 'proto', 'port', 'base_url', 'cookie_name')
def _change(self, change):
if self.orm_server:
setattr(self.orm_server, change.name, change.new)
@property
def host(self):
ip = self.ip
if ip in {'', '0.0.0.0'}:
# when listening on all interfaces, connect to localhost
ip = '127.0.0.1'
return "{proto}://{ip}:{port}".format(
proto=self.proto,
ip=ip,
port=self.port,
)
@property
def url(self):
return "{host}{uri}".format(
host=self.host,
uri=self.base_url,
)
@property
def bind_url(self):
"""representation of URL used for binding
Never used in APIs, only logging,
since it can be non-connectable value, such as '', meaning all interfaces.
"""
if self.ip in {'', '0.0.0.0'}:
return self.url.replace('127.0.0.1', self.ip or '*', 1)
return self.url
@gen.coroutine
def wait_up(self, timeout=10, http=False):
"""Wait for this server to come up"""
if http:
yield wait_for_http_server(self.url, timeout=timeout)
else:
yield wait_for_server(self.ip or '127.0.0.1', self.port, timeout=timeout)
def is_up(self):
"""Is the server accepting connections?"""
return can_connect(self.ip or '127.0.0.1', self.port)
class Hub(Server):
"""Bring it all together at the hub.
The Hub is a server, plus its API path suffix
the api_url is the full URL plus the api_path suffix on the end
of the server base_url.
"""
@property
def server(self):
"""backward-compat"""
return self
public_host = Unicode()
@property
def api_url(self):
"""return the full API url (with proto://host...)"""
return url_path_join(self.url, 'api')
def __repr__(self):
return "<%s %s:%s>" % (
self.__class__.__name__, self.server.ip, self.server.port,
)

262
jupyterhub/orm.py
View File

@@ -79,251 +79,6 @@ class Server(Base):
def __repr__(self):
return "<Server(%s:%s)>" % (self.ip, self.port)
@property
def host(self):
ip = self.ip
if ip in {'', '0.0.0.0'}:
# when listening on all interfaces, connect to localhost
ip = '127.0.0.1'
return "{proto}://{ip}:{port}".format(
proto=self.proto,
ip=ip,
port=self.port,
)
@property
def url(self):
return "{host}{uri}".format(
host=self.host,
uri=self.base_url,
)
@property
def bind_url(self):
"""representation of URL used for binding
Never used in APIs, only logging,
since it can be non-connectable value, such as '', meaning all interfaces.
"""
if self.ip in {'', '0.0.0.0'}:
return self.url.replace('127.0.0.1', self.ip or '*', 1)
return self.url
@gen.coroutine
def wait_up(self, timeout=10, http=False):
"""Wait for this server to come up"""
if http:
yield wait_for_http_server(self.url, timeout=timeout)
else:
yield wait_for_server(self.ip or '127.0.0.1', self.port, timeout=timeout)
def is_up(self):
"""Is the server accepting connections?"""
return can_connect(self.ip or '127.0.0.1', self.port)
class Proxy(Base):
"""A configurable-http-proxy instance.
A proxy consists of the API server info and the public-facing server info,
plus an auth token for configuring the proxy table.
"""
__tablename__ = 'proxies'
id = Column(Integer, primary_key=True)
auth_token = None
_public_server_id = Column(Integer, ForeignKey('servers.id'))
public_server = relationship(Server, primaryjoin=_public_server_id == Server.id)
_api_server_id = Column(Integer, ForeignKey('servers.id'))
api_server = relationship(Server, primaryjoin=_api_server_id == Server.id)
def __repr__(self):
if self.public_server:
return "<%s %s:%s>" % (
self.__class__.__name__, self.public_server.ip, self.public_server.port,
)
else:
return "<%s [unconfigured]>" % self.__class__.__name__
def api_request(self, path, method='GET', body=None, client=None):
"""Make an authenticated API request of the proxy"""
client = client or AsyncHTTPClient()
url = url_path_join(self.api_server.url, path)
if isinstance(body, dict):
body = json.dumps(body)
self.log.debug("Fetching %s %s", method, url)
req = HTTPRequest(url,
method=method,
headers={'Authorization': 'token {}'.format(self.auth_token)},
body=body,
)
return client.fetch(req)
@gen.coroutine
def add_service(self, service, client=None):
"""Add a service's server to the proxy table."""
if not service.server:
raise RuntimeError(
"Service %s does not have an http endpoint to add to the proxy.", service.name)
self.log.info("Adding service %s to proxy %s => %s",
service.name, service.proxy_path, service.server.host,
)
yield self.api_request(service.proxy_path,
method='POST',
body=dict(
target=service.server.host,
service=service.name,
),
client=client,
)
@gen.coroutine
def delete_service(self, service, client=None):
"""Remove a service's server from the proxy table."""
self.log.info("Removing service %s from proxy", service.name)
yield self.api_request(service.proxy_path,
method='DELETE',
client=client,
)
# FIX-ME
# we need to add a reference to a specific server
@gen.coroutine
def add_user(self, user, client=None):
"""Add a user's server to the proxy table."""
self.log.info("Adding user %s to proxy %s => %s",
user.name, user.proxy_path, user.server.host,
)
if user.spawn_pending:
raise RuntimeError(
"User %s's spawn is pending, shouldn't be added to the proxy yet!", user.name)
yield self.api_request(user.proxy_path,
method='POST',
body=dict(
target=user.server.host,
user=user.name,
),
client=client,
)
@gen.coroutine
def delete_user(self, user, client=None):
"""Remove a user's server from the proxy table."""
self.log.info("Removing user %s from proxy", user.name)
yield self.api_request(user.proxy_path,
method='DELETE',
client=client,
)
@gen.coroutine
def add_all_services(self, service_dict):
"""Update the proxy table from the database.
Used when loading up a new proxy.
"""
db = inspect(self).session
futures = []
for orm_service in db.query(Service):
service = service_dict[orm_service.name]
if service.server:
futures.append(self.add_service(service))
# wait after submitting them all
for f in futures:
yield f
@gen.coroutine
def add_all_users(self, user_dict):
"""Update the proxy table from the database.
Used when loading up a new proxy.
"""
db = inspect(self).session
futures = []
for orm_user in db.query(User):
user = user_dict[orm_user]
if user.running:
futures.append(self.add_user(user))
# wait after submitting them all
for f in futures:
yield f
@gen.coroutine
def get_routes(self, client=None):
"""Fetch the proxy's routes"""
resp = yield self.api_request('', client=client)
return json.loads(resp.body.decode('utf8', 'replace'))
# FIX-ME
# we need to add a reference to a specific server
@gen.coroutine
def check_routes(self, user_dict, service_dict, routes=None):
"""Check that all users are properly routed on the proxy"""
if not routes:
routes = yield self.get_routes()
user_routes = { r['user'] for r in routes.values() if 'user' in r }
futures = []
db = inspect(self).session
for orm_user in db.query(User):
user = user_dict[orm_user]
if user.running:
if user.name not in user_routes:
self.log.warning("Adding missing route for %s (%s)", user.name, user.server)
futures.append(self.add_user(user))
else:
# User not running, make sure it's not in the table
if user.name in user_routes:
self.log.warning("Removing route for not running %s", user.name)
futures.append(self.delete_user(user))
# check service routes
service_routes = { r['service'] for r in routes.values() if 'service' in r }
for orm_service in db.query(Service).filter(Service.server != None):
service = service_dict[orm_service.name]
if service.server is None:
# This should never be True, but seems to be on rare occasion.
# catch filter bug, either in sqlalchemy or my understanding of its behavior
self.log.error("Service %s has no server, but wasn't filtered out.", service)
continue
if service.name not in service_routes:
self.log.warning("Adding missing route for %s (%s)", service.name, service.server)
futures.append(self.add_service(service))
for f in futures:
yield f
class Hub(Base):
"""Bring it all together at the hub.
The Hub is a server, plus its API path suffix
the api_url is the full URL plus the api_path suffix on the end
of the server base_url.
"""
__tablename__ = 'hubs'
id = Column(Integer, primary_key=True)
_server_id = Column(Integer, ForeignKey('servers.id'))
server = relationship(Server, primaryjoin=_server_id == Server.id)
host = ''
@property
def api_url(self):
"""return the full API url (with proto://host...)"""
return url_path_join(self.server.url, 'api')
def __repr__(self):
if self.server:
return "<%s %s:%s>" % (
self.__class__.__name__, self.server.ip, self.server.port,
)
else:
return "<%s [unconfigured]>" % self.__class__.__name__
# user:group many:many mapping table
user_group_map = Table('user_group_map', Base.metadata,
@@ -393,23 +148,14 @@ class User(Base):
# group mapping
groups = relationship('Group', secondary='user_group_map', back_populates='users')
@property
def server(self):
"""Returns the first element of servers.
Returns None if the list is empty.
"""
if len(self.servers) == 0:
return None
else:
return self.servers[0]
def __repr__(self):
if self.server:
if self.servers:
server = self.servers[0]
return "<{cls}({name}@{ip}:{port})>".format(
cls=self.__class__.__name__,
name=self.name,
ip=self.server.ip,
port=self.server.port,
ip=server.ip,
port=server.port,
)
else:
return "<{cls}({name} [unconfigured])>".format(

416
jupyterhub/proxy.py Normal file
View File

@@ -0,0 +1,416 @@
"""API for JupyterHub's proxy."""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import json
import os
from subprocess import Popen
import time
from tornado import gen
from tornado.httpclient import AsyncHTTPClient, HTTPRequest
from tornado.ioloop import PeriodicCallback
from traitlets import (
Any, Bool, Instance, Integer, Unicode,
default,
)
from jupyterhub.traitlets import Command
from traitlets.config import LoggingConfigurable
from .objects import Server
from .orm import Service, User
from . import utils
from .utils import url_path_join
class Proxy(LoggingConfigurable):
"""Base class for configurable proxies that JupyterHub can use."""
db = Any()
app = Any()
hub = Any()
public_url = Unicode()
ssl_key = Unicode()
ssl_cert = Unicode()
should_start = Bool(True, config=True,
help="""Should the Hub start the proxy.
If True, the Hub will start the proxy and stop it.
Set to False if the proxy is managed externally,
such as by systemd, docker, or another service manager.
""")
def start(self):
"""Start the proxy.
Will be called during startup if should_start is True.
"""
def stop(self):
"""Stop the proxy.
Will be called during teardown if should_start is True.
"""
@gen.coroutine
def add_route(self, routespec, target, data):
"""Add a route to the proxy.
Args:
routespec (str): A specification for which this route will be matched.
Could be either a url_prefix or a fqdn.
target (str): A URL that will be the target of this route.
data (dict): A JSONable dict that will be associated with this route, and will
be returned when retrieving information about this route.
Will raise an appropriate Exception (FIXME: find what?) if the route could
not be added.
The proxy implementation should also have a way to associate the fact that a
route came from JupyterHub.
"""
pass
@gen.coroutine
def delete_route(self, routespec):
"""Delete a route with a given routespec if it exists."""
pass
@gen.coroutine
def get_route(self, routespec):
"""Return the route info for a given routespec.
Args:
routespec (str): The route specification that was used to add this routespec
Returns:
result (dict): with the following keys:
`routespec`: The normalized route specification passed in to add_route
`target`: The target for this route
`data`: The arbitrary data that was passed in by JupyterHub when adding this
route.
None: if there are no routes matching the given routespec
"""
pass
@gen.coroutine
def get_all_routes(self):
"""Fetch and return all the routes associated by JupyterHub from the
proxy.
Should return a dictionary of routes, where the keys are
routespecs and each value is the dict that would be returned by
`get_route(routespec)`.
"""
pass
# Most basic implementers must only implement above methods
@gen.coroutine
def add_service(self, service, client=None):
"""Add a service's server to the proxy table."""
if not service.server:
raise RuntimeError(
"Service %s does not have an http endpoint to add to the proxy.", service.name)
self.log.info("Adding service %s to proxy %s => %s",
service.name, service.proxy_path, service.server.host,
)
yield self.add_route(
service.proxy_path,
service.server.host,
{'service': service.name}
)
@gen.coroutine
def delete_service(self, service, client=None):
"""Remove a service's server from the proxy table."""
self.log.info("Removing service %s from proxy", service.name)
yield self.delete_route(service.proxy_path)
@gen.coroutine
def add_user(self, user, client=None):
"""Add a user's server to the proxy table."""
self.log.info("Adding user %s to proxy %s => %s",
user.name, user.proxy_path, user.server.host,
)
if user.spawn_pending:
raise RuntimeError(
"User %s's spawn is pending, shouldn't be added to the proxy yet!", user.name)
yield self.add_route(
user.proxy_path,
user.server.host,
{'user': user.name}
)
@gen.coroutine
def delete_user(self, user):
"""Remove a user's server from the proxy table."""
self.log.info("Removing user %s from proxy", user.name)
yield self.delete_route(user.proxy_path)
@gen.coroutine
def add_all_services(self, service_dict):
"""Update the proxy table from the database.
Used when loading up a new proxy.
"""
db = self.db
futures = []
for orm_service in db.query(Service):
service = service_dict[orm_service.name]
if service.server:
futures.append(self.add_service(service))
# wait after submitting them all
for f in futures:
yield f
@gen.coroutine
def add_all_users(self, user_dict):
"""Update the proxy table from the database.
Used when loading up a new proxy.
"""
db = self.db
futures = []
for orm_user in db.query(User):
user = user_dict[orm_user]
if user.running:
futures.append(self.add_user(user))
# wait after submitting them all
for f in futures:
yield f
@gen.coroutine
def check_routes(self, user_dict, service_dict, routes=None):
"""Check that all users are properly routed on the proxy."""
if not routes:
routes = yield self.get_all_routes()
user_routes = {r['data']['user'] for r in routes.values() if 'user' in r['data']}
futures = []
db = self.db
for orm_user in db.query(User):
user = user_dict[orm_user]
if user.running:
if user.name not in user_routes:
self.log.warning(
"Adding missing route for %s (%s)", user.name, user.server)
futures.append(self.add_user(user))
else:
# User not running, make sure it's not in the table
if user.name in user_routes:
self.log.warning(
"Removing route for not running %s", user.name)
futures.append(self.delete_user(user))
# check service routes
service_routes = {r['data']['service']
for r in routes.values() if 'service' in r['data']}
for orm_service in db.query(Service).filter(
Service.server is not None):
service = service_dict[orm_service.name]
if service.server is None:
# This should never be True, but seems to be on rare occasion.
# catch filter bug, either in sqlalchemy or my understanding of
# its behavior
self.log.error(
"Service %s has no server, but wasn't filtered out.", service)
continue
if service.name not in service_routes:
self.log.warning("Adding missing route for %s (%s)",
service.name, service.server)
futures.append(self.add_service(service))
for f in futures:
yield f
@gen.coroutine
def restore_routes(self):
self.log.info("Setting up routes on new proxy")
yield self.add_all_users(self.app.users)
yield self.add_all_services(self.app.services)
self.log.info("New proxy back up, and good to go")
class ConfigurableHTTPProxy(Proxy):
"""Proxy implementation for the default configurable-http-proxy."""
proxy_process = Any()
client = Instance(AsyncHTTPClient, ())
debug = Bool(False, help="Add debug-level logging to the Proxy", config=True)
auth_token = Unicode(
help="""The Proxy Auth token.
Loaded from the CONFIGPROXY_AUTH_TOKEN env variable by default.
""",
).tag(config=True)
check_running_interval = Integer(5, config=True)
@default('auth_token')
def _auth_token_default(self):
token = os.environ.get('CONFIGPROXY_AUTH_TOKEN', None)
if not token:
self.log.warning('\n'.join([
"",
"Generating CONFIGPROXY_AUTH_TOKEN. Restarting the Hub will require restarting the proxy.",
"Set CONFIGPROXY_AUTH_TOKEN env or JupyterHub.proxy_auth_token config to avoid this message.",
"",
]))
token = utils.new_token()
return token
api_url = Unicode('http://127.0.0.1:8001', config=True,
help="""The ip (or hostname) of the proxy's API endpoint"""
)
command = Command('configurable-http-proxy', config=True,
help="""The command to start the proxy"""
)
@gen.coroutine
def start(self):
public_server = Server.from_url(self.public_url)
api_server = Server.from_url(self.api_url)
env = os.environ.copy()
env['CONFIGPROXY_AUTH_TOKEN'] = self.auth_token
cmd = self.command + [
'--ip', public_server.ip,
'--port', str(public_server.port),
'--api-ip', api_server.ip,
'--api-port', str(api_server.port),
'--default-target', self.hub.host,
'--error-target', url_path_join(self.hub.url, 'error'),
]
if self.app.subdomain_host:
cmd.append('--host-routing')
if self.debug:
cmd.extend(['--log-level', 'debug'])
if self.ssl_key:
cmd.extend(['--ssl-key', self.ssl_key])
if self.ssl_cert:
cmd.extend(['--ssl-cert', self.ssl_cert])
if self.app.statsd_host:
cmd.extend([
'--statsd-host', self.app.statsd_host,
'--statsd-port', str(self.app.statsd_port),
'--statsd-prefix', self.app.statsd_prefix + '.chp'
])
# Warn if SSL is not used
if ' --ssl' not in ' '.join(cmd):
self.log.warning("Running JupyterHub without SSL."
" I hope there is SSL termination happening somewhere else...")
self.log.info("Starting proxy @ %s", public_server.bind_url)
self.log.debug("Proxy cmd: %s", cmd)
try:
self.proxy_process = Popen(cmd, env=env, start_new_session=True)
except FileNotFoundError as e:
self.log.error(
"Failed to find proxy %r\n"
"The proxy can be installed with `npm install -g configurable-http-proxy`"
% self.cmd
)
self.exit(1)
def _check_process():
status = self.proxy_process.poll()
if status is not None:
e = RuntimeError(
"Proxy failed to start with exit code %i" % status)
# py2-compatible `raise e from None`
e.__cause__ = None
raise e
for server in (public_server, api_server):
for i in range(10):
_check_process()
try:
yield server.wait_up(1)
except TimeoutError:
continue
else:
break
yield server.wait_up(1)
time.sleep(1)
_check_process()
self.log.debug("Proxy started and appears to be up")
pc = PeriodicCallback(self.check_running, 1e3 * self.check_running_interval)
pc.start()
def stop(self):
self.log.info("Cleaning up proxy[%i]...", self.proxy_process.pid)
if self.proxy_process.poll() is None:
try:
self.proxy_process.terminate()
except Exception as e:
self.log.error("Failed to terminate proxy process: %s", e)
@gen.coroutine
def check_running(self):
"""Check if the proxy is still running"""
if self.proxy_process.poll() is None:
return
self.log.error("Proxy stopped with exit code %r",
'unknown' if self.proxy_process is None else self.proxy_process.poll()
)
yield self.start()
yield self.restore_routes()
def api_request(self, path, method='GET', body=None, client=None):
"""Make an authenticated API request of the proxy."""
client = client or AsyncHTTPClient()
url = url_path_join(self.api_url, 'api/routes', path)
if isinstance(body, dict):
body = json.dumps(body)
self.log.debug("Proxy: Fetching %s %s", method, url)
req = HTTPRequest(url,
method=method,
headers={'Authorization': 'token {}'.format(
self.auth_token)},
body=body,
)
return client.fetch(req)
def add_route(self, routespec, target, data=None):
body = data or {}
body['target'] = target
return self.api_request(routespec,
method='POST',
body=body,
)
def delete_route(self, routespec):
return self.api_request(routespec, method='DELETE')
def _reformat_routespec(self, routespec, chp_data):
"""Reformat CHP data format to JupyterHub's proxy API."""
target = chp_data.pop('target')
return {
'routespec': routespec,
'target': target,
'data': chp_data,
}
@gen.coroutine
def get_route(self, routespec):
chp_data = yield self.api_request(routespec, method='DELETE')
return self._reformat_routespec(routespec, chp_data)
@gen.coroutine
def get_all_routes(self, client=None):
"""Fetch the proxy's routes."""
resp = yield self.api_request('', client=client)
chp_routes = json.loads(resp.body.decode('utf8', 'replace'))
all_routes = {}
for routespec, chp_data in chp_routes.items():
all_routes[routespec] = self._reformat_routespec(
routespec, chp_data)
return all_routes

8
jupyterhub/services/service.py
View File

@@ -52,6 +52,7 @@ from traitlets import (
from traitlets.config import LoggingConfigurable
from .. import orm
from ..objects import Server
from ..traitlets import Command
from ..spawner import LocalProcessSpawner, set_user_setuid
from ..utils import url_path_join
@@ -60,7 +61,7 @@ class _MockUser(HasTraits):
name = Unicode()
server = Instance(orm.Server, allow_none=True)
state = Dict()
service = Instance(__module__ + '.Service')
service = Instance(__name__ + '.Service')
host = Unicode()
@property
@@ -221,7 +222,10 @@ class Service(LoggingConfigurable):
@property
def server(self):
return self.orm.server
if self.orm.server:
return Server(orm_server=self.orm.server)
else:
return None
@property
def prefix(self):

6
jupyterhub/singleuser.py
View File

@@ -228,12 +228,6 @@ class SingleUserNotebookApp(NotebookApp):
value = value + '/'
return value
@default('cookie_name')
def _cookie_name_default(self):
if os.environ.get('JUPYTERHUB_SERVICE_NAME'):
# if I'm a service, use the services cookie name
return 'jupyterhub-services'
@default('port')
def _port_default(self):
if os.environ.get('JUPYTERHUB_SERVICE_URL'):

6
jupyterhub/spawner.py
View File

@@ -431,7 +431,7 @@ class Spawner(LoggingConfigurable):
env['JUPYTERHUB_ADMIN_ACCESS'] = '1'
# OAuth settings
env['JUPYTERHUB_CLIENT_ID'] = self.oauth_client_id
env['JUPYTERHUB_HOST'] = self.hub.host
env['JUPYTERHUB_HOST'] = self.hub.public_host
env['JUPYTERHUB_OAUTH_CALLBACK_URL'] = \
url_path_join(self.user.url, 'oauth_callback')
@@ -496,8 +496,8 @@ class Spawner(LoggingConfigurable):
args = [
'--user="%s"' % self.user.name,
'--base-url="%s"' % self.user.server.base_url,
'--hub-host="%s"' % self.hub.host,
'--hub-prefix="%s"' % self.hub.server.base_url,
'--hub-host="%s"' % self.hub.public_host,
'--hub-prefix="%s"' % self.hub.base_url,
'--hub-api-url="%s"' % self.hub.api_url,
]
if self.ip:

16
jupyterhub/tests/conftest.py
View File

@@ -9,7 +9,7 @@ from subprocess import TimeoutExpired
import time
from unittest import mock
from pytest import fixture, raises
from tornado import ioloop
from tornado import ioloop, gen
from .. import orm
from ..utils import random_port
@@ -32,11 +32,7 @@ def db():
name=getuser(),
)
user.servers.append(orm.Server())
hub = orm.Hub(
server=orm.Server(),
)
_db.add(user)
_db.add(hub)
_db.commit()
return _db
@@ -85,10 +81,14 @@ def _mockservice(request, app, url=False):
with mock.patch.object(jupyterhub.services.service, '_ServiceSpawner', MockServiceSpawner):
app.services = [spec]
app.init_services()
app.io_loop.add_callback(app.proxy.add_all_services, app._service_map)
assert name in app._service_map
service = app._service_map[name]
app.io_loop.add_callback(service.start)
@gen.coroutine
def start():
# wait for proxy to be updated before starting the service
yield app.proxy.add_all_services(app._service_map)
service.start()
app.io_loop.add_callback(start)
def cleanup():
service.stop()
app.services[:] = []
@@ -100,6 +100,8 @@ def _mockservice(request, app, url=False):
# ensure process finishes starting
with raises(TimeoutExpired):
service.proc.wait(1)
if url:
ioloop.IOLoop().run_sync(service.server.wait_up)
return service

12
jupyterhub/tests/mocking.py
View File

@@ -4,7 +4,6 @@ import os
import sys
from tempfile import NamedTemporaryFile
import threading
from unittest import mock
import requests
@@ -18,6 +17,7 @@ from traitlets import default
from ..app import JupyterHub
from ..auth import PAMAuthenticator
from .. import orm
from ..objects import Server
from ..spawner import LocalProcessSpawner
from ..singleuser import SingleUserNotebookApp
from ..utils import random_port, url_path_join
@@ -165,7 +165,7 @@ class MockHub(JupyterHub):
self.db.add(user)
self.db.commit()
yield super(MockHub, self).start()
yield self.hub.server.wait_up(http=True)
yield self.hub.wait_up(http=True)
self.io_loop.add_callback(evt.set)
def _start():
@@ -207,7 +207,7 @@ def public_host(app):
if app.subdomain_host:
return app.subdomain_host
else:
return app.proxy.public_server.host
return Server.from_url(app.proxy.public_url).host
def public_url(app, user_or_service=None, path=''):
@@ -220,7 +220,7 @@ def public_url(app, user_or_service=None, path=''):
prefix = user_or_service.server.base_url
else:
host = public_host(app)
prefix = app.proxy.public_server.base_url
prefix = Server.from_url(app.proxy.public_url).base_url
if path:
return host + url_path_join(prefix, path)
else:
@@ -246,7 +246,8 @@ class StubSingleUserSpawner(MockSpawner):
_thread = None
@gen.coroutine
def start(self):
self.user.server.port = random_port()
ip = self.user.server.ip
port = self.user.server.port = random_port()
env = self.get_env()
args = self.get_args()
evt = threading.Event()
@@ -267,6 +268,7 @@ class StubSingleUserSpawner(MockSpawner):
self._thread.start()
ready = evt.wait(timeout=3)
assert ready
return (ip, port)
@gen.coroutine
def stop(self):

8
jupyterhub/tests/test_api.py
View File

@@ -83,7 +83,7 @@ def auth_header(db, name):
@check_db_locks
def api_request(app, *api_path, **kwargs):
"""Make an API request"""
base_url = app.hub.server.url
base_url = app.hub.url
headers = kwargs.setdefault('headers', {})
if 'Authorization' not in headers:
@@ -94,7 +94,7 @@ def api_request(app, *api_path, **kwargs):
f = getattr(requests, method)
resp = f(url, **kwargs)
assert "frame-ancestors 'self'" in resp.headers['Content-Security-Policy']
assert ujoin(app.hub.server.base_url, "security/csp-report") in resp.headers['Content-Security-Policy']
assert ujoin(app.hub.base_url, "security/csp-report") in resp.headers['Content-Security-Policy']
assert 'http' not in resp.headers['Content-Security-Policy']
return resp
@@ -132,7 +132,7 @@ def test_auth_api(app):
def test_referer_check(app, io_loop):
url = ujoin(public_host(app), app.hub.server.base_url)
url = ujoin(public_host(app), app.hub.base_url)
host = urlparse(url).netloc
user = find_user(app.db, 'admin')
if user is None:
@@ -779,7 +779,7 @@ def test_get_service(app, mockservice_url):
def test_root_api(app):
base_url = app.hub.server.url
base_url = app.hub.url
url = ujoin(base_url, 'api')
r = requests.get(url)
r.raise_for_status()

43
jupyterhub/tests/test_orm.py
View File

@@ -7,6 +7,7 @@ import pytest
from tornado import gen
from .. import orm
from .. import objects
from ..user import User
from .mocking import MockSpawner
@@ -20,6 +21,9 @@ def test_server(db):
assert server.proto == 'http'
assert isinstance(server.port, int)
assert isinstance(server.cookie_name, str)
# test wrapper
server = objects.Server(orm_server=server)
assert server.host == 'http://127.0.0.1:%i' % server.port
assert server.url == server.host + '/'
assert server.bind_url == 'http://*:%i/' % server.port
@@ -28,45 +32,10 @@ def test_server(db):
assert server.url == server.host + '/'
def test_proxy(db):
proxy = orm.Proxy(
auth_token='abc-123',
public_server=orm.Server(
ip='192.168.1.1',
port=8000,
),
api_server=orm.Server(
ip='127.0.0.1',
port=8001,
),
)
db.add(proxy)
db.commit()
assert proxy.public_server.ip == '192.168.1.1'
assert proxy.api_server.ip == '127.0.0.1'
assert proxy.auth_token == 'abc-123'
def test_hub(db):
hub = orm.Hub(
server=orm.Server(
ip = '1.2.3.4',
port = 1234,
base_url='/hubtest/',
),
)
db.add(hub)
db.commit()
assert hub.server.ip == '1.2.3.4'
assert hub.server.port == 1234
assert hub.api_url == 'http://1.2.3.4:1234/hubtest/api'
def test_user(db):
user = orm.User(name='kaylee',
user = User(orm.User(name='kaylee',
state={'pid': 4234},
)
))
server = orm.Server()
user.servers.append(server)
db.add(user)

16
jupyterhub/tests/test_pages.py
View File

@@ -16,7 +16,7 @@ from .test_api import api_request
def get_page(path, app, hub=True, **kw):
if hub:
prefix = app.hub.server.base_url
prefix = app.hub.base_url
else:
prefix = app.base_url
base_url = ujoin(public_host(app), prefix)
@@ -24,11 +24,11 @@ def get_page(path, app, hub=True, **kw):
return requests.get(ujoin(base_url, path), **kw)
def test_root_no_auth(app, io_loop):
print(app.hub.server.is_up())
routes = io_loop.run_sync(app.proxy.get_routes)
print(app.hub.is_up())
routes = io_loop.run_sync(app.proxy.get_all_routes)
print(routes)
print(app.hub.server)
url = ujoin(public_host(app), app.hub.server.base_url)
url = ujoin(public_host(app), app.hub.base_url)
print(url)
r = requests.get(url)
r.raise_for_status()
@@ -123,7 +123,7 @@ def test_spawn_page(app):
def test_spawn_form(app, io_loop):
with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}):
base_url = ujoin(public_host(app), app.hub.server.base_url)
base_url = ujoin(public_host(app), app.hub.base_url)
cookies = app.login_user('jones')
orm_u = orm.User.find(app.db, 'jones')
u = app.users[orm_u]
@@ -145,7 +145,7 @@ def test_spawn_form(app, io_loop):
def test_spawn_form_with_file(app, io_loop):
with mock.patch.dict(app.users.settings, {'spawner_class': FormSpawner}):
base_url = ujoin(public_host(app), app.hub.server.base_url)
base_url = ujoin(public_host(app), app.hub.base_url)
cookies = app.login_user('jones')
orm_u = orm.User.find(app.db, 'jones')
u = app.users[orm_u]
@@ -181,7 +181,7 @@ def test_user_redirect(app):
assert path == ujoin(app.base_url, '/hub/login')
query = urlparse(r.url).query
assert query == urlencode({
'next': ujoin(app.hub.server.base_url, '/user-redirect/tree/top/')
'next': ujoin(app.hub.base_url, '/user-redirect/tree/top/')
})
r = get_page('/user-redirect/notebooks/test.ipynb', app, cookies=cookies)
@@ -320,7 +320,7 @@ def test_login_no_whitelist_adds_user(app):
def test_static_files(app):
base_url = ujoin(public_host(app), app.hub.server.base_url)
base_url = ujoin(public_host(app), app.hub.base_url)
r = requests.get(ujoin(base_url, 'logo'))
r.raise_for_status()
assert r.headers['content-type'] == 'image/png'

45
jupyterhub/tests/test_proxy.py
View File

@@ -6,6 +6,8 @@ from queue import Queue
from subprocess import Popen
from urllib.parse import urlparse, unquote
from traitlets.config import Config
import pytest
from .. import orm
@@ -19,12 +21,12 @@ def test_external_proxy(request, io_loop):
auth_token = 'secret!'
proxy_ip = '127.0.0.1'
proxy_port = 54321
cfg = Config()
cfg.ConfigurableHTTPProxy.auth_token = auth_token
cfg.ConfigurableHTTPProxy.api_url = 'http://%s:%i' % (proxy_ip, proxy_port)
cfg.ConfigurableHTTPProxy.should_start = False
app = MockHub.instance(
proxy_api_ip=proxy_ip,
proxy_api_port=proxy_port,
proxy_auth_token=auth_token,
)
app = MockHub.instance(config=cfg)
def fin():
MockHub.clear_instance()
@@ -35,7 +37,8 @@ def test_external_proxy(request, io_loop):
# configures and starts proxy process
env = os.environ.copy()
env['CONFIGPROXY_AUTH_TOKEN'] = auth_token
cmd = app.proxy_cmd + [
cmd = [
'configurable-http-proxy',
'--ip', app.ip,
'--port', str(app.port),
'--api-ip', proxy_ip,
@@ -57,10 +60,10 @@ def test_external_proxy(request, io_loop):
wait_for_proxy()
app.start([])
assert app.proxy_process is None
assert app.proxy.proxy_process is None
# test if api service has a root route '/'
routes = io_loop.run_sync(app.proxy.get_routes)
routes = io_loop.run_sync(app.proxy.get_all_routes)
assert list(routes.keys()) == ['/']
# add user to the db and start a single user server
@@ -70,7 +73,7 @@ def test_external_proxy(request, io_loop):
r = api_request(app, 'users', name, 'server', method='post')
r.raise_for_status()
routes = io_loop.run_sync(app.proxy.get_routes)
routes = io_loop.run_sync(app.proxy.get_all_routes)
# sets the desired path result
user_path = unquote(ujoin(app.base_url, 'user/river'))
if app.subdomain_host:
@@ -83,7 +86,8 @@ def test_external_proxy(request, io_loop):
proxy = Popen(cmd, env=env)
wait_for_proxy()
routes = io_loop.run_sync(app.proxy.get_routes)
routes = io_loop.run_sync(app.proxy.get_all_routes)
assert list(routes.keys()) == ['/']
# poke the server to update the proxy
@@ -91,7 +95,7 @@ def test_external_proxy(request, io_loop):
r.raise_for_status()
# check that the routes are correct
routes = io_loop.run_sync(app.proxy.get_routes)
routes = io_loop.run_sync(app.proxy.get_all_routes)
assert sorted(routes.keys()) == ['/', user_path]
# teardown the proxy, and start a new one with different auth and port
@@ -99,10 +103,10 @@ def test_external_proxy(request, io_loop):
new_auth_token = 'different!'
env['CONFIGPROXY_AUTH_TOKEN'] = new_auth_token
proxy_port = 55432
cmd = app.proxy_cmd + [
cmd = ['configurable-http-proxy',
'--ip', app.ip,
'--port', str(app.port),
'--api-ip', app.proxy_api_ip,
'--api-ip', proxy_ip,
'--api-port', str(proxy_port),
'--default-target', 'http://%s:%i' % (app.hub_ip, app.hub_port),
]
@@ -112,14 +116,13 @@ def test_external_proxy(request, io_loop):
wait_for_proxy()
# tell the hub where the new proxy is
new_api_url = 'http://{}:{}'.format(proxy_ip, proxy_port)
r = api_request(app, 'proxy', method='patch', data=json.dumps({
'port': proxy_port,
'protocol': 'http',
'ip': app.ip,
'api_url': new_api_url,
'auth_token': new_auth_token,
}))
r.raise_for_status()
assert app.proxy.api_server.port == proxy_port
assert app.proxy.api_url == new_api_url
# get updated auth token from main thread
def get_app_proxy_token():
@@ -131,7 +134,7 @@ def test_external_proxy(request, io_loop):
app.proxy.auth_token = new_auth_token
# check that the routes are correct
routes = io_loop.run_sync(app.proxy.get_routes)
routes = io_loop.run_sync(app.proxy.get_all_routes)
assert sorted(routes.keys()) == ['/', user_path]
@@ -152,18 +155,18 @@ def test_check_routes(app, io_loop, username, endpoints):
# check a valid route exists for user
test_user = app.users[username]
before = sorted(io_loop.run_sync(app.proxy.get_routes))
before = sorted(io_loop.run_sync(app.proxy.get_all_routes))
assert unquote(test_user.proxy_path) in before
# check if a route is removed when user deleted
io_loop.run_sync(lambda: app.proxy.check_routes(app.users, app._service_map))
io_loop.run_sync(lambda: proxy.delete_user(test_user))
during = sorted(io_loop.run_sync(app.proxy.get_routes))
during = sorted(io_loop.run_sync(app.proxy.get_all_routes))
assert unquote(test_user.proxy_path) not in during
# check if a route exists for user
io_loop.run_sync(lambda: app.proxy.check_routes(app.users, app._service_map))
after = sorted(io_loop.run_sync(app.proxy.get_routes))
after = sorted(io_loop.run_sync(app.proxy.get_all_routes))
assert unquote(test_user.proxy_path) in after
# check that before and after state are the same

4
jupyterhub/tests/test_services.py
View File

@@ -25,7 +25,7 @@ def external_service(app, name='mockservice'):
env = {
'JUPYTERHUB_API_TOKEN': hexlify(os.urandom(5)),
'JUPYTERHUB_SERVICE_NAME': name,
'JUPYTERHUB_API_URL': url_path_join(app.hub.server.url, 'api/'),
'JUPYTERHUB_API_URL': url_path_join(app.hub.url, 'api/'),
'JUPYTERHUB_SERVICE_URL': 'http://127.0.0.1:%i' % random_port(),
}
proc = Popen(mockservice_cmd, env=env)
@@ -64,7 +64,7 @@ def test_managed_service(mockservice):
def test_proxy_service(app, mockservice_url, io_loop):
service = mockservice_url
name = service.name
io_loop.run_sync(app.proxy.get_routes)
io_loop.run_sync(app.proxy.get_all_routes)
url = public_url(app, service) + '/foo'
r = requests.get(url, allow_redirects=False)
path = '/services/{}/foo'.format(name)

3
jupyterhub/tests/test_spawner.py
View File

@@ -17,6 +17,7 @@ import requests
from tornado import gen
from ..user import User
from ..objects import Hub
from .. import spawner as spawnermod
from ..spawner import LocalProcessSpawner
from .. import orm
@@ -43,8 +44,8 @@ def setup():
def new_spawner(db, **kwargs):
kwargs.setdefault('cmd', [sys.executable, '-c', _echo_sleep])
kwargs.setdefault('hub', Hub())
kwargs.setdefault('user', User(db.query(orm.User).first(), {}))
kwargs.setdefault('hub', db.query(orm.Hub).first())
kwargs.setdefault('notebook_dir', os.getcwd())
kwargs.setdefault('default_url', '/user/{username}/lab')
kwargs.setdefault('INTERRUPT_TIMEOUT', 1)

38
jupyterhub/user.py
View File

@@ -9,9 +9,10 @@ from sqlalchemy import inspect
from tornado import gen
from tornado.log import app_log
from .utils import url_path_join, default_server_name, new_token
from .utils import url_path_join, default_server_name
from . import orm
from .objects import Server
from traitlets import HasTraits, Any, Dict, observe, default
from .spawner import LocalProcessSpawner
@@ -111,23 +112,20 @@ class User(HasTraits):
def spawner_class(self):
return self.settings.get('spawner_class', LocalProcessSpawner)
def __init__(self, orm_user, settings, **kwargs):
def __init__(self, orm_user, settings=None, **kwargs):
self.orm_user = orm_user
self.settings = settings
self.settings = settings or {}
super().__init__(**kwargs)
hub = self.db.query(orm.Hub).first()
self.allow_named_servers = self.settings.get('allow_named_servers', False)
self.cookie_name = '%s-%s' % (hub.server.cookie_name, quote(self.name, safe=''))
self.base_url = url_path_join(
self.settings.get('base_url', '/'), 'user', self.escaped_name)
self.spawner = self.spawner_class(
user=self,
db=self.db,
hub=hub,
hub=self.settings.get('hub'),
authenticator=self.authenticator,
config=self.settings.get('config'),
)
@@ -158,6 +156,13 @@ class User(HasTraits):
return False
return True
@property
def server(self):
if len(self.servers) == 0:
return None
else:
return Server(orm_server=self.servers[0])
@property
def escaped_name(self):
"""My name, escaped for use in URLs, cookies, etc."""
@@ -223,18 +228,17 @@ class User(HasTraits):
server_name = ''
base_url = self.base_url
server = orm.Server(
name = server_name,
cookie_name=self.cookie_name,
orm_server = orm.Server(
name=server_name,
base_url=base_url,
)
self.servers.append(server)
db.add(self)
db.commit()
self.servers.append(orm_server)
api_token = self.new_api_token()
db.commit()
server = Server(orm_server=orm_server)
spawner = self.spawner
# Passing server_name to the spawner
spawner.server_name = server_name
@@ -278,7 +282,7 @@ class User(HasTraits):
ip_port = yield gen.with_timeout(timedelta(seconds=spawner.start_timeout), f)
if ip_port:
# get ip, port info from return value of start()
self.server.ip, self.server.port = ip_port
server.ip, server.port = ip_port
else:
# prior to 0.7, spawners had to store this info in user.server themselves.
# Handle < 0.7 behavior with a warning, assuming info was stored in db by the Spawner.
@@ -316,14 +320,14 @@ class User(HasTraits):
db.commit()
self.waiting_for_response = True
try:
yield self.server.wait_up(http=True, timeout=spawner.http_timeout)
yield server.wait_up(http=True, timeout=spawner.http_timeout)
except Exception as e:
if isinstance(e, TimeoutError):
self.log.warning(
"{user}'s server never showed up at {url} "
"after {http_timeout} seconds. Giving up".format(
user=self.name,
url=self.server.url,
url=server.url,
http_timeout=spawner.http_timeout,
)
)
@@ -331,7 +335,7 @@ class User(HasTraits):
else:
e.reason = 'error'
self.log.error("Unhandled error waiting for {user}'s server to show up at {url}: {error}".format(
user=self.name, url=self.server.url, error=e,
user=self.name, url=server.url, error=e,
))
try:
yield self.stop()

4
jupyterhub/utils.py
View File

@@ -37,6 +37,8 @@ def can_connect(ip, port):
Return True if we can connect, False otherwise.
"""
if ip in {'', '0.0.0.0'}:
ip = '127.0.0.1'
try:
socket.create_connection((ip, port))
except socket.error as e:
@@ -50,6 +52,8 @@ def can_connect(ip, port):
@gen.coroutine
def wait_for_server(ip, port, timeout=10):
"""Wait for any server to show up at ip:port."""
if ip in {'', '0.0.0.0'}:
ip = '127.0.0.1'
loop = ioloop.IOLoop.current()
tic = loop.time()
while loop.time() - tic < timeout: