From 7b0c845c3a65c7d0466cf18639e295f7eb9e5c1e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 27 Jul 2017 19:09:08 -0700 Subject: [PATCH 1/7] Allow Hub to listen on a unix-socket Add the hub_socket option to the JupyterHub class, which takes precedence over the hub_ip and hub_port setting. It does not forward this setting to the Hub class though, and a few log messages still say the hub is listening on `http://:8000` that works fine when testing with netcat: ``` $ nc -U /tmp/jhub.sock GET /login HTTP/1.1 HTTP/1.1 302 Found Server: TornadoServer/4.5.1 Content-Type: text/html; charset=UTF-8 Date: Fri, 28 Jul 2017 02:05:36 GMT X-Jupyterhub-Version: 0.8.0.dev Content-Security-Policy: frame-ancestors 'self'; report-uri /hub/security/csp-report Location: /hub/login Content-Length: 0 ``` Should still be better documented I guess. --- jupyterhub/app.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 29e23ef0..510568bb 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -19,9 +19,11 @@ import sys from textwrap import dedent from urllib.parse import urlparse + if sys.version_info[:2] < (3, 3): raise ValueError("Python < 3.3 not supported: %s" % sys.version) + from dateutil.parser import parse as parse_date from jinja2 import Environment, FileSystemLoader, PrefixLoader, ChoiceLoader from sqlalchemy.exc import OperationalError @@ -33,6 +35,7 @@ from tornado.log import app_log, access_log, gen_log import tornado.options from tornado import gen, web from tornado.platform.asyncio import AsyncIOMainLoop +from tornado.netutil import bind_unix_socket from traitlets import ( Unicode, Integer, Dict, TraitError, List, Bool, Any, @@ -424,6 +427,13 @@ class JupyterHub(Application): """ ).tag(config=True) + hub_socket = Unicode('', + help="""Set the tornado application to listen on a unix socket. + + If set, take precedence over the `hub_port` and `hub_ip` settings. + """ + ).tag(config=True) + hub_connect_ip = Unicode('', help="""The ip or hostname for proxies and spawners to use for connecting to the Hub. @@ -1627,12 +1637,16 @@ class JupyterHub(Application): # start the webserver self.http_server = tornado.httpserver.HTTPServer(self.tornado_application, xheaders=True) try: - self.http_server.listen(self.hub_port, address=self.hub_ip) + if self.hub_socket: + socket = bind_unix_socket(self.hub_socket) + self.http_server.add_socket(socket) + self.log.info("Hub API listening on %s", self.hub_socket) + else: + self.http_server.listen(self.hub_port, address=self.hub_ip) + self.log.info("Hub API listening on %s", self.hub.bind_url) except Exception: self.log.error("Failed to bind hub to %s", self.hub.bind_url) raise - else: - self.log.info("Hub API listening on %s", self.hub.bind_url) # start the proxy if self.proxy.should_start: From 138bad5913123024162f9c6e88fe8dab4b196b2f Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 3 May 2018 16:32:31 +0200 Subject: [PATCH 2/7] add connect_url, bind_url overrides enables `c.JupyterHub.bind_url = 'unix+http://%2Fsrv%2Fjupyterhub%2Fjupyterhub.sock'` for listening on a bsd socket. Similarly, bind_url and connect_url work as overrides everywhere --- jupyterhub/app.py | 93 +++++++++++++++++++++++++++++++------------ jupyterhub/objects.py | 32 ++++++++++----- 2 files changed, 89 insertions(+), 36 deletions(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 510568bb..f2686c2d 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -17,7 +17,7 @@ import re import signal import sys from textwrap import dedent -from urllib.parse import urlparse +from urllib.parse import unquote, urlparse if sys.version_info[:2] < (3, 3): @@ -302,11 +302,11 @@ class JupyterHub(Application): """ ).tag(config=True) ip = Unicode('', - help="""The public facing ip of the whole JupyterHub application + help="""The public facing ip of the whole JupyterHub application (specifically referred to as the proxy). - - This is the address on which the proxy will listen. The default is to - listen on all interfaces. This is the only address through which JupyterHub + + This is the address on which the proxy will listen. The default is to + listen on all interfaces. This is the only address through which JupyterHub should be accessed by users. """ ).tag(config=True) @@ -410,7 +410,7 @@ class JupyterHub(Application): hub_port = Integer(8081, help="""The internal port for the Hub process. - + This is the internal port of the hub itself. It should never be accessed directly. See JupyterHub.port for the public port to use when accessing jupyterhub. It is rare that this port should be set except in cases of port conflict. @@ -418,7 +418,7 @@ class JupyterHub(Application): ).tag(config=True) hub_ip = Unicode('127.0.0.1', help="""The ip address for the Hub process to *bind* to. - + By default, the hub listens on localhost only. This address must be accessible from the proxy and user servers. You may need to set this to a public ip or '' for all interfaces if the proxy or user servers are in containers or on a different host. @@ -427,13 +427,6 @@ class JupyterHub(Application): """ ).tag(config=True) - hub_socket = Unicode('', - help="""Set the tornado application to listen on a unix socket. - - If set, take precedence over the `hub_port` and `hub_ip` settings. - """ - ).tag(config=True) - hub_connect_ip = Unicode('', help="""The ip or hostname for proxies and spawners to use for connecting to the Hub. @@ -450,14 +443,42 @@ class JupyterHub(Application): """ ).tag(config=True) + hub_connect_url = Unicode( + help=""" + The URL for connecting to the Hub. + Spawners, services, and the proxy will use this URL + to talk to the Hub. + + Only needs to be specified if the default hub URL is not + connectable (e.g. using a unix+http:// bind url). + + .. seealso:: + JupyterHub.hub_connect_ip + JupyterHub.hub_bind_url + .. versionadded:: 0.9 + """ + ) + hub_bind_url = Unicode( + help=""" + The URL for binding the Hub. + The Hub will listen on this URL. + + Can be a unix+http:// url for listening on a BSD socket + + .. versionadded:: 0.9 + """, + config=True, + ) + hub_connect_port = Integer( 0, help=""" - The port for proxies & spawners to connect to the hub on. + DEPRECATED - Used alongside `hub_connect_ip` and only when different from hub_port. + Use hub_connect_url .. versionadded:: 0.8 + .. versiondeprecated:: 0.9 """ ).tag(config=True) @@ -979,17 +1000,30 @@ class JupyterHub(Application): self.exit(e) def init_hub(self): - """Load the Hub config into the database""" - self.hub = Hub( - ip=self.hub_ip, - port=self.hub_port, + """Load the Hub URL config""" + hub_args = dict( base_url=self.hub_prefix, public_host=self.subdomain_host, ) + if self.hub_bind_url: + hub_args['bind_url'] = self.hub_bind_url + else: + hub_args['ip'] = self.hub_ip + hub_args['port'] = self.hub_port + self.hub = Hub(**hub_args) + if self.hub_connect_ip: self.hub.connect_ip = self.hub_connect_ip if self.hub_connect_port: self.hub.connect_port = self.hub_connect_port + self.log.warning( + "JupyterHub.hub_connect_port is deprecated as of 0.9." + " Use JupyterHub.hub_connect_url to fully specify" + " the URL for connecting to the Hub." + ) + + if self.hub_connect_url: + self.hub.connect_url = self.hub_connect_url async def init_users(self): """Load users into and from the database""" @@ -1636,14 +1670,23 @@ class JupyterHub(Application): # start the webserver self.http_server = tornado.httpserver.HTTPServer(self.tornado_application, xheaders=True) + bind_url = urlparse(self.hub.bind_url) try: - if self.hub_socket: - socket = bind_unix_socket(self.hub_socket) + if bind_url.scheme.startswith('unix+'): + socket = bind_unix_socket(unquote(bind_url.netloc)) self.http_server.add_socket(socket) - self.log.info("Hub API listening on %s", self.hub_socket) else: - self.http_server.listen(self.hub_port, address=self.hub_ip) - self.log.info("Hub API listening on %s", self.hub.bind_url) + ip = bind_url.hostname + port = bind_url.port + if not port: + if bind_url.scheme == 'https': + port = 443 + else: + port = 80 + self.http_server.listen(port, address=ip) + self.log.info("Hub API listening on %s", self.hub.bind_url) + if self.hub.url != self.hub.bind_url: + self.log.info("Private Hub API connect url %s", self.hub.url) except Exception: self.log.error("Failed to bind hub to %s", self.hub.bind_url) raise diff --git a/jupyterhub/objects.py b/jupyterhub/objects.py index 1364d58c..a97ac814 100644 --- a/jupyterhub/objects.py +++ b/jupyterhub/objects.py @@ -33,6 +33,19 @@ class Server(HasTraits): port = Integer() base_url = URLPrefix('/') cookie_name = Unicode('') + connect_url = Unicode('') + bind_url = Unicode('') + + @default('bind_url') + def bind_url_default(self): + """representation of URL used for binding + + Never used in APIs, only logging, + since it can be non-connectable value, such as '', meaning all interfaces. + """ + if self.ip in {'', '0.0.0.0'}: + return self.url.replace(self._connect_ip, self.ip or '*', 1) + return self.url @property def _connect_ip(self): @@ -107,6 +120,12 @@ class Server(HasTraits): @property def host(self): + if self.connect_url: + parsed = urlparse(self.connect_url) + return "{proto}://{host}".format( + proto=parsed.scheme, + host=parsed.netloc, + ) return "{proto}://{ip}:{port}".format( proto=self.proto, ip=self._connect_ip, @@ -115,22 +134,13 @@ class Server(HasTraits): @property def url(self): + if self.connect_url: + return self.connect_url return "{host}{uri}".format( host=self.host, uri=self.base_url, ) - @property - def bind_url(self): - """representation of URL used for binding - - Never used in APIs, only logging, - since it can be non-connectable value, such as '', meaning all interfaces. - """ - if self.ip in {'', '0.0.0.0'}: - return self.url.replace(self._connect_ip, self.ip or '*', 1) - return self.url - def wait_up(self, timeout=10, http=False): """Wait for this server to come up""" if http: From 3cae550b13ec8993acad813e70f7c22e6f8616e0 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 3 May 2018 16:32:56 +0200 Subject: [PATCH 3/7] remove redundant "Adding default route" log the same message is logged immediately after with the URL --- jupyterhub/proxy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jupyterhub/proxy.py b/jupyterhub/proxy.py index 12c1f348..2038501e 100644 --- a/jupyterhub/proxy.py +++ b/jupyterhub/proxy.py @@ -305,7 +305,6 @@ class Proxy(LoggingConfigurable): hub = self.app.hub if '/' not in routes: - self.log.warning("Adding default route") futures.append(self.add_hub_route(hub)) else: route = routes['/'] From 804a9b7be852dbc0b26444f4a2a7393c351e397c Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 3 May 2018 16:41:02 +0200 Subject: [PATCH 4/7] Spawner.start can return a URL enables internal HTTPS, if setup by the Spawner --- jupyterhub/user.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/jupyterhub/user.py b/jupyterhub/user.py index 9d31db2f..d0c07d45 100644 --- a/jupyterhub/user.py +++ b/jupyterhub/user.py @@ -401,15 +401,30 @@ class User: f = maybe_future(spawner.start()) # commit any changes in spawner.start (always commit db changes before yield) db.commit() - ip_port = await gen.with_timeout(timedelta(seconds=spawner.start_timeout), f) - if ip_port: + url = await gen.with_timeout(timedelta(seconds=spawner.start_timeout), f) + if url: # get ip, port info from return value of start() - server.ip, server.port = ip_port + if isinstance(url, str): + # >= 0.9 can return a full URL string + pass + else: + # >= 0.7 returns (ip, port) + url = 'http://%s:%i' % url + urlinfo = urlparse(url) + server.proto = urlinfo.scheme + server.ip = urlinfo.hostname + port = urlinfo.port + if not port: + if urlinfo.scheme == 'https': + port = 443 + else: + port = 80 + server.port = port db.commit() else: # prior to 0.7, spawners had to store this info in user.server themselves. # Handle < 0.7 behavior with a warning, assuming info was stored in db by the Spawner. - self.log.warning("DEPRECATION: Spawner.start should return (ip, port) in JupyterHub >= 0.7") + self.log.warning("DEPRECATION: Spawner.start should return a url or (ip, port) tuple in JupyterHub >= 0.9") if spawner.api_token and spawner.api_token != api_token: # Spawner re-used an API token, discard the unused api_token orm_token = orm.APIToken.find(self.db, api_token) From 9591fd88c5962d3662dc9e286f1fb5b8a5ae573a Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 4 May 2018 10:57:40 +0200 Subject: [PATCH 5/7] add JupyterHub.bind_url for public bind URL --- jupyterhub/app.py | 63 ++++++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index f2686c2d..2999ea90 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -17,7 +17,7 @@ import re import signal import sys from textwrap import dedent -from urllib.parse import unquote, urlparse +from urllib.parse import unquote, urlparse, urlunparse if sys.version_info[:2] < (3, 3): @@ -87,6 +87,7 @@ aliases = { 'y': 'JupyterHub.answer_yes', 'ssl-key': 'JupyterHub.ssl_key', 'ssl-cert': 'JupyterHub.ssl_cert', + 'url': 'JupyterHub.bind_url', 'ip': 'JupyterHub.ip', 'port': 'JupyterHub.port', 'pid-file': 'JupyterHub.pid_file', @@ -307,9 +308,29 @@ class JupyterHub(Application): This is the address on which the proxy will listen. The default is to listen on all interfaces. This is the only address through which JupyterHub - should be accessed by users. """ + should be accessed by users.""" ).tag(config=True) + @observe('ip', 'port') + def _ip_port_changed(self, change): + urlinfo = urlparse(self.bind_url) + urlinfo = urlinfo._replace(netloc='%s:%i' % (self.ip, self.port)) + self.bind_url = urlunparse(urlinfo) + + bind_url = Unicode( + "http://127.0.0.1:8000", + help="""The public facing URL of the whole JupyterHub application. + + This is the address on which the proxy will bind. + Sets protocol, ip, base_url + """ + ).tag(config=True) + + @observe('bind_url') + def _bind_url_changed(self, change): + urlinfo = urlparse(change.new) + self.base_url = urlinfo.path + subdomain_host = Unicode('', help="""Run single-user servers on subdomains of this host. @@ -343,16 +364,16 @@ class JupyterHub(Application): port = Integer(8000, help="""The public facing port of the proxy. - - This is the port on which the proxy will listen. - This is the only port through which JupyterHub + + This is the port on which the proxy will listen. + This is the only port through which JupyterHub should be accessed by users. """ ).tag(config=True) base_url = URLPrefix('/', help="""The base URL of the entire application. - - Add this to the begining of all JupyterHub URLs. + + Add this to the begining of all JupyterHub URLs. Use base_url to run JupyterHub within an existing website. """ ).tag(config=True) @@ -460,10 +481,15 @@ class JupyterHub(Application): ) hub_bind_url = Unicode( help=""" - The URL for binding the Hub. - The Hub will listen on this URL. + The URL on which the Hub will listen. + This is a private URL for internal communication. + Typically set in combination with hub_connect_url. + If a unix socket, hub_connect_url **must** also be set. - Can be a unix+http:// url for listening on a BSD socket + For example: + + "http://127.0.0.1:8081" + "unix+http://%2Fsrv%2Fjupyterhub%2Fjupyterhub.sock" .. versionadded:: 0.9 """, @@ -478,7 +504,9 @@ class JupyterHub(Application): Use hub_connect_url .. versionadded:: 0.8 - .. versiondeprecated:: 0.9 + + .. deprecated:: 0.9 + Use hub_connect_url """ ).tag(config=True) @@ -856,10 +884,6 @@ class JupyterHub(Application): logger.parent = self.log logger.setLevel(self.log.level) - def init_ports(self): - if self.hub_port == self.port: - raise TraitError("The hub and proxy cannot both listen on port %i" % self.port) - @staticmethod def add_url_prefix(prefix, handlers): """add a url prefix to handlers""" @@ -1390,15 +1414,9 @@ class JupyterHub(Application): def init_proxy(self): """Load the Proxy config""" # FIXME: handle deprecated config here - public_url = 'http{s}://{ip}:{port}{base_url}'.format( - s='s' if self.ssl_cert else '', - ip=self.ip, - port=self.port, - base_url=self.base_url, - ) self.proxy = self.proxy_class( db_factory=lambda: self.db, - public_url=public_url, + public_url=self.bind_url, parent=self, app=self, log=self.log, @@ -1513,7 +1531,6 @@ class JupyterHub(Application): self.update_config(cfg) self.write_pid_file() self.init_pycurl() - self.init_ports() self.init_secrets() self.init_db() self.init_hub() From c39244168bd8d4f09500319673f449405e954a65 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 4 May 2018 11:03:38 +0200 Subject: [PATCH 6/7] note deprecations for ip/port in favor of bind_url --- jupyterhub/app.py | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index 2999ea90..f729d628 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -308,7 +308,22 @@ class JupyterHub(Application): This is the address on which the proxy will listen. The default is to listen on all interfaces. This is the only address through which JupyterHub - should be accessed by users.""" + should be accessed by users. + + .. deprecated: 0.9 + """ + ).tag(config=True) + + port = Integer(8000, + help="""The public facing port of the proxy. + + This is the port on which the proxy will listen. + This is the only port through which JupyterHub + should be accessed by users. + + .. deprecated: 0.9 + Use JupyterHub.bind_url + """ ).tag(config=True) @observe('ip', 'port') @@ -323,6 +338,9 @@ class JupyterHub(Application): This is the address on which the proxy will bind. Sets protocol, ip, base_url + + .. deprecated: 0.9 + Use JupyterHub.bind_url """ ).tag(config=True) @@ -331,6 +349,17 @@ class JupyterHub(Application): urlinfo = urlparse(change.new) self.base_url = urlinfo.path + base_url = URLPrefix('/', + help="""The base URL of the entire application. + + Add this to the beginning of all JupyterHub URLs. + Use base_url to run JupyterHub within an existing website. + + .. deprecated: 0.9 + Use JupyterHub.bind_url + """ + ).tag(config=True) + subdomain_host = Unicode('', help="""Run single-user servers on subdomains of this host. @@ -362,21 +391,6 @@ class JupyterHub(Application): return '' return urlparse(self.subdomain_host).hostname - port = Integer(8000, - help="""The public facing port of the proxy. - - This is the port on which the proxy will listen. - This is the only port through which JupyterHub - should be accessed by users. - """ - ).tag(config=True) - base_url = URLPrefix('/', - help="""The base URL of the entire application. - - Add this to the begining of all JupyterHub URLs. - Use base_url to run JupyterHub within an existing website. - """ - ).tag(config=True) logo_file = Unicode('', help="Specify path to a logo image to override the Jupyter logo in the banner." ).tag(config=True) From fdbb1dad79611cdb9a7fee55d038e2ab85c1d531 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 4 May 2018 12:36:59 +0200 Subject: [PATCH 7/7] use bind_url in tests --- jupyterhub/app.py | 6 ++++++ jupyterhub/tests/mocking.py | 16 ++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/jupyterhub/app.py b/jupyterhub/app.py index f729d628..b4af74e0 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -360,6 +360,12 @@ class JupyterHub(Application): """ ).tag(config=True) + @default('base_url') + def _default_base_url(self): + # call validate to ensure leading/trailing slashes + print(self.bind_url) + return JupyterHub.base_url.validate(self, urlparse(self.bind_url).path) + subdomain_host = Unicode('', help="""Run single-user servers on subdomains of this host. diff --git a/jupyterhub/tests/mocking.py b/jupyterhub/tests/mocking.py index 19b29b85..792779e5 100644 --- a/jupyterhub/tests/mocking.py +++ b/jupyterhub/tests/mocking.py @@ -213,17 +213,21 @@ class MockHub(JupyterHub): """Hub with various mock bits""" db_file = None - last_activity_interval = 2 - - base_url = '/@/space%20word/' - log_datefmt = '%M:%S' - + @default('subdomain_host') def _subdomain_host_default(self): return os.environ.get('JUPYTERHUB_TEST_SUBDOMAIN_HOST', '') - + + @default('bind_url') + def _default_bind_url(self): + if self.subdomain_host: + port = urlparse(self.subdomain_host).port + else: + port = random_port() + return 'http://127.0.0.1:%i/@/space%%20word/' % port + @default('ip') def _ip_default(self): return '127.0.0.1'