Merge branch 'master' into add_pagination_admin

docs/environment.yml

@@ -2,15 +2,18 @@
 # if you change the dependencies of JupyterHub in the various `requirements.txt`
 name: jhub_docs
 channels:
-  - conda-forge
+  # use metachannel for faster solves / less memory on RTD
+  # which has memory issues with full conda-forge
+  # only need to list 'leaf' dependencies here
+  - https://metachannel.conda-forge.org/conda-forge/jupyterhub,sphinx,recommonmark
 dependencies:
 - pip
-- nodejs
+- nodejs=12
-- python=3.6
+- python=3.7
 - alembic
 - jinja2
 - pamela
-- recommonmark==0.6.0
+- recommonmark>=0.6
 - requests
 - sqlalchemy>=1
 - tornado>=5.0

docs/requirements.txt

@@ -4,7 +4,7 @@
 alabaster_jupyterhub
 autodoc-traits
 git+https://github.com/pandas-dev/pandas-sphinx-theme.git@master
-recommonmark==0.5.0
+recommonmark>=0.6
 sphinx-copybutton
 sphinx-jsonschema
 sphinx>=1.7

docs/source/contributing/tests.rst

@@ -64,5 +64,5 @@ Troubleshooting Test Failures
 All the tests are failing
 -------------------------
 
-Make sure you have completed all the steps in :ref:`contributing/setup` sucessfully, and
+Make sure you have completed all the steps in :ref:`contributing/setup` successfully, and
 can launch ``jupyterhub`` from the terminal.

docs/source/getting-started/faq.md

@@ -0,0 +1,36 @@
+# Frequently asked questions
+
+
+### How do I share links to notebooks?
+
+In short, where you see `/user/name/notebooks/foo.ipynb` use `/hub/user-redirect/notebooks/foo.ipynb` (replace `/user/name` with `/hub/user-redirect`).
+
+Sharing links to notebooks is a common activity,
+and can look different based on what you mean.
+Your first instinct might be to copy the URL you see in the browser,
+e.g. `hub.jupyter.org/user/yourname/notebooks/coolthing.ipynb`.
+However, let's break down what this URL means:
+
+`hub.jupyter.org/user/yourname/` is the URL prefix handled by *your server*,
+which means that sharing this URL is asking the person you share the link with
+to come to *your server* and look at the exact same file.
+In most circumstances, this is forbidden by permissions because the person you share with does not have access to your server.
+What actually happens when someone visits this URL will depend on whether your server is running and other factors.
+
+But what is our actual goal?
+A typical situation is that you have some shared or common filesystem,
+such that the same path corresponds to the same document
+(either the exact same document or another copy of it).
+Typically, what folks want when they do sharing like this
+is for each visitor to open the same file *on their own server*,
+so Breq would open `/user/breq/notebooks/foo.ipynb` and
+Seivarden would open `/user/seivarden/notebooks/foo.ipynb`, etc.
+
+JupyterHub has a special URL that does exactly this!
+It's called `/hub/user-redirect/...` and after the visitor logs in,
+So if you replace `/user/yourname` in your URL bar
+with `/hub/user-redirect` any visitor should get the same
+URL on their own server, rather than visiting yours.
+
+In JupyterLab 2.0, this should also be the result of the "Copy Shareable Link"
+action in the file browser.

docs/source/getting-started/index.rst

@@ -15,4 +15,5 @@ own JupyterHub.
    authenticators-users-basics
    spawners-basics
    services-basics
+   faq
    institutional-faq

docs/source/getting-started/security-basics.rst

@@ -80,6 +80,49 @@ To achieve this, simply omit the configuration settings
 ``c.JupyterHub.ssl_key`` and ``c.JupyterHub.ssl_cert``
 (setting them to ``None`` does not have the same effect, and is an error).
 
+.. _authentication-token:
+
+Proxy authentication token
+--------------------------
+
+The Hub authenticates its requests to the Proxy using a secret token that
+the Hub and Proxy agree upon. Note that this applies to the default
+``ConfigurableHTTPProxy`` implementation. Not all proxy implementations
+use an auth token.
+
+The value of this token should be a random string (for example, generated by
+``openssl rand -hex 32``). You can store it in the configuration file or an
+environment variable
+
+Generating and storing token in the configuration file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can set the value in the configuration file, ``jupyterhub_config.py``:
+
+.. code-block:: python
+
+    c.ConfigurableHTTPProxy.api_token = 'abc123...' # any random string
+
+Generating and storing as an environment variable
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can pass this value of the proxy authentication token to the Hub and Proxy
+using the ``CONFIGPROXY_AUTH_TOKEN`` environment variable:
+
+.. code-block:: bash
+
+    export CONFIGPROXY_AUTH_TOKEN=$(openssl rand -hex 32)
+
+This environment variable needs to be visible to the Hub and Proxy.
+
+Default if token is not set
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you don't set the Proxy authentication token, the Hub will generate a random
+key itself, which means that any time you restart the Hub you **must also
+restart the Proxy**. If the proxy is a subprocess of the Hub, this should happen
+automatically (this is the default configuration).
+
 .. _cookie-secret:
 
 Cookie secret
@@ -146,41 +189,73 @@ itself, ``jupyterhub_config.py``, as a binary string:
    If the cookie secret value changes for the Hub, all single-user notebook
    servers must also be restarted.
 
+.. _cookies:
 
-.. _authentication-token:
+Cookies used by JupyterHub authentication
+-----------------------------------------
 
-Proxy authentication token
+The following cookies are used by the Hub for handling user authentication.
---------------------------
 
-The Hub authenticates its requests to the Proxy using a secret token that
+This section was created based on this post_ from Discourse.
-the Hub and Proxy agree upon. The value of this string should be a random
-string (for example, generated by ``openssl rand -hex 32``).
 
-Generating and storing token in the configuration file
+.. _post: https://discourse.jupyter.org/t/how-to-force-re-login-for-users/1998/6
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Or you can set the value in the configuration file, ``jupyterhub_config.py``:
+jupyterhub-hub-login
+~~~~~~~~~~~~~~~~~~~~
 
-.. code-block:: python
+This is the login token used when visiting Hub-served pages that are
+protected by authentication such as the main home, the spawn form, etc.
+If this cookie is set, then the user is logged in.
 
-    c.JupyterHub.proxy_auth_token = '0bc02bede919e99a26de1e2a7a5aadfaf6228de836ec39a05a6c6942831d8fe5'
+Resetting the Hub cookie secret effectively revokes this cookie.
 
-Generating and storing as an environment variable
+This cookie is restricted to the path ``/hub/``.
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You can pass this value of the proxy authentication token to the Hub and Proxy
+jupyterhub-user-<username>
-using the ``CONFIGPROXY_AUTH_TOKEN`` environment variable:
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. code-block:: bash
+This is the cookie used for authenticating with a single-user server.
+It is set by the single-user server after OAuth with the Hub.
 
-    export CONFIGPROXY_AUTH_TOKEN=$(openssl rand -hex 32)
+Effectively the same as ``jupyterhub-hub-login``, but for the
+single-user server instead of the Hub. It contains an OAuth access token,
+which is checked with the Hub to authenticate the browser.
 
-This environment variable needs to be visible to the Hub and Proxy.
+Each OAuth access token is associated with a session id (see ``jupyterhub-session-id`` section
+below).
 
-Default if token is not set
+To avoid hitting the Hub on every request, the authentication response
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+is cached. And to avoid a stale cache the cache key is comprised of both
+the token and session id.
 
-If you don't set the Proxy authentication token, the Hub will generate a random
+Resetting the Hub cookie secret effectively revokes this cookie.
-key itself, which means that any time you restart the Hub you **must also
+
-restart the Proxy**. If the proxy is a subprocess of the Hub, this should happen
+This cookie is restricted to the path ``/user/<username>``, so that
-automatically (this is the default configuration).
+only the user’s server receives it.
+
+jupyterhub-session-id
+~~~~~~~~~~~~~~~~~~~~~
+
+This is a random string, meaningless in itself, and the only cookie
+shared by the Hub and single-user servers.
+
+Its sole purpose is to coordinate logout of the multiple OAuth cookies.
+
+This cookie is set to ``/`` so all endpoints can receive it, or clear it, etc.
+
+jupyterhub-user-<username>-oauth-state
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A short-lived cookie, used solely to store and validate OAuth state.
+It is only set while OAuth between the single-user server and the Hub
+is processing.
+
+If you use your browser development tools, you should see this cookie
+for a very brief moment before your are logged in,
+with an expiration date shorter than ``jupyterhub-hub-login`` or
+``jupyterhub-user-<username>``.
+
+This cookie should not exist after you have successfully logged in.
+
+This cookie is restricted to the path ``/user/<username>``, so that only
+the user’s server receives it.

docs/source/index.rst

@@ -3,7 +3,7 @@ JupyterHub
 ==========
 
 `JupyterHub`_ is the best way to serve `Jupyter notebook`_ for multiple users.
-It can be used in a classes of students, a corporate data science group or scientific
+It can be used in a class of students, a corporate data science group or scientific
 research group. It is a multi-user **Hub** that spawns, manages, and proxies multiple
 instances of the single-user `Jupyter notebook`_ server.
 

docs/source/reference/services.md

@@ -360,7 +360,7 @@ and taking note of the following process:
 
 An example of using an Externally-Managed Service and authentication is
 in [nbviewer README][nbviewer example] section on securing the notebook viewer,
-and an example of its configuration is found [here](https://github.com/jupyter/nbviewer/blob/master/nbviewer/providers/base.py#L94).
+and an example of its configuration is found [here](https://github.com/jupyter/nbviewer/blob/ed942b10a52b6259099e2dd687930871dc8aac22/nbviewer/providers/base.py#L95).
 nbviewer can also be run as a Hub-Managed Service as described [nbviewer README][nbviewer example]
 section on securing the notebook viewer.
 

docs/source/troubleshooting.md

@@ -269,7 +269,7 @@ where `ssl_cert` is example-chained.crt and ssl_key to your private key.
 
 Then restart JupyterHub.
 
-See also [JupyterHub SSL encryption](getting-started.md#ssl-encryption).
+See also [JupyterHub SSL encryption](./getting-started/security-basics.html#ssl-encryption).
 
 ### Install JupyterHub without a network connection
 

jupyterhub/app.py

@@ -576,6 +576,22 @@ class JupyterHub(Application):
         """,
     ).tag(config=True)
 
+    @validate('bind_url')
+    def _validate_bind_url(self, proposal):
+        """ensure protocol field of bind_url matches ssl"""
+        v = proposal['value']
+        proto, sep, rest = v.partition('://')
+        if self.ssl_cert and proto != 'https':
+            return 'https' + sep + rest
+        elif proto != 'http' and not self.ssl_cert:
+            return 'http' + sep + rest
+        return v
+
+    @default('bind_url')
+    def _bind_url_default(self):
+        proto = 'https' if self.ssl_cert else 'http'
+        return proto + '://:8000'
+
     subdomain_host = Unicode(
         '',
         help="""Run single-user servers on subdomains of this host.
@@ -917,6 +933,25 @@ class JupyterHub(Application):
     def _authenticator_default(self):
         return self.authenticator_class(parent=self, db=self.db)
 
+    implicit_spawn_seconds = Float(
+        0,
+        help="""Trigger implicit spawns after this many seconds.
+
+        When a user visits a URL for a server that's not running,
+        they are shown a page indicating that the requested server
+        is not running with a button to spawn the server.
+
+        Setting this to a positive value will redirect the user
+        after this many seconds, effectively clicking this button
+        automatically for the users,
+        automatically beginning the spawn process.
+
+        Warning: this can result in errors and surprising behavior
+        when sharing access URLs to actual servers,
+        since the wrong server is likely to be started.
+        """,
+    ).tag(config=True)
+
     allow_named_servers = Bool(
         False, help="Allow named single-user servers per user"
     ).tag(config=True)
@@ -1670,9 +1705,12 @@ class JupyterHub(Application):
         # This lets whitelist be used to set up initial list,
         # but changes to the whitelist can occur in the database,
         # and persist across sessions.
+        total_users = 0
         for user in db.query(orm.User):
             try:
-                await maybe_future(self.authenticator.add_user(user))
+                f = self.authenticator.add_user(user)
+                if f:
+                    await maybe_future(f)
             except Exception:
                 self.log.exception("Error adding user %s already in db", user.name)
                 if self.authenticator.delete_invalid_users:
@@ -1694,6 +1732,7 @@ class JupyterHub(Application):
                         )
                     )
             else:
+                total_users += 1
                 # handle database upgrades where user.created is undefined.
                 # we don't want to allow user.created to be undefined,
                 # so initialize it to last_activity (if defined) or now.
@@ -1705,6 +1744,8 @@ class JupyterHub(Application):
         # From this point on, any user changes should be done simultaneously
         # to the whitelist set and user db, unless the whitelist is empty (all users allowed).
 
+        TOTAL_USERS.set(total_users)
+
     async def init_groups(self):
         """Load predefined groups into the database"""
         db = self.db
@@ -2005,21 +2046,23 @@ class JupyterHub(Application):
             spawner._check_pending = False
 
         # parallelize checks for running Spawners
+        # run query on extant Server objects
+        # so this is O(running servers) not O(total users)
         check_futures = []
-        for orm_user in db.query(orm.User):
+        for orm_server in db.query(orm.Server):
-            user = self.users[orm_user]
+            orm_spawners = orm_server.spawner
-            self.log.debug("Loading state for %s from db", user.name)
+            if not orm_spawners:
-            for name, orm_spawner in user.orm_spawners.items():
+                continue
-                if orm_spawner.server is not None:
+            orm_spawner = orm_spawners[0]
-                    # spawner should be running
+            # instantiate Spawner wrapper and check if it's still alive
-                    # instantiate Spawner wrapper and check if it's still alive
+            # spawner should be running
-                    spawner = user.spawners[name]
+            user = self.users[orm_spawner.user]
-                    # signal that check is pending to avoid race conditions
+            spawner = user.spawners[orm_spawner.name]
-                    spawner._check_pending = True
+            self.log.debug("Loading state for %s from db", spawner._log_name)
-                    f = asyncio.ensure_future(check_spawner(user, name, spawner))
+            # signal that check is pending to avoid race conditions
-                    check_futures.append(f)
+            spawner._check_pending = True
-
+            f = asyncio.ensure_future(check_spawner(user, spawner.name, spawner))
-        TOTAL_USERS.set(len(self.users))
+            check_futures.append(f)
 
         # it's important that we get here before the first await
         # so that we know all spawners are instantiated and in the check-pending state
@@ -2158,6 +2201,7 @@ class JupyterHub(Application):
             subdomain_host=self.subdomain_host,
             domain=self.domain,
             statsd=self.statsd,
+            implicit_spawn_seconds=self.implicit_spawn_seconds,
             allow_named_servers=self.allow_named_servers,
             default_server_name=self._default_server_name,
             named_server_limit_per_user=self.named_server_limit_per_user,

jupyterhub/handlers/base.py

@@ -1426,11 +1426,12 @@ class UserUrlHandler(BaseHandler):
         # serve a page prompting for spawn and 503 error
         # visiting /user/:name no longer triggers implicit spawn
         # without explicit user action
-        self.set_status(503)
         spawn_url = url_concat(
             url_path_join(self.hub.base_url, "spawn", user.escaped_name, server_name),
             {"next": self.request.uri},
         )
+        self.set_status(503)
+
         auth_state = await user.get_auth_state()
         html = self.render_template(
             "not_running.html",
@@ -1438,6 +1439,7 @@ class UserUrlHandler(BaseHandler):
             server_name=server_name,
             spawn_url=spawn_url,
             auth_state=auth_state,
+            implicit_spawn_seconds=self.settings.get("implicit_spawn_seconds", 0),
         )
         self.finish(html)
 

jupyterhub/orm.py

@@ -230,7 +230,7 @@ class Spawner(Base):
     user_id = Column(Integer, ForeignKey('users.id', ondelete='CASCADE'))
 
     server_id = Column(Integer, ForeignKey('servers.id', ondelete='SET NULL'))
-    server = relationship(Server, cascade="all")
+    server = relationship(Server, backref='spawner', cascade="all")
 
     state = Column(JSONDict)
     name = Column(Unicode(255))
@@ -282,7 +282,7 @@ class Service(Base):
 
     # service-specific interface
     _server_id = Column(Integer, ForeignKey('servers.id', ondelete='SET NULL'))
-    server = relationship(Server, cascade='all')
+    server = relationship(Server, backref='service', cascade='all')
     pid = Column(Integer)
 
     def new_api_token(self, token=None, **kwargs):

jupyterhub/tests/test_spawner.py

@@ -103,7 +103,8 @@ async def wait_for_spawner(spawner, timeout=10):
 
 
 async def test_single_user_spawner(app, request):
-    user = next(iter(app.users.values()), None)
+    orm_user = app.db.query(orm.User).first()
+    user = app.users[orm_user]
     spawner = user.spawner
     spawner.cmd = ['jupyterhub-singleuser']
     await user.spawn()

share/jupyterhub/templates/not_running.html

@@ -23,7 +23,14 @@
         {% endif %}
         Would you like to retry starting it?
         {% else %}
-        Your server {{ server_name }} is not running. Would you like to start it?
+        Your server {{ server_name }} is not running.
+          {% if implicit_spawn_seconds %}
+          It will be restarted automatically.
+          If you are not redirected in a few seconds,
+          click below to launch your server.
+          {% else %}
+          Would you like to start it?
+          {% endif %}
         {% endif %}
       </p>
       {% endblock %}
@@ -42,3 +49,18 @@
 </div>
 
 {% endblock %}
+{% block script %}
+{{ super () }}
+{% if implicit_spawn_seconds %}
+<script type="text/javascript">
+  var spawn_url = "{{ spawn_url }}";
+  var implicit_spawn_seconds = {{ implicit_spawn_seconds }};
+  setTimeout(function () {
+      console.log("redirecting to spawn at", spawn_url);
+      window.location = spawn_url;
+    },
+    1000 * implicit_spawn_seconds
+  );
+</script>
+{% endif %}
+{% endblock script %}

share/jupyterhub/templates/page.html

@@ -34,6 +34,9 @@
     {% block stylesheet %}
     <link rel="stylesheet" href="{{ static_url("css/style.min.css") }}" type="text/css"/>
     {% endblock %}
+    {% block favicon %}
+    <link rel="icon" href="{{ static_url("favicon.ico") }}" type="image/x-icon">
+    {% endblock %}
     {% block scripts %}
     <script src="{{static_url("components/requirejs/require.js") }}" type="text/javascript" charset="utf-8"></script>
     <script src="{{static_url("components/jquery/dist/jquery.min.js") }}" type="text/javascript" charset="utf-8"></script>