Merge pull request #2545 from minrk/changelog-1.0

releasing 1.0

.travis.yml

@@ -27,8 +27,11 @@ before_install:
       unset MYSQL_UNIX_PORT
       DB=mysql bash ci/docker-db.sh
       DB=mysql bash ci/init-db.sh
-      pip install 'mysql-connector-python'
+      # FIXME: mysql-connector-python 8.0.16 incorrectly decodes bytes to str
+      # ref: https://bugs.mysql.com/bug.php?id=94944
+      pip install 'mysql-connector-python==8.0.15'
     elif [[ $JUPYTERHUB_TEST_DB_URL == postgresql* ]]; then
+      psql -c "CREATE USER $PGUSER WITH PASSWORD '$PGPASSWORD';" -U postgres
       DB=postgres bash ci/init-db.sh
       pip install psycopg2-binary
     fi
@@ -87,7 +90,10 @@ matrix:
         - JUPYTERHUB_TEST_DB_URL=mysql+mysqlconnector://root@127.0.0.1:$MYSQL_TCP_PORT/jupyterhub
     - python: 3.6
       env:
-        - JUPYTERHUB_TEST_DB_URL=postgresql://postgres@127.0.0.1/jupyterhub
+        - PGUSER=jupyterhub
+        - PGPASSWORD=hub[test/:?
+        # password in url is url-encoded (urllib.parse.quote($PGPASSWORD, safe=''))
+        - JUPYTERHUB_TEST_DB_URL=postgresql://jupyterhub:hub%5Btest%2F%3A%3F@127.0.0.1/jupyterhub
     - python: 3.7
       dist: xenial
   allow_failures:

README.md

@@ -150,7 +150,7 @@ To start the Hub on a specific url and port ``10.0.1.2:443`` with **https**:
 | PAMAuthenticator                                                            | Default, built-in authenticator                   |
 | [OAuthenticator](https://github.com/jupyterhub/oauthenticator)              | OAuth + JupyterHub Authenticator = OAuthenticator |
 | [ldapauthenticator](https://github.com/jupyterhub/ldapauthenticator)        | Simple LDAP Authenticator Plugin for JupyterHub   |
-| [kdcAuthenticator](https://github.com/bloomberg/jupyterhub-kdcauthenticator)| Kerberos Authenticator Plugin for JupyterHub      |
+| [kerberosauthenticator](https://github.com/jcrist/kerberosauthenticator)    | Kerberos Authenticator Plugin for JupyterHub      |
 
 ### Spawners
 
@@ -162,6 +162,7 @@ To start the Hub on a specific url and port ``10.0.1.2:443`` with **https**:
 | [sudospawner](https://github.com/jupyterhub/sudospawner)       | Spawn single-user servers without being root                               |
 | [systemdspawner](https://github.com/jupyterhub/systemdspawner) | Spawn single-user notebook servers using systemd                           |
 | [batchspawner](https://github.com/jupyterhub/batchspawner)     | Designed for clusters using batch scheduling software                      |
+| [yarnspawner](https://github.com/jcrist/yarnspawner)           | Spawn single-user notebook servers distributed on a Hadoop cluster         |
 | [wrapspawner](https://github.com/jupyterhub/wrapspawner)       | WrapSpawner and ProfilesSpawner enabling runtime configuration of spawners |
 
 ## Docker

ci/docker-db.sh

@@ -2,7 +2,7 @@
 # source this file to setup postgres and mysql
 # for local testing (as similar as possible to docker)
 
-set -e
+set -eu
 
 export MYSQL_HOST=127.0.0.1
 export MYSQL_TCP_PORT=${MYSQL_TCP_PORT:-13306}
@@ -40,6 +40,15 @@ for i in {1..60}; do
 done
 $CHECK
 
+case "$DB" in
+"mysql")
+  ;;
+"postgres")
+  # create the user
+  psql --user postgres -c "CREATE USER $PGUSER WITH PASSWORD '$PGPASSWORD';"
+  ;;
+*)
+esac
 
 echo -e "
 Set these environment variables:

ci/init-db.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # initialize jupyterhub databases for testing
 
-set -e
+set -eu
 
 MYSQL="mysql --user root --host $MYSQL_HOST --port $MYSQL_TCP_PORT -e "
 PSQL="psql --user postgres -c "
@@ -23,5 +23,5 @@ set -x
 
 for SUFFIX in '' _upgrade_072 _upgrade_081 _upgrade_094; do
     $SQL "DROP DATABASE jupyterhub${SUFFIX};" 2>/dev/null || true
-    $SQL "CREATE DATABASE jupyterhub${SUFFIX} ${EXTRA_CREATE};"
+    $SQL "CREATE DATABASE jupyterhub${SUFFIX} ${EXTRA_CREATE:-};"
 done

docs/source/changelog.md

@@ -9,7 +9,7 @@ command line for details.
 
 ## 1.0
 
-### [1.0.0] 2019-04-XX
+### [1.0.0] 2019-05-03
 
 JupyterHub 1.0 is a major milestone for JupyterHub.
 Huge thanks to the many people who have contributed to this release,
@@ -577,7 +577,7 @@ First preview release
 
 
 [Unreleased]: https://github.com/jupyterhub/jupyterhub/compare/1.0.0...HEAD
-[1.0.0]: https://github.com/jupyterhub/jupyterhub/compare/0.9.5...HEAD
+[1.0.0]: https://github.com/jupyterhub/jupyterhub/compare/0.9.6...1.0.0
 [0.9.6]: https://github.com/jupyterhub/jupyterhub/compare/0.9.4...0.9.6
 [0.9.4]: https://github.com/jupyterhub/jupyterhub/compare/0.9.3...0.9.4
 [0.9.3]: https://github.com/jupyterhub/jupyterhub/compare/0.9.2...0.9.3

docs/source/gallery-jhub-deployments.md

@@ -66,7 +66,7 @@ easy to do with RStudio too.
 
 ### University of Colorado Boulder
 
-- (CU Research Computing) CURC 
+- (CU Research Computing) CURC
     - [JupyterHub User Guide](https://www.rc.colorado.edu/support/user-guide/jupyterhub.html)
         - Slurm job dispatched on Crestone compute cluster
         - log troubleshooting
@@ -77,13 +77,17 @@ easy to do with RStudio too.
 - Earth Lab at CU
     - [Tutorial on Parallel R on JupyterHub](https://earthdatascience.org/tutorials/parallel-r-on-jupyterhub/)
 
+### George Washington University
+
+- [Jupyter Hub](http://go.gwu.edu/jupyter) with university single-sign-on. Deployed early 2017.
+
 ### HTCondor
 
 - [HTCondor Python Bindings Tutorial from HTCondor Week 2017 includes information on their JupyterHub tutorials](https://research.cs.wisc.edu/htcondor/HTCondorWeek2017/presentations/TueBockelman_Python.pdf)
 
 ### University of Illinois
 
-- https://datascience.business.illinois.edu
+- https://datascience.business.illinois.edu (currently down; checked 04/26/19)
 
 ### IllustrisTNG Simulation Project
 
@@ -110,7 +114,11 @@ easy to do with RStudio too.
 - [Data Science (DICE) group](https://dice.cs.uni-paderborn.de/)
     - [nbgraderutils](https://github.com/dice-group/nbgraderutils): Use JupyterHub + nbgrader + iJava kernel for online Java exercises. Used in lecture Statistical Natural Language Processing.
 
-### University of Rochester CIRC 
+### Penn State University
+
+- [Press release](https://news.psu.edu/story/523093/2018/05/24/new-open-source-web-apps-available-students-and-faculty): "New open-source web apps available for students and faculty" (but Hub is currently down; checked 04/26/19)
+
+### University of Rochester CIRC
 
 - [JupyterHub Userguide](https://info.circ.rochester.edu/Web_Applications/JupyterHub.html) - Slurm, beehive
 
@@ -126,7 +134,7 @@ easy to do with RStudio too.
 
 - Educational Technology Services - Paul Jamason
     - [jupyterhub.ucsd.edu](https://jupyterhub.ucsd.edu)
-    
+
 ### TACC University of Texas
 
 ### Texas A&M
@@ -160,6 +168,10 @@ easy to do with RStudio too.
 - https://getcarina.com/blog/learning-how-to-whale/
 - http://carolynvanslyck.com/talk/carina/jupyterhub/#/
 
+### Hadoop
+
+- [Deploying JupyterHub on Hadoop](https://jcrist.github.io/jupyterhub-on-hadoop/)
+
 
 ## Miscellaneous
 

docs/source/index.rst

@@ -2,21 +2,37 @@
 JupyterHub
 ==========
 
-`JupyterHub`_, a multi-user **Hub**, spawns, manages, and proxies multiple
+`JupyterHub`_ is the best way to serve `Jupyter notebook`_ for multiple users. 
+It can be used in a classes of students, a corporate data science group or scientific 
+research group. It is a multi-user **Hub** that spawns, manages, and proxies multiple
 instances of the single-user `Jupyter notebook`_ server.
-JupyterHub can be used to serve notebooks to a class of students, a corporate
-data science group, or a scientific research group.
 
-.. image:: images/jhub-parts.png
+To make life easier, JupyterHub have distributions. Be sure to 
+take a look at them before continuing with the configuration of the broad 
+original system of `JupyterHub`_. Today, you can find two main cases: 
+
+1. If you need a simple case for a small amount of users (0-100) and single server 
+   take a look at 
+   `The Littlest JupyterHub <https://github.com/jupyterhub/the-littlest-jupyterhub>`__ distribution. 
+2. If you need to allow for even more users, a dynamic amount of servers can be used on a cloud,
+   take a look at the `Zero to JupyterHub with Kubernetes <https://github.com/jupyterhub/zero-to-jupyterhub-k8s>`__ .
+
+
+Four subsystems make up JupyterHub:
+
+* a **Hub** (tornado process) that is the heart of JupyterHub
+* a **configurable http proxy** (node-http-proxy) that receives the requests from the client's browser
+* multiple **single-user Jupyter notebook servers** (Python/IPython/tornado) that are monitored by Spawners
+* an **authentication class** that manages how users can access the system
+
+
+Besides these central pieces, you can add optional configurations through a `config.py` file and manage users kernels on an admin panel. A simplification of the whole system can be seen in the figure below:
+
+.. image:: images/jhub-fluxogram.jpeg
    :alt: JupyterHub subsystems
-   :width: 40%
-   :align: right
+   :width: 80%
+   :align: center
 
-Three subsystems make up JupyterHub:
-
-* a multi-user **Hub** (tornado process)
-* a **configurable http proxy** (node-http-proxy)
-* multiple **single-user Jupyter notebook servers** (Python/IPython/tornado)
 
 JupyterHub performs the following functions:
 

docs/source/reference/spawners.md

@@ -25,6 +25,8 @@ Some examples include:
   run without being root, by spawning an intermediate process via `sudo`
 - [BatchSpawner](https://github.com/jupyterhub/batchspawner) for spawning remote
   servers using batch systems
+- [YarnSpawner](https://github.com/jcrist/yarnspawner) for spawning notebook
+  servers in YARN containers on a Hadoop cluster
 - [RemoteSpawner](https://github.com/zonca/remotespawner) to spawn notebooks
   and a remote server and tunnel the port via SSH
 

docs/source/reference/websecurity.md

@@ -127,3 +127,11 @@ A handy website for testing your deployment is
 
 
 [configurable-http-proxy]: https://github.com/jupyterhub/configurable-http-proxy
+
+## Vulnerability reporting
+
+If you believe you’ve found a security vulnerability in JupyterHub, or any
+Jupyter project, please report it to
+[security@ipython.org](mailto:security@iypthon.org). If you prefer to encrypt
+your security reports, you can use [this PGP public
+key](https://jupyter-notebook.readthedocs.io/en/stable/_downloads/ipython_security.asc).

docs/sphinxext/autodoc_traits.py

@@ -38,17 +38,18 @@ class TraitDocumenter(AttributeDocumenter):
     def can_document_member(cls, member, membername, isattr, parent):
         return isinstance(member, TraitType)
 
-    def format_name(self):
-        return 'config c.' + super().format_name()
-
     def add_directive_header(self, sig):
         default = self.object.get_default_value()
         if default is Undefined:
             default_s = ''
         else:
             default_s = repr(default)
-        sig = ' = {}({})'.format(self.object.__class__.__name__, default_s)
-        return super().add_directive_header(sig)
+        self.options.annotation = 'c.{name} = {trait}({default})'.format(
+            name=self.format_name(),
+            trait=self.object.__class__.__name__,
+            default=default_s,
+        )
+        super().add_directive_header(sig)
 
 
 def setup(app):

jupyterhub/_version.py

@@ -6,7 +6,7 @@ version_info = (
     1,
     0,
     0,
-    "b2",  # release (b1, rc1, or "" for final or dev)
+    # "b2",  # release (b1, rc1, or "" for final or dev)
     # "dev",  # dev or nothing
 )
 

jupyterhub/app.py

@@ -277,7 +277,7 @@ class JupyterHub(Application):
                     try:
                         cls = entry_point.load()
                     except Exception as e:
-                        self.log.warning(
+                        self.log.debug(
                             "Failed to load %s entrypoint %r: %r",
                             trait.entry_point_group,
                             key,
@@ -2184,7 +2184,6 @@ class JupyterHub(Application):
             self.log.info("Cleaning up PID file %s", self.pid_file)
             os.remove(self.pid_file)
 
-        # finally stop the loop once we are all cleaned up
         self.log.info("...done")
 
     def write_config_file(self):
@@ -2422,37 +2421,51 @@ class JupyterHub(Application):
 
         self.log.info("JupyterHub is now running at %s", self.proxy.public_url)
         # register cleanup on both TERM and INT
-        atexit.register(self.atexit)
         self.init_signal()
 
     def init_signal(self):
-        signal.signal(signal.SIGTERM, self.sigterm)
+        loop = asyncio.get_event_loop()
+        for s in (signal.SIGTERM, signal.SIGINT):
+            loop.add_signal_handler(
+                s, lambda s=s: asyncio.ensure_future(self.shutdown_cancel_tasks(s))
+            )
+        infosignals = [signal.SIGUSR1]
         if hasattr(signal, 'SIGINFO'):
-            signal.signal(signal.SIGINFO, self.log_status)
+            infosignals.append(signal.SIGINFO)
+        for s in infosignals:
+            loop.add_signal_handler(
+                s, lambda s=s: asyncio.ensure_future(self.log_status(s))
+            )
 
-    def log_status(self, signum, frame):
+    async def log_status(self, sig):
         """Log current status, triggered by SIGINFO (^T in many terminals)"""
-        self.log.debug("Received signal %s[%s]", signum, signal.getsignal(signum))
+        self.log.critical("Received signal %s...", sig.name)
         print_ps_info()
         print_stacks()
 
-    def sigterm(self, signum, frame):
-        self.log.critical("Received SIGTERM, shutting down")
-        raise SystemExit(128 + signum)
+    async def shutdown_cancel_tasks(self, sig):
+        """Cancel all other tasks of the event loop and initiate cleanup"""
+        self.log.critical("Received signal %s, initiating shutdown...", sig.name)
+        tasks = [
+            t for t in asyncio.Task.all_tasks() if t is not asyncio.Task.current_task()
+        ]
 
-    _atexit_ran = False
+        if tasks:
+            self.log.debug("Cancelling pending tasks")
+            [t.cancel() for t in tasks]
 
-    def atexit(self):
-        """atexit callback"""
-        if self._atexit_ran:
-            return
-        self._atexit_ran = True
-        # run the cleanup step (in a new loop, because the interrupted one is unclean)
-        asyncio.set_event_loop(asyncio.new_event_loop())
-        IOLoop.clear_current()
-        loop = IOLoop()
-        loop.make_current()
-        loop.run_sync(self.cleanup)
+            try:
+                await asyncio.wait(tasks)
+            except asyncio.CancelledError as e:
+                self.log.debug("Caught Task CancelledError. Ignoring")
+            except StopAsyncIteration as e:
+                self.log.error("Caught StopAsyncIteration Exception", exc_info=True)
+
+            tasks = [t for t in asyncio.Task.all_tasks()]
+            for t in tasks:
+                self.log.debug("Task status: %s", t)
+        await self.cleanup()
+        asyncio.get_event_loop().stop()
 
     def stop(self):
         if not self.io_loop:
@@ -2479,6 +2492,9 @@ class JupyterHub(Application):
             loop.start()
         except KeyboardInterrupt:
             print("\nInterrupted")
+        finally:
+            loop.stop()
+            loop.close()
 
 
 NewToken.classes.append(JupyterHub)

jupyterhub/handlers/login.py

@@ -18,33 +18,73 @@ class LogoutHandler(BaseHandler):
     def shutdown_on_logout(self):
         return self.settings.get('shutdown_on_logout', False)
 
-    async def get(self):
+    async def _shutdown_servers(self, user):
+        """Shutdown servers for logout
+
+        Get all active servers for the provided user, stop them.
+        """
+        active_servers = [
+            name
+            for (name, spawner) in user.spawners.items()
+            if spawner.active and not spawner.pending
+        ]
+        if active_servers:
+            self.log.info("Shutting down %s's servers", user.name)
+            futures = []
+            for server_name in active_servers:
+                futures.append(maybe_future(self.stop_single_user(user, server_name)))
+            await asyncio.gather(*futures)
+
+    def _backend_logout_cleanup(self, name):
+        """Default backend logout actions
+
+        Send a log message, clear some cookies, increment the logout counter.
+        """
+        self.log.info("User logged out: %s", name)
+        self.clear_login_cookie()
+        self.statsd.incr('logout')
+
+    async def default_handle_logout(self):
+        """The default logout action
+
+        Optionally cleans up servers, clears cookies, increments logout counter
+        Cleaning up servers can be prevented by setting shutdown_on_logout to
+        False.
+        """
         user = self.current_user
         if user:
             if self.shutdown_on_logout:
-                active_servers = [
-                    name
-                    for (name, spawner) in user.spawners.items()
-                    if spawner.active and not spawner.pending
-                ]
-                if active_servers:
-                    self.log.info("Shutting down %s's servers", user.name)
-                    futures = []
-                    for server_name in active_servers:
-                        futures.append(
-                            maybe_future(self.stop_single_user(user, server_name))
-                        )
-                    await asyncio.gather(*futures)
+                await self._shutdown_servers(user)
 
-            self.log.info("User logged out: %s", user.name)
-            self.clear_login_cookie()
-            self.statsd.incr('logout')
+            self._backend_logout_cleanup(user.name)
+
+    async def handle_logout(self):
+        """Custom user action during logout
+
+        By default a no-op, this function should be overridden in subclasses
+        to have JupyterHub take a custom action on logout.
+        """
+        return
+
+    async def render_logout_page(self):
+        """Render the logout page, if any
+
+        Override this function to set a custom logout page.
+        """
         if self.authenticator.auto_login:
             html = self.render_template('logout.html')
             self.finish(html)
         else:
             self.redirect(self.settings['login_url'], permanent=False)
 
+    async def get(self):
+        """Log the user out, call the custom action, forward the user
+            to the logout page
+        """
+        await self.default_handle_logout()
+        await self.handle_logout()
+        await self.render_logout_page()
+
 
 class LoginHandler(BaseHandler):
     """Render the login page."""

jupyterhub/tests/test_db.py

@@ -30,7 +30,7 @@ def generate_old_db(env_dir, hub_version, db_url):
     if 'mysql' in db_url:
         pkgs.append('mysql-connector-python')
     elif 'postgres' in db_url:
-        pkgs.append('psycopg2')
+        pkgs.append('psycopg2-binary')
     check_call([env_pip, 'install'] + pkgs)
     check_call([env_py, populate_db, db_url])
 

jupyterhub/tests/test_pages.py

@@ -393,7 +393,7 @@ async def test_user_redirect(app, username):
     path = urlparse(r.url).path
     while '/spawn-pending/' in path:
         await asyncio.sleep(0.1)
-        r = await get_page(r.url, app, cookies=cookies)
+        r = await async_requests.get(r.url, cookies=cookies)
         path = urlparse(r.url).path
     assert path == ujoin(app.base_url, '/user/%s/notebooks/test.ipynb' % name)
 

jupyterhub/tests/utils.py

@@ -159,6 +159,10 @@ async def api_request(
 
 
 def get_page(path, app, hub=True, **kw):
+    if "://" in path:
+        raise ValueError(
+            "Not a hub page path: %r. Did you mean async_requests.get?" % path
+        )
     if hub:
         prefix = app.hub.base_url
     else: