Merge pull request #508 from minrk/alembic

Use alembic for database migrations

jupyterhub/alembic.ini

@@ -0,0 +1,66 @@
+# A generic, single database configuration.
+
+[alembic]
+script_location = {alembic_dir}
+sqlalchemy.url = {db_url}
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to jupyterhub/alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat jupyterhub/alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

jupyterhub/alembic/README

@@ -0,0 +1 @@
+This is the alembic configuration for JupyterHub data base migrations.

jupyterhub/alembic/env.py

@@ -0,0 +1,70 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

jupyterhub/alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

jupyterhub/alembic/versions/19c0846f6344_base_revision_for_0_5.py

@@ -0,0 +1,24 @@
+"""base revision for 0.5
+
+Revision ID: 19c0846f6344
+Revises: 
+Create Date: 2016-04-11 16:05:34.873288
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '19c0846f6344'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

jupyterhub/alembic/versions/eeb276e51423_auth_state.py

@@ -0,0 +1,26 @@
+"""auth_state
+
+Adds auth_state column to Users table.
+
+Revision ID: eeb276e51423
+Revises: 19c0846f6344
+Create Date: 2016-04-11 16:06:49.239831
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'eeb276e51423'
+down_revision = '19c0846f6344'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+from jupyterhub.orm import JSONDict
+
+def upgrade():
+    op.add_column('users', sa.Column('auth_state', JSONDict))
+
+
+def downgrade():
+    # sqlite cannot downgrade because of limited ALTER TABLE support (no DROP COLUMN)
+    op.drop_column('users', 'auth_state')

jupyterhub/app.py

@@ -46,7 +46,7 @@ import jupyterhub
 from . import handlers, apihandlers
 from .handlers.static import CacheControlStaticFilesHandler, LogoHandler
 
-from . import orm
+from . import dbutil, orm
 from .user import User, UserDict
 from ._data import DATA_FILES_PATH
 from .log import CoroutineLogFormatter, log_request
@@ -103,6 +103,7 @@ SECRET_BYTES = 2048 # the number of bytes to use when generating new secrets
 class NewToken(Application):
     """Generate and print a new API token"""
     name = 'jupyterhub-token'
+    version = jupyterhub.__version__
     description = """Generate and return new API token for a user.
 
     Usage:
@@ -142,6 +143,26 @@ class NewToken(Application):
         token = user.new_api_token()
         print(token)
 
+class UpgradeDB(Application):
+    """Upgrade the JupyterHub database schema."""
+    
+    name = 'jupyterhub-upgrade-db'
+    version = jupyterhub.__version__
+    description = """Upgrade the JupyterHub database to the current schema.
+    
+    Usage:
+
+        jupyterhub upgrade-db
+    """
+    aliases = common_aliases
+    classes = []
+    
+    def start(self):
+        hub = JupyterHub(parent=self)
+        hub.load_config_file(hub.config_file)
+        self.log.info("Upgrading %s", hub.db_url)
+        dbutil.upgrade(hub.db_url)
+
 
 class JupyterHub(Application):
     """An Application for starting a Multi-User Jupyter Notebook server."""
@@ -170,7 +191,8 @@ class JupyterHub(Application):
     flags = Dict(flags)
 
     subcommands = {
-        'token': (NewToken, "Generate an API token for a user")
+        'token': (NewToken, "Generate an API token for a user"),
+        'upgrade-db': (UpgradeDB, "Upgrade your JupyterHub state database to the current version."),
     }
 
     classes = List([
@@ -706,6 +728,11 @@ class JupyterHub(Application):
             self.log.debug("Database error was:", exc_info=True)
             if self.db_url.startswith('sqlite:///'):
                 self._check_db_path(self.db_url.split(':///', 1)[1])
+            self.log.critical('\n'.join([
+                "If you recently upgraded JupyterHub, try running",
+                "    jupyterhub upgrade-db",
+                "to upgrade your JupyterHub database schema",
+            ]))
             self.exit(1)
 
     def init_hub(self):
@@ -1308,6 +1335,7 @@ class JupyterHub(Application):
             print("\nInterrupted")
 
 NewToken.classes.append(JupyterHub)
+UpgradeDB.classes.append(JupyterHub)
 
 main = JupyterHub.launch_instance
 

jupyterhub/dbutil.py

@@ -0,0 +1,82 @@
+"""Database utilities for JupyterHub"""
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+# Based on pgcontents.utils.migrate, used under the Apache license.
+
+from contextlib import contextmanager
+import os
+from subprocess import check_call
+import sys
+from tempfile import TemporaryDirectory
+
+_here = os.path.abspath(os.path.dirname(__file__))
+
+ALEMBIC_INI_TEMPLATE_PATH = os.path.join(_here, 'alembic.ini')
+ALEMBIC_DIR = os.path.join(_here, 'alembic')
+
+
+def write_alembic_ini(alembic_ini='alembic.ini', db_url='sqlite:///jupyterhub.sqlite'):
+    """Write a complete alembic.ini from our template.
+    
+    Parameters
+    ----------
+    
+    alembic_ini: str
+        path to the alembic.ini file that should be written.
+    db_url: str
+        The SQLAlchemy database url, e.g. `sqlite:///jupyterhub.sqlite`.
+    """
+    with open(ALEMBIC_INI_TEMPLATE_PATH) as f:
+        alembic_ini_tpl = f.read()
+    
+    with open(alembic_ini, 'w') as f:
+        f.write(
+            alembic_ini_tpl.format(
+                alembic_dir=ALEMBIC_DIR,
+                db_url=db_url,
+            )
+        )
+    
+
+
+@contextmanager
+def _temp_alembic_ini(db_url):
+    """Context manager for temporary JupyterHub alembic directory
+    
+    Temporarily write an alembic.ini file for use with alembic migration scripts.
+    
+    Context manager yields alembic.ini path.
+    
+    Parameters
+    ----------
+    
+    db_url: str
+        The SQLAlchemy database url, e.g. `sqlite:///jupyterhub.sqlite`.
+    
+    Returns
+    -------
+    
+    alembic_ini: str
+        The path to the temporary alembic.ini that we have created.
+        This file will be cleaned up on exit from the context manager.
+    """
+    with TemporaryDirectory() as td:
+        alembic_ini = os.path.join(td, 'alembic.ini')
+        write_alembic_ini(alembic_ini, db_url)
+        yield alembic_ini
+
+
+def upgrade(db_url, revision='head'):
+    """Upgrade the given database to revision.
+    
+    db_url: str
+        The SQLAlchemy database url, e.g. `sqlite:///jupyterhub.sqlite`.
+    revision: str [default: head]
+        The alembic revision to upgrade to.
+    """
+    with _temp_alembic_ini(db_url) as alembic_ini:
+        check_call(
+            ['alembic', '-c', alembic_ini, 'upgrade', revision]
+        )
+

jupyterhub/orm.py

@@ -285,7 +285,11 @@ class User(Base):
 
     api_tokens = relationship("APIToken", backref="user")
     cookie_id = Column(Unicode(1023), default=new_token)
+    # User.state is actually Spawner state
+    # We will need to figure something else out if/when we have multiple spawners per user
     state = Column(JSONDict)
+    # Authenticators can store their state here:
+    auth_state = Column(JSONDict)
 
     other_user_cookies = set([])
 

jupyterhub/tests/test_db.py

@@ -0,0 +1,40 @@
+import os
+import shutil
+
+from sqlalchemy.exc import OperationalError
+from pytest import raises
+
+from ..dbutil import upgrade
+from ..app import NewToken, UpgradeDB, JupyterHub
+
+
+here = os.path.dirname(__file__)
+old_db = os.path.join(here, 'old-jupyterhub.sqlite')
+
+def generate_old_db(path):
+    db_path = os.path.join(path, "jupyterhub.sqlite")
+    print(old_db, db_path)
+    shutil.copy(old_db, db_path)
+    return 'sqlite:///%s' % db_path
+
+def test_upgrade(tmpdir):
+    print(tmpdir)
+    db_url = generate_old_db(str(tmpdir))
+    print(db_url)
+    upgrade(db_url)
+
+def test_upgrade_entrypoint(tmpdir):
+    generate_old_db(str(tmpdir))
+    tmpdir.chdir()
+    tokenapp = NewToken()
+    tokenapp.initialize(['kaylee'])
+    with raises(OperationalError):
+        tokenapp.start()
+    
+    upgradeapp = UpgradeDB()
+    upgradeapp.initialize([])
+    upgradeapp.start()
+    
+    # run tokenapp again, it should work
+    tokenapp.start()
+    

requirements.txt

@@ -1,3 +1,4 @@
+alembic
 traitlets>=4.1
 tornado>=4.1
 jinja2

setup.py

@@ -55,7 +55,7 @@ def get_data_files():
     """Get data files in share/jupyter"""
     
     data_files = []
-    ntrim = len(here) + 1
+    ntrim = len(here + os.path.sep)
     
     for (d, dirs, filenames) in os.walk(share_jupyter):
         data_files.append((
@@ -64,6 +64,18 @@ def get_data_files():
         ))
     return data_files
 
+def get_package_data():
+    """Get package data
+
+    (mostly alembic config)
+    """
+    package_data = {}
+    pkg = pjoin(here, 'jupyterhub')
+    package_data['jupyterhub'] = [
+        'alembic/*',
+        'alembic/versions/*',
+    ]
+    return package_data
 
 ns = {}
 with open(pjoin(here, 'jupyterhub', 'version.py')) as f:
@@ -82,6 +94,7 @@ setup_args = dict(
                         # dummy, so that install_data doesn't get skipped
                         # this will be overridden when bower is run anyway
     data_files          = get_data_files() or ['dummy'],
+    package_data        = get_package_data(),
     version             = ns['__version__'],
     description         = "JupyterHub: A multi-user server for Jupyter notebooks",
     long_description    = "See https://jupyterhub.readthedocs.org for more info.",