diff --git a/jupyterhub/alembic.ini b/jupyterhub/alembic.ini new file mode 100644 index 00000000..a7354c41 --- /dev/null +++ b/jupyterhub/alembic.ini @@ -0,0 +1,66 @@ +# A generic, single database configuration. + +[alembic] +script_location = {alembic_dir} +sqlalchemy.url = {db_url} + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to jupyterhub/alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat jupyterhub/alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/jupyterhub/alembic/README b/jupyterhub/alembic/README new file mode 100644 index 00000000..6c9b8092 --- /dev/null +++ b/jupyterhub/alembic/README @@ -0,0 +1 @@ +This is the alembic configuration for JupyterHub data base migrations. \ No newline at end of file diff --git a/jupyterhub/alembic/env.py b/jupyterhub/alembic/env.py new file mode 100644 index 00000000..058378b9 --- /dev/null +++ b/jupyterhub/alembic/env.py @@ -0,0 +1,70 @@ +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/jupyterhub/alembic/script.py.mako b/jupyterhub/alembic/script.py.mako new file mode 100644 index 00000000..43c09401 --- /dev/null +++ b/jupyterhub/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/jupyterhub/alembic/versions/19c0846f6344_base_revision_for_0_5.py b/jupyterhub/alembic/versions/19c0846f6344_base_revision_for_0_5.py new file mode 100644 index 00000000..cdc02ce0 --- /dev/null +++ b/jupyterhub/alembic/versions/19c0846f6344_base_revision_for_0_5.py @@ -0,0 +1,24 @@ +"""base revision for 0.5 + +Revision ID: 19c0846f6344 +Revises: +Create Date: 2016-04-11 16:05:34.873288 + +""" + +# revision identifiers, used by Alembic. +revision = '19c0846f6344' +down_revision = None +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/jupyterhub/alembic/versions/eeb276e51423_auth_state.py b/jupyterhub/alembic/versions/eeb276e51423_auth_state.py new file mode 100644 index 00000000..8b7dfe77 --- /dev/null +++ b/jupyterhub/alembic/versions/eeb276e51423_auth_state.py @@ -0,0 +1,26 @@ +"""auth_state + +Adds auth_state column to Users table. + +Revision ID: eeb276e51423 +Revises: 19c0846f6344 +Create Date: 2016-04-11 16:06:49.239831 +""" + +# revision identifiers, used by Alembic. +revision = 'eeb276e51423' +down_revision = '19c0846f6344' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +from jupyterhub.orm import JSONDict + +def upgrade(): + op.add_column('users', sa.Column('auth_state', JSONDict)) + + +def downgrade(): + # sqlite cannot downgrade because of limited ALTER TABLE support (no DROP COLUMN) + op.drop_column('users', 'auth_state') diff --git a/jupyterhub/app.py b/jupyterhub/app.py index ccb72fc3..15d379d8 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -46,7 +46,7 @@ import jupyterhub from . import handlers, apihandlers from .handlers.static import CacheControlStaticFilesHandler, LogoHandler -from . import orm +from . import dbutil, orm from .user import User, UserDict from ._data import DATA_FILES_PATH from .log import CoroutineLogFormatter, log_request @@ -103,6 +103,7 @@ SECRET_BYTES = 2048 # the number of bytes to use when generating new secrets class NewToken(Application): """Generate and print a new API token""" name = 'jupyterhub-token' + version = jupyterhub.__version__ description = """Generate and return new API token for a user. Usage: @@ -142,6 +143,26 @@ class NewToken(Application): token = user.new_api_token() print(token) +class UpgradeDB(Application): + """Upgrade the JupyterHub database schema.""" + + name = 'jupyterhub-upgrade-db' + version = jupyterhub.__version__ + description = """Upgrade the JupyterHub database to the current schema. + + Usage: + + jupyterhub upgrade-db + """ + aliases = common_aliases + classes = [] + + def start(self): + hub = JupyterHub(parent=self) + hub.load_config_file(hub.config_file) + self.log.info("Upgrading %s", hub.db_url) + dbutil.upgrade(hub.db_url) + class JupyterHub(Application): """An Application for starting a Multi-User Jupyter Notebook server.""" @@ -170,7 +191,8 @@ class JupyterHub(Application): flags = Dict(flags) subcommands = { - 'token': (NewToken, "Generate an API token for a user") + 'token': (NewToken, "Generate an API token for a user"), + 'upgrade-db': (UpgradeDB, "Upgrade your JupyterHub state database to the current version."), } classes = List([ @@ -706,6 +728,11 @@ class JupyterHub(Application): self.log.debug("Database error was:", exc_info=True) if self.db_url.startswith('sqlite:///'): self._check_db_path(self.db_url.split(':///', 1)[1]) + self.log.critical('\n'.join([ + "If you recently upgraded JupyterHub, try running", + " jupyterhub upgrade-db", + "to upgrade your JupyterHub database schema", + ])) self.exit(1) def init_hub(self): @@ -1308,6 +1335,7 @@ class JupyterHub(Application): print("\nInterrupted") NewToken.classes.append(JupyterHub) +UpgradeDB.classes.append(JupyterHub) main = JupyterHub.launch_instance diff --git a/jupyterhub/dbutil.py b/jupyterhub/dbutil.py new file mode 100644 index 00000000..d88c5f42 --- /dev/null +++ b/jupyterhub/dbutil.py @@ -0,0 +1,82 @@ +"""Database utilities for JupyterHub""" +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Based on pgcontents.utils.migrate, used under the Apache license. + +from contextlib import contextmanager +import os +from subprocess import check_call +import sys +from tempfile import TemporaryDirectory + +_here = os.path.abspath(os.path.dirname(__file__)) + +ALEMBIC_INI_TEMPLATE_PATH = os.path.join(_here, 'alembic.ini') +ALEMBIC_DIR = os.path.join(_here, 'alembic') + + +def write_alembic_ini(alembic_ini='alembic.ini', db_url='sqlite:///jupyterhub.sqlite'): + """Write a complete alembic.ini from our template. + + Parameters + ---------- + + alembic_ini: str + path to the alembic.ini file that should be written. + db_url: str + The SQLAlchemy database url, e.g. `sqlite:///jupyterhub.sqlite`. + """ + with open(ALEMBIC_INI_TEMPLATE_PATH) as f: + alembic_ini_tpl = f.read() + + with open(alembic_ini, 'w') as f: + f.write( + alembic_ini_tpl.format( + alembic_dir=ALEMBIC_DIR, + db_url=db_url, + ) + ) + + + +@contextmanager +def _temp_alembic_ini(db_url): + """Context manager for temporary JupyterHub alembic directory + + Temporarily write an alembic.ini file for use with alembic migration scripts. + + Context manager yields alembic.ini path. + + Parameters + ---------- + + db_url: str + The SQLAlchemy database url, e.g. `sqlite:///jupyterhub.sqlite`. + + Returns + ------- + + alembic_ini: str + The path to the temporary alembic.ini that we have created. + This file will be cleaned up on exit from the context manager. + """ + with TemporaryDirectory() as td: + alembic_ini = os.path.join(td, 'alembic.ini') + write_alembic_ini(alembic_ini, db_url) + yield alembic_ini + + +def upgrade(db_url, revision='head'): + """Upgrade the given database to revision. + + db_url: str + The SQLAlchemy database url, e.g. `sqlite:///jupyterhub.sqlite`. + revision: str [default: head] + The alembic revision to upgrade to. + """ + with _temp_alembic_ini(db_url) as alembic_ini: + check_call( + ['alembic', '-c', alembic_ini, 'upgrade', revision] + ) + diff --git a/jupyterhub/orm.py b/jupyterhub/orm.py index 2f780459..23d2eb33 100644 --- a/jupyterhub/orm.py +++ b/jupyterhub/orm.py @@ -285,7 +285,11 @@ class User(Base): api_tokens = relationship("APIToken", backref="user") cookie_id = Column(Unicode(1023), default=new_token) + # User.state is actually Spawner state + # We will need to figure something else out if/when we have multiple spawners per user state = Column(JSONDict) + # Authenticators can store their state here: + auth_state = Column(JSONDict) other_user_cookies = set([]) diff --git a/jupyterhub/tests/old-jupyterhub.sqlite b/jupyterhub/tests/old-jupyterhub.sqlite new file mode 100644 index 00000000..f43267de Binary files /dev/null and b/jupyterhub/tests/old-jupyterhub.sqlite differ diff --git a/jupyterhub/tests/test_db.py b/jupyterhub/tests/test_db.py new file mode 100644 index 00000000..4d844d04 --- /dev/null +++ b/jupyterhub/tests/test_db.py @@ -0,0 +1,40 @@ +import os +import shutil + +from sqlalchemy.exc import OperationalError +from pytest import raises + +from ..dbutil import upgrade +from ..app import NewToken, UpgradeDB, JupyterHub + + +here = os.path.dirname(__file__) +old_db = os.path.join(here, 'old-jupyterhub.sqlite') + +def generate_old_db(path): + db_path = os.path.join(path, "jupyterhub.sqlite") + print(old_db, db_path) + shutil.copy(old_db, db_path) + return 'sqlite:///%s' % db_path + +def test_upgrade(tmpdir): + print(tmpdir) + db_url = generate_old_db(str(tmpdir)) + print(db_url) + upgrade(db_url) + +def test_upgrade_entrypoint(tmpdir): + generate_old_db(str(tmpdir)) + tmpdir.chdir() + tokenapp = NewToken() + tokenapp.initialize(['kaylee']) + with raises(OperationalError): + tokenapp.start() + + upgradeapp = UpgradeDB() + upgradeapp.initialize([]) + upgradeapp.start() + + # run tokenapp again, it should work + tokenapp.start() + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index df714b52..febbf042 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +alembic traitlets>=4.1 tornado>=4.1 jinja2 diff --git a/setup.py b/setup.py index b5a5c490..cb3aef9f 100755 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ def get_data_files(): """Get data files in share/jupyter""" data_files = [] - ntrim = len(here) + 1 + ntrim = len(here + os.path.sep) for (d, dirs, filenames) in os.walk(share_jupyter): data_files.append(( @@ -64,6 +64,18 @@ def get_data_files(): )) return data_files +def get_package_data(): + """Get package data + + (mostly alembic config) + """ + package_data = {} + pkg = pjoin(here, 'jupyterhub') + package_data['jupyterhub'] = [ + 'alembic/*', + 'alembic/versions/*', + ] + return package_data ns = {} with open(pjoin(here, 'jupyterhub', 'version.py')) as f: @@ -82,6 +94,7 @@ setup_args = dict( # dummy, so that install_data doesn't get skipped # this will be overridden when bower is run anyway data_files = get_data_files() or ['dummy'], + package_data = get_package_data(), version = ns['__version__'], description = "JupyterHub: A multi-user server for Jupyter notebooks", long_description = "See https://jupyterhub.readthedocs.org for more info.",