diff --git a/examples/docker-compose/.gitignore b/examples/docker-compose/.gitignore new file mode 100644 index 00000000..e43b0f98 --- /dev/null +++ b/examples/docker-compose/.gitignore @@ -0,0 +1 @@ +.DS_Store diff --git a/examples/docker-compose/README.md b/examples/docker-compose/README.md new file mode 100644 index 00000000..38893a39 --- /dev/null +++ b/examples/docker-compose/README.md @@ -0,0 +1,85 @@ +This folder contains files and sub-folders that demonstrate how to run docker-stack notebook containers using Docker Compose on a Docker Machine-controlled host. + +## Pre-requisites + +* [Docker Engine](https://docs.docker.com/engine/) 1.10.0+ +* [Docker Machine](https://docs.docker.com/machine/) 0.6.0+ +* [Docker Compose](https://docs.docker.com/compose/) 1.6.0+ + +See the [installation instructions](https://docs.docker.com/engine/installation/) for your environment. + +## Provision Docker machines + +### Provision a VirtualBox VM on local desktop + +``` +bin/vbox.sh mymachine +``` + +### Provision a virtual device on IBM SoftLayer + +``` +export SOFTLAYER_USER=my_softlayer_username +export SOFTLAYER_API_KEY=my_softlayer_api_key +export SOFTLAYER_DOMAIN=my.domain + +# Create virtual device +bin/softlayer.sh myhost + +# Add DNS entry (SoftLayer DNS zone must exist for SOFTLAYER_DOMAIN) +bin/sl-dns.sh myhost +``` + +## Deploy stand-alone Jupyter Notebook + +Build and run a `jupyter/minimal-notebook` container on an existing Docker machine. + +``` +# activate docker machine +eval "$(docker-machine env mymachine)" + +# build notebook image on the machine +notebook/build.sh + +# bring up notebook container +notebook/up.sh +``` + +To stop and remove the container: + +``` +notebook/down.sh +``` + +See [notebook README](notebook/README.md) for more details. + +## Let's Encrypt + +If you want to secure access to publicly addressable notebook containers, you can generate a free certificate using the [Let's Encrypt](https://letsencrypt.org) service. + +The following command creates a Docker volume, runs the `letsencrypt` client to create a full-chain certificate and private key, and stores them in the volume. Note: The script uses several `letsencrypt` options, one of which automatically agrees to the Let's Encrypt Terms of Service. + +``` +FQDN=host.mydomain.com EMAIL=myemail@somewhere.com bin/letsencrypt.sh +``` + +Be aware that Let's Encrypt has a pretty [low rate limit per domain](https://community.letsencrypt.org/t/public-beta-rate-limits/4772/3) at the moment. You can avoid exhausting your limit by testing against the Let's Encrypt staging servers. To hit their staging servers, set the environment variable `CERT_SERVER=--staging`. + +``` +FQDN=host.mydomain.com EMAIL=myemail@somewhere.com \ + CERT_SERVER=--staging \ + bin/letsencrypt.sh +``` + +Also, be aware that Let's Encrypt certificates are short lived (90 days). If you need them for a longer period of time, you'll need to manually setup a cron job to run the renewal steps. (You can reuse the command above.) + +## Troubleshooting + +### Unable to connect to VirtualBox VM on Mac OS X when using Cisco VPN client. + +The Cisco VPN client blocks access to IP addresses that it does not know about, and may block access to a new VM if it is created while the Cisco VPN client is running. + +1. Stop Cisco VPN client. (It does not allow modifications to route table). +2. Run `ifconfig` to list `vboxnet` virtual network devices. +3. Run `sudo route -nv add -net 192.168.99 -interface vboxnetX`, where X is the number of the virtual device assigned to the VirtualBox VM. +4. Start Cisco VPN client. diff --git a/examples/docker-compose/bin/letsencrypt.sh b/examples/docker-compose/bin/letsencrypt.sh new file mode 100755 index 00000000..72c3d6ec --- /dev/null +++ b/examples/docker-compose/bin/letsencrypt.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Use https://letsencrypt.org to create a certificate for a single domain +# and store it in a Docker volume. + +set -e + +# Get domain and email from environment +[ -z "$FQDN" ] && \ + echo "ERROR: Must set FQDN environment varable" && \ + exit 1 + +[ -z "$EMAIL" ] && \ + echo "ERROR: Must set EMAIL environment varable" && \ + exit 1 + +# letsencrypt certificate server type (default is production). +# Set `CERT_SERVER=--staging` for staging. +: ${CERT_SERVER=''} + +# Create Docker volume to contain the cert +: ${SECRETS_VOLUME:=my-notebook-secrets} +docker volume create --name $SECRETS_VOLUME 1>/dev/null +# Generate the cert and save it to the Docker volume +docker run --rm -it \ + -p 80:80 \ + -v $SECRETS_VOLUME:/etc/letsencrypt \ + quay.io/letsencrypt/letsencrypt:latest \ + certonly \ + --non-interactive \ + --keep-until-expiring \ + --standalone \ + --standalone-supported-challenges http-01 \ + --agree-tos \ + --domain "$FQDN" \ + --email "$EMAIL" \ + $CERT_SERVER + +# Set permissions so nobody can read the cert and key. +# Also symlink the certs into the root of the /etc/letsencrypt +# directory so that the FQDN doesn't have to be known later. +docker run --rm -it \ + -v $SECRETS_VOLUME:/etc/letsencrypt \ + debian:jessie \ + bash -c "ln -s /etc/letsencrypt/live/$FQDN/* /etc/letsencrypt/ && \ + find /etc/letsencrypt -type d -exec chmod 755 {} +" diff --git a/examples/docker-compose/bin/sl-dns.sh b/examples/docker-compose/bin/sl-dns.sh new file mode 100755 index 00000000..354bf07c --- /dev/null +++ b/examples/docker-compose/bin/sl-dns.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +set -e + +# User must have slcli installed +which slcli > /dev/null || (echo "SoftLayer cli not found (pip install softlayer)"; exit 1) + +USAGE="Usage: `basename $0` machine_name [domain]" +E_BADARGS=85 + +# Machine name is first command line arg +MACHINE_NAME=$1 && [ -z "$MACHINE_NAME" ] && echo "$USAGE" && exit $E_BADARGS + +# Use SOFTLAYER_DOMAIN env var if domain name not set as second arg +DOMAIN="${2:-$SOFTLAYER_DOMAIN}" && [ -z "$DOMAIN" ] && \ + echo "Must specify domain or set SOFTLAYER_DOMAIN environment varable" && \ + echo "$USAGE" && exit $E_BADARGS + +IP=$(docker-machine ip "$MACHINE_NAME") + +slcli dns record-add $DOMAIN $MACHINE_NAME A $IP diff --git a/examples/docker-compose/bin/softlayer.sh b/examples/docker-compose/bin/softlayer.sh new file mode 100755 index 00000000..6c3fb536 --- /dev/null +++ b/examples/docker-compose/bin/softlayer.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Set default SoftLayer VM settings +: ${SOFTLAYER_CPU:=4} +export SOFTLAYER_CPU +: ${SOFTLAYER_DISK_SIZE:=100} +export SOFTLAYER_DISK_SIZE +: ${SOFTLAYER_MEMORY:=4096} +export SOFTLAYER_MEMORY +: ${SOFTLAYER_REGION:=wdc01} +export SOFTLAYER_REGION + +docker-machine create --driver softlayer "$@" diff --git a/examples/docker-compose/bin/vbox.sh b/examples/docker-compose/bin/vbox.sh new file mode 100755 index 00000000..8dfdc721 --- /dev/null +++ b/examples/docker-compose/bin/vbox.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Set reasonable default VM settings +: ${VIRTUALBOX_CPUS:=4} +export VIRTUALBOX_CPUS +: ${VIRTUALBOX_MEMORY_SIZE:=4096} +export VIRTUALBOX_MEMORY_SIZE + +docker-machine create --driver virtualbox "$@" diff --git a/examples/docker-compose/notebook/Dockerfile b/examples/docker-compose/notebook/Dockerfile new file mode 100644 index 00000000..f364738c --- /dev/null +++ b/examples/docker-compose/notebook/Dockerfile @@ -0,0 +1,15 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Pick your favorite docker-stacks image +FROM jupyter/minimal-notebook:55d5ca6be183 + +USER jovyan + +# Add permanent pip/conda installs, data files, other user libs here +# e.g., RUN pip install jupyter_dashboards + +USER root + +# Add permanent apt-get installs and other root commands here +# e.g., RUN apt-get install npm nodejs diff --git a/examples/docker-compose/notebook/README.md b/examples/docker-compose/notebook/README.md new file mode 100644 index 00000000..5150992d --- /dev/null +++ b/examples/docker-compose/notebook/README.md @@ -0,0 +1,96 @@ +Run a docker-stack notebook container using Docker Compose on a Docker Machine-controlled host. + +## Pre-requisites + +* [Docker Engine](https://docs.docker.com/engine/) 1.10.0+ +* [Docker Machine](https://docs.docker.com/machine/) 0.6.0+ +* [Docker Compose](https://docs.docker.com/compose/) 1.6.0+ + +See the [installation instructions](https://docs.docker.com/engine/installation/) for your environment. + +## Quickstart + +Here's how to build and run a `jupyter/minimal-notebook` container on an existing Docker machine. + +``` +# activate docker machine +eval "$(docker-machine env mymachine)" + +# build notebook image on the machine +notebook/build.sh + +# bring up notebook container +notebook/up.sh +``` + +To stop and remove the container: + +``` +notebook/down.sh +``` + +## FAQ + +### Can I run multiple notebook containers on the same VM? + +Yes. Set environment variables to specify unique names and ports when running the `up.sh` command. + +``` +NAME=my-notebook PORT=9000 notebook/up.sh +NAME=your-notebook PORT=9001 notebook/up.sh +``` + +To stop and remove the containers: + +``` +NAME=my-notebook notebook/down.sh +NAME=your-notebook notebook/down.sh +``` + +### Where are my notebooks stored? + +The `up.sh` creates a Docker volume named after the notebook container with a `-work` suffix, e.g., `my-notebook-work`. + + +### Can multiple notebook containers share the same notebook volume? + +Yes. Set the `WORK_VOLUME` environment variable to the same value for each notebook. + +``` +NAME=my-notebook PORT=9000 WORK_VOLUME=our-work notebook/up.sh +NAME=your-notebook PORT=9001 WORK_VOLUME=our-work notebook/up.sh +``` + +### How do I run over HTTPS? + +To run the notebook server with a self-signed certificate, pass the `--secure` option to the `up.sh` script. You must also provide a password, which will be used to secure the notebook server. You can specify the password by setting the `PASSWORD` environment variable, or by passing it to the `up.sh` script. + +``` +PASSWORD=a_secret notebook/up.sh --secure + +# or +notebook/up.sh --secure --password a_secret +``` + +To use a real certificate from Let's Encrypt, first run the `bin/letsencrypt.sh` script to create the certificate chain and store it in a Docker volume. + +``` +FQDN=host.mydomain.com EMAIL=myemail@somewhere.com bin/letsencrypt.sh +``` + +The following command will store the certificate chain in a Docker volume named `mydomain-secrets`. + +``` +FQDN=host.mydomain.com EMAIL=myemail@somewhere.com \ + SECRETS_VOLUME=mydomain-secrets \ + bin/letsencrypt.sh +``` + +Now run `up.sh` with the `--letsencrypt` option. You must also provide the name of the secrets volume and a password. + +``` +PASSWORD=a_secret SECRETS_VOLUME=mydomain-secrets notebook/up.sh --letsencrypt + +# or +notebook/up.sh --letsencrypt --password a_secret --secrets mydomain-secrets +``` diff --git a/examples/docker-compose/notebook/build.sh b/examples/docker-compose/notebook/build.sh new file mode 100755 index 00000000..6a9dc502 --- /dev/null +++ b/examples/docker-compose/notebook/build.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# Setup environment +source "$DIR/env.sh" + +# Build the notebook image +docker-compose -f "$DIR/notebook.yml" build diff --git a/examples/docker-compose/notebook/down.sh b/examples/docker-compose/notebook/down.sh new file mode 100755 index 00000000..fd472fc5 --- /dev/null +++ b/examples/docker-compose/notebook/down.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# Setup environment +source "$DIR/env.sh" + +# Bring down the notebook container, using container name as project name +docker-compose -f "$DIR/notebook.yml" -p "$NAME" down diff --git a/examples/docker-compose/notebook/env.sh b/examples/docker-compose/notebook/env.sh new file mode 100755 index 00000000..551d2666 --- /dev/null +++ b/examples/docker-compose/notebook/env.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Set default values for environment variables required by notebook compose +# configuration file. + +# Container name +: "${NAME:=my-notebook}" +export NAME + +# Exposed container port +: ${PORT:=80} +export PORT + +# Container work volume name +: "${WORK_VOLUME:=$NAME-work}" +export WORK_VOLUME + +# Container secrets volume name +: "${SECRETS_VOLUME:=$NAME-secrets}" +export SECRETS_VOLUME diff --git a/examples/docker-compose/notebook/letsencrypt-notebook.yml b/examples/docker-compose/notebook/letsencrypt-notebook.yml new file mode 100644 index 00000000..8e918f61 --- /dev/null +++ b/examples/docker-compose/notebook/letsencrypt-notebook.yml @@ -0,0 +1,30 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +version: "2" + +services: + notebook: + build: . + image: my-notebook + container_name: ${NAME} + volumes: + - "work:/home/jovyan/work" + - "secrets:/etc/letsencrypt" + ports: + - "${PORT}:8888" + environment: + USE_HTTPS: "yes" + PASSWORD: ${PASSWORD} + command: > + start-notebook.sh + --NotebookApp.certfile=/etc/letsencrypt/fullchain.pem + --NotebookApp.keyfile=/etc/letsencrypt/privkey.pem + +volumes: + work: + external: + name: ${WORK_VOLUME} + secrets: + external: + name: ${SECRETS_VOLUME} diff --git a/examples/docker-compose/notebook/notebook.yml b/examples/docker-compose/notebook/notebook.yml new file mode 100644 index 00000000..52bcb2ee --- /dev/null +++ b/examples/docker-compose/notebook/notebook.yml @@ -0,0 +1,19 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +version: "2" + +services: + notebook: + build: . + image: my-notebook + container_name: ${NAME} + volumes: + - "work:/home/jovyan/work" + ports: + - "${PORT}:8888" + +volumes: + work: + external: + name: ${WORK_VOLUME} diff --git a/examples/docker-compose/notebook/secure-notebook.yml b/examples/docker-compose/notebook/secure-notebook.yml new file mode 100644 index 00000000..5c4e53a4 --- /dev/null +++ b/examples/docker-compose/notebook/secure-notebook.yml @@ -0,0 +1,22 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +version: "2" + +services: + notebook: + build: . + image: my-notebook + container_name: ${NAME} + volumes: + - "work:/home/jovyan/work" + ports: + - "${PORT}:8888" + environment: + USE_HTTPS: "yes" + PASSWORD: ${PASSWORD} + +volumes: + work: + external: + name: ${WORK_VOLUME} diff --git a/examples/docker-compose/notebook/up.sh b/examples/docker-compose/notebook/up.sh new file mode 100755 index 00000000..074c3852 --- /dev/null +++ b/examples/docker-compose/notebook/up.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +USAGE="Usage: `basename $0` [--secure | --letsencrypt] [--password PASSWORD] [--secrets SECRETS_VOLUME]" + +# Parse args to determine security settings +SECURE=${SECURE:=no} +LETSENCRYPT=${LETSENCRYPT:=no} +while [[ $# > 0 ]] +do +key="$1" +case $key in + --secure) + SECURE=yes + ;; + --letsencrypt) + LETSENCRYPT=yes + ;; + --secrets) + SECRETS_VOLUME="$2" + shift # past argument + ;; + --password) + PASSWORD="$2" + export PASSWORD + shift # past argument + ;; + *) # unknown option + ;; +esac +shift # past argument or value +done + +if [[ "$LETSENCRYPT" == yes || "$SECURE" == yes ]]; then + if [ -z "${PASSWORD:+x}" ]; then + echo "ERROR: Must set PASSWORD if running in secure mode" + echo "$USAGE" + exit 1 + fi + if [ "$LETSENCRYPT" == yes ]; then + CONFIG=letsencrypt-notebook.yml + if [ -z "${SECRETS_VOLUME:+x}" ]; then + echo "ERROR: Must set SECRETS_VOLUME if running in letsencrypt mode" + echo "$USAGE" + exit 1 + fi + else + CONFIG=secure-notebook.yml + fi + export PORT=${PORT:=443} +else + CONFIG=notebook.yml + export PORT=${PORT:=80} +fi + +# Setup environment +source "$DIR/env.sh" + +# Create a Docker volume to store notebooks +docker volume create --name "$WORK_VOLUME" + +# Bring up a notebook container, using container name as project name +echo "Bringing up notebook '$NAME'" +docker-compose -f "$DIR/$CONFIG" -p "$NAME" up -d + +IP=$(docker-machine ip $(docker-machine active)) +echo "Notebook $NAME listening on $IP:$PORT"