From bad4f6860891898f2d7697ab49edeba9184920be Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sun, 18 Feb 2018 20:58:32 +1100 Subject: [PATCH 01/10] Add example Source-to-Image scripts for building custom notebook images. --- examples/source-to-image/assemble | 37 +++++++++++++++++++++++++++++++ examples/source-to-image/run | 5 +++++ 2 files changed, 42 insertions(+) create mode 100755 examples/source-to-image/assemble create mode 100755 examples/source-to-image/run diff --git a/examples/source-to-image/assemble b/examples/source-to-image/assemble new file mode 100755 index 00000000..f3484bfe --- /dev/null +++ b/examples/source-to-image/assemble @@ -0,0 +1,37 @@ +#!/bin/bash + +set -eo pipefail + +# Copy injected files to correct place in 'work' directory. + +cp -Rf /tmp/src/. /home/$NB_USER/work + +rm -rf /tmp/src + +# Install any Python modules. If we find an 'environment.yml' file we +# assume we should use 'conda' to install packages. If 'requirements.txt' +# use 'pip' instead. Ensure we are in the 'work' directory so relative +# directory paths in these files resolve okay if installing packages +# from subdirectories. + +if [ -f /home/$NB_USER/work/environment.yml ]; then + (cd /home/$NB_USER/work && conda env update --name root --file environment.yml) + conda clean -tipsy +else + if [ -f /home/$NB_USER/work/requirements.txt ]; then + (cd /home/$NB_USER/work && pip --no-cache-dir install -r requirements.txt) + fi +fi + +# Remove any 'environment.yml' or 'requirements.txt' file when done in +# case we are producing an image which will in turn be used as an S2I +# builder image. + +rm -f /home/$NB_USER/work/environment.yml +rm -f /home/$NB_USER/work/requirements.txt + +# Fix up permissions on home directory and Python installation so that +# everything is still writable by 'users' group. + +fix-permissions $CONDA_DIR +fix-permissions /home/$NB_USER diff --git a/examples/source-to-image/run b/examples/source-to-image/run new file mode 100755 index 00000000..b5b641b8 --- /dev/null +++ b/examples/source-to-image/run @@ -0,0 +1,5 @@ +#!/bin/bash + +# Start up the notebook instance. + +exec start-notebook.sh "$@" From 92843575a4f4fe87a1eff0aed7f559f7c4221491 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sun, 18 Feb 2018 21:43:00 +1100 Subject: [PATCH 02/10] Add save-artifacts script to Source-to-Image to avoid warnings/errors. --- examples/source-to-image/save-artifacts | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 examples/source-to-image/save-artifacts diff --git a/examples/source-to-image/save-artifacts b/examples/source-to-image/save-artifacts new file mode 100755 index 00000000..0ab29c89 --- /dev/null +++ b/examples/source-to-image/save-artifacts @@ -0,0 +1,3 @@ +#!/bin/bash + +tar cf - --files-from /dev/null From 49fe302e4aa7beb14298b7ab1e2ebd7667a97658 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sun, 18 Feb 2018 21:54:47 +1100 Subject: [PATCH 03/10] Ensure assemble script of Source-to-Image script logs progress. --- examples/source-to-image/assemble | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/source-to-image/assemble b/examples/source-to-image/assemble index f3484bfe..2c80b679 100755 --- a/examples/source-to-image/assemble +++ b/examples/source-to-image/assemble @@ -1,5 +1,7 @@ #!/bin/bash +set -x + set -eo pipefail # Copy injected files to correct place in 'work' directory. From f87dec5936beca9b0b04de4caf4fdc4b34a8f6c1 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sun, 18 Feb 2018 21:58:43 +1100 Subject: [PATCH 04/10] Add template for using Source-to-Image builder scripts with OpenShift. --- examples/source-to-image/templates.json | 266 ++++++++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 examples/source-to-image/templates.json diff --git a/examples/source-to-image/templates.json b/examples/source-to-image/templates.json new file mode 100644 index 00000000..a923450c --- /dev/null +++ b/examples/source-to-image/templates.json @@ -0,0 +1,266 @@ +{ + "kind": "Template", + "apiVersion": "v1", + "metadata": { + "name": "jupyter-notebook-quickstart", + "annotations": { + "openshift.io/display-name": "Jupyter Notebook Quickstart", + "description": "Template for deploying Jupyter Notebook images with bundled notebooks and files.", + "iconClass": "icon-python", + "tags": "python,jupyter" + } + }, + "parameters": [ + { + "name": "APPLICATION_NAME", + "value": "notebook", + "required": true + }, + { + "name": "BUILDER_IMAGE", + "value": "jupyter/minimal-notebook:latest", + "required": true + }, + { + "name": "BUILDER_SCRIPTS", + "value": "https://raw.githubusercontent.com/jupyter/docker-stacks/source-to-image/examples/source-to-image", + "required": true + }, + { + "name": "GIT_REPOSITORY_URL", + "value": "", + "required": true + }, + { + "name": "GIT_REFERENCE", + "value": "master", + "required": true + }, + { + "name": "CONTEXT_DIR", + "value": "", + "required": false + }, + { + "name": "NOTEBOOK_PASSWORD", + "from": "[a-f0-9]{32}", + "generate": "expression" + } + ], + "objects": [ + { + "apiVersion": "v1", + "kind": "ImageStream", + "metadata": { + "name": "${APPLICATION_NAME}", + "labels": { + "app": "${APPLICATION_NAME}" + } + }, + "spec": { + "lookupPolicy": { + "local": false + } + } + }, + { + "apiVersion": "v1", + "kind": "BuildConfig", + "metadata": { + "name": "${APPLICATION_NAME}", + "labels": { + "app": "${APPLICATION_NAME}" + } + }, + "spec": { + "output": { + "to": { + "kind": "ImageStreamTag", + "name": "${APPLICATION_NAME}:latest" + } + }, + "source": { + "type": "Git", + "git": { + "uri": "${GIT_REPOSITORY_URL}", + "ref": "${GIT_REFERENCE}" + }, + "contextDir": "${CONTEXT_DIR}" + }, + "strategy": { + "type": "Source", + "sourceStrategy": { + "from": { + "kind": "DockerImage", + "name": "${BUILDER_IMAGE}" + }, + "scripts": "${BUILDER_SCRIPTS}" + } + }, + "triggers": [ + { + "type": "ConfigChange" + } + ] + } + }, + { + "kind": "ConfigMap", + "apiVersion": "v1", + "metadata": { + "name": "${APPLICATION_NAME}-cfg", + "labels": { + "app": "${APPLICATION_NAME}" + } + }, + "data": { + "jupyter_notebook_config.py": "import os\n\npassword = os.environ.get('JUPYTER_NOTEBOOK_PASSWORD')\n\nif password:\n import notebook.auth\n c.NotebookApp.password = notebook.auth.passwd(password)\n del password\n del os.environ['JUPYTER_NOTEBOOK_PASSWORD']\n\nimage_config_file = '/home/jovyan/work/.jupyter/jupyter_notebook_config.py'\n\nif os.path.exists(image_config_file):\n with open(image_config_file) as fp:\n exec(compile(fp.read(), image_config_file, 'exec'), globals())\n" + } + }, + { + "kind": "DeploymentConfig", + "apiVersion": "v1", + "metadata": { + "name": "${APPLICATION_NAME}", + "labels": { + "app": "${APPLICATION_NAME}" + } + }, + "spec": { + "strategy": { + "type": "Recreate" + }, + "triggers": [ + { + "type": "ConfigChange" + }, + { + "type": "ImageChange", + "imageChangeParams": { + "automatic": true, + "containerNames": [ + "jupyter-notebook" + ], + "from": { + "kind": "ImageStreamTag", + "name": "${APPLICATION_NAME}:latest" + } + } + } + ], + "replicas": 1, + "selector": { + "app": "${APPLICATION_NAME}", + "deploymentconfig": "${APPLICATION_NAME}" + }, + "template": { + "metadata": { + "annotations": { + "alpha.image.policy.openshift.io/resolve-names": "*" + }, + "labels": { + "app": "${APPLICATION_NAME}", + "deploymentconfig": "${APPLICATION_NAME}" + } + }, + "spec": { + "containers": [ + { + "name": "jupyter-notebook", + "image": "${APPLICATION_NAME}:latest", + "command": [ + "start-notebook.sh", + "--config=/home/jovyan/configs/jupyter_notebook_config.py", + "--no-browser", + "--ip=0.0.0.0" + ], + + "ports": [ + { + "containerPort": 8888, + "protocol": "TCP" + } + ], + "env": [ + { + "name": "JUPYTER_NOTEBOOK_PASSWORD", + "value": "${NOTEBOOK_PASSWORD}" + } + ], + "volumeMounts": [ + { + "mountPath": "/home/jovyan/configs", + "name": "configs" + } + ] + } + ], + "securityContext": { + "supplementalGroups": [ + 100 + ] + }, + "volumes": [ + { + "configMap": { + "name": "${APPLICATION_NAME}-cfg" + }, + "name": "configs" + } + ] + } + } + } + }, + { + "kind": "Route", + "apiVersion": "v1", + "metadata": { + "name": "${APPLICATION_NAME}", + "labels": { + "app": "${APPLICATION_NAME}" + } + }, + "spec": { + "host": "", + "to": { + "kind": "Service", + "name": "${APPLICATION_NAME}", + "weight": 100 + }, + "port": { + "targetPort": "8888-tcp" + }, + "tls": { + "termination": "edge", + "insecureEdgeTerminationPolicy": "Redirect" + } + } + }, + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "${APPLICATION_NAME}", + "labels": { + "app": "${APPLICATION_NAME}" + } + }, + "spec": { + "ports": [ + { + "name": "8888-tcp", + "protocol": "TCP", + "port": 8888, + "targetPort": 8888 + } + ], + "selector": { + "app": "${APPLICATION_NAME}", + "deploymentconfig": "${APPLICATION_NAME}" + }, + "type": "ClusterIP" + } + } + ] +} From 52fba5967fc309d6a8ede484f15c249ff004e3a8 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Mon, 19 Feb 2018 11:43:20 +1100 Subject: [PATCH 05/10] Add OpenShift template for building, but not deploying an image using Source-to-Image builder. --- examples/source-to-image/templates.json | 591 ++++++++++++++---------- 1 file changed, 350 insertions(+), 241 deletions(-) diff --git a/examples/source-to-image/templates.json b/examples/source-to-image/templates.json index a923450c..af5127e7 100644 --- a/examples/source-to-image/templates.json +++ b/examples/source-to-image/templates.json @@ -1,266 +1,375 @@ { - "kind": "Template", + "kind": "List", "apiVersion": "v1", - "metadata": { - "name": "jupyter-notebook-quickstart", - "annotations": { - "openshift.io/display-name": "Jupyter Notebook Quickstart", - "description": "Template for deploying Jupyter Notebook images with bundled notebooks and files.", - "iconClass": "icon-python", - "tags": "python,jupyter" - } - }, - "parameters": [ - { - "name": "APPLICATION_NAME", - "value": "notebook", - "required": true - }, - { - "name": "BUILDER_IMAGE", - "value": "jupyter/minimal-notebook:latest", - "required": true - }, - { - "name": "BUILDER_SCRIPTS", - "value": "https://raw.githubusercontent.com/jupyter/docker-stacks/source-to-image/examples/source-to-image", - "required": true - }, - { - "name": "GIT_REPOSITORY_URL", - "value": "", - "required": true - }, - { - "name": "GIT_REFERENCE", - "value": "master", - "required": true - }, - { - "name": "CONTEXT_DIR", - "value": "", - "required": false - }, - { - "name": "NOTEBOOK_PASSWORD", - "from": "[a-f0-9]{32}", - "generate": "expression" - } - ], - "objects": [ - { - "apiVersion": "v1", - "kind": "ImageStream", - "metadata": { - "name": "${APPLICATION_NAME}", - "labels": { - "app": "${APPLICATION_NAME}" - } - }, - "spec": { - "lookupPolicy": { - "local": false - } - } - }, - { - "apiVersion": "v1", - "kind": "BuildConfig", - "metadata": { - "name": "${APPLICATION_NAME}", - "labels": { - "app": "${APPLICATION_NAME}" - } - }, - "spec": { - "output": { - "to": { - "kind": "ImageStreamTag", - "name": "${APPLICATION_NAME}:latest" - } - }, - "source": { - "type": "Git", - "git": { - "uri": "${GIT_REPOSITORY_URL}", - "ref": "${GIT_REFERENCE}" - }, - "contextDir": "${CONTEXT_DIR}" - }, - "strategy": { - "type": "Source", - "sourceStrategy": { - "from": { - "kind": "DockerImage", - "name": "${BUILDER_IMAGE}" - }, - "scripts": "${BUILDER_SCRIPTS}" - } - }, - "triggers": [ - { - "type": "ConfigChange" - } - ] - } - }, - { - "kind": "ConfigMap", - "apiVersion": "v1", - "metadata": { - "name": "${APPLICATION_NAME}-cfg", - "labels": { - "app": "${APPLICATION_NAME}" - } - }, - "data": { - "jupyter_notebook_config.py": "import os\n\npassword = os.environ.get('JUPYTER_NOTEBOOK_PASSWORD')\n\nif password:\n import notebook.auth\n c.NotebookApp.password = notebook.auth.passwd(password)\n del password\n del os.environ['JUPYTER_NOTEBOOK_PASSWORD']\n\nimage_config_file = '/home/jovyan/work/.jupyter/jupyter_notebook_config.py'\n\nif os.path.exists(image_config_file):\n with open(image_config_file) as fp:\n exec(compile(fp.read(), image_config_file, 'exec'), globals())\n" - } - }, + "items": [ { - "kind": "DeploymentConfig", + "kind": "Template", "apiVersion": "v1", "metadata": { - "name": "${APPLICATION_NAME}", - "labels": { - "app": "${APPLICATION_NAME}" + "name": "jupyter-notebook-builder", + "annotations": { + "openshift.io/display-name": "Jupyter Notebook Builder", + "description": "Template for building Jupyter Notebook images with bundled notebooks and files.", + "iconClass": "icon-python", + "tags": "python,jupyter" } }, - "spec": { - "strategy": { - "type": "Recreate" + "parameters": [ + { + "name": "IMAGE_NAME", + "value": "notebook", + "required": true }, - "triggers": [ - { - "type": "ConfigChange" - }, - { - "type": "ImageChange", - "imageChangeParams": { - "automatic": true, - "containerNames": [ - "jupyter-notebook" - ], - "from": { - "kind": "ImageStreamTag", - "name": "${APPLICATION_NAME}:latest" - } - } - } - ], - "replicas": 1, - "selector": { - "app": "${APPLICATION_NAME}", - "deploymentconfig": "${APPLICATION_NAME}" + { + "name": "BUILDER_IMAGE", + "value": "jupyter/minimal-notebook:latest", + "required": true }, - "template": { + { + "name": "BUILDER_SCRIPTS", + "value": "https://raw.githubusercontent.com/jupyter/docker-stacks/source-to-image/examples/source-to-image", + "required": true + }, + { + "name": "GIT_REPOSITORY_URL", + "value": "", + "required": true + }, + { + "name": "GIT_REFERENCE", + "value": "master", + "required": true + }, + { + "name": "CONTEXT_DIR", + "value": "", + "required": false + } + ], + "objects": [ + { + "apiVersion": "v1", + "kind": "ImageStream", "metadata": { - "annotations": { - "alpha.image.policy.openshift.io/resolve-names": "*" - }, + "name": "${IMAGE_NAME}", "labels": { - "app": "${APPLICATION_NAME}", - "deploymentconfig": "${APPLICATION_NAME}" + "app": "${IMAGE_NAME}" + } + } + }, + { + "apiVersion": "v1", + "kind": "BuildConfig", + "metadata": { + "name": "${IMAGE_NAME}", + "labels": { + "app": "${IMAGE_NAME}" } }, "spec": { - "containers": [ + "output": { + "to": { + "kind": "ImageStreamTag", + "name": "${IMAGE_NAME}:latest" + } + }, + "resources": { + "limits": { + "memory": "1Gi" + } + }, + "source": { + "type": "Git", + "git": { + "uri": "${GIT_REPOSITORY_URL}", + "ref": "${GIT_REFERENCE}" + }, + "contextDir": "${CONTEXT_DIR}" + }, + "strategy": { + "type": "Source", + "sourceStrategy": { + "from": { + "kind": "DockerImage", + "name": "${BUILDER_IMAGE}" + }, + "scripts": "${BUILDER_SCRIPTS}" + } + }, + "triggers": [ { - "name": "jupyter-notebook", - "image": "${APPLICATION_NAME}:latest", - "command": [ - "start-notebook.sh", - "--config=/home/jovyan/configs/jupyter_notebook_config.py", - "--no-browser", - "--ip=0.0.0.0" - ], - - "ports": [ - { - "containerPort": 8888, - "protocol": "TCP" + "type": "ConfigChange" + } + ] + } + } + ] + }, + { + "kind": "Template", + "apiVersion": "v1", + "metadata": { + "name": "jupyter-notebook-quickstart", + "annotations": { + "openshift.io/display-name": "Jupyter Notebook QuickStart", + "description": "Template for deploying Jupyter Notebook images with bundled notebooks and files.", + "iconClass": "icon-python", + "tags": "python,jupyter" + } + }, + "parameters": [ + { + "name": "APPLICATION_NAME", + "value": "notebook", + "required": true + }, + { + "name": "BUILDER_IMAGE", + "value": "jupyter/minimal-notebook:latest", + "required": true + }, + { + "name": "BUILDER_SCRIPTS", + "value": "https://raw.githubusercontent.com/jupyter/docker-stacks/source-to-image/examples/source-to-image", + "required": true + }, + { + "name": "GIT_REPOSITORY_URL", + "value": "", + "required": true + }, + { + "name": "GIT_REFERENCE", + "value": "master", + "required": true + }, + { + "name": "CONTEXT_DIR", + "value": "", + "required": false + }, + { + "name": "NOTEBOOK_PASSWORD", + "from": "[a-f0-9]{32}", + "generate": "expression" + } + ], + "objects": [ + { + "apiVersion": "v1", + "kind": "ImageStream", + "metadata": { + "name": "${APPLICATION_NAME}", + "labels": { + "app": "${APPLICATION_NAME}" + } + } + }, + { + "apiVersion": "v1", + "kind": "BuildConfig", + "metadata": { + "name": "${APPLICATION_NAME}", + "labels": { + "app": "${APPLICATION_NAME}" + } + }, + "spec": { + "output": { + "to": { + "kind": "ImageStreamTag", + "name": "${APPLICATION_NAME}:latest" + } + }, + "resources": { + "limits": { + "memory": "1Gi" + } + }, + "source": { + "type": "Git", + "git": { + "uri": "${GIT_REPOSITORY_URL}", + "ref": "${GIT_REFERENCE}" + }, + "contextDir": "${CONTEXT_DIR}" + }, + "strategy": { + "type": "Source", + "sourceStrategy": { + "from": { + "kind": "DockerImage", + "name": "${BUILDER_IMAGE}" + }, + "scripts": "${BUILDER_SCRIPTS}" + } + }, + "triggers": [ + { + "type": "ConfigChange" + } + ] + } + }, + { + "kind": "ConfigMap", + "apiVersion": "v1", + "metadata": { + "name": "${APPLICATION_NAME}-cfg", + "labels": { + "app": "${APPLICATION_NAME}" + } + }, + "data": { + "jupyter_notebook_config.py": "import os\n\npassword = os.environ.get('JUPYTER_NOTEBOOK_PASSWORD')\n\nif password:\n import notebook.auth\n c.NotebookApp.password = notebook.auth.passwd(password)\n del password\n del os.environ['JUPYTER_NOTEBOOK_PASSWORD']\n\nimage_config_file = '/home/jovyan/work/.jupyter/jupyter_notebook_config.py'\n\nif os.path.exists(image_config_file):\n with open(image_config_file) as fp:\n exec(compile(fp.read(), image_config_file, 'exec'), globals())\n" + } + }, + { + "kind": "DeploymentConfig", + "apiVersion": "v1", + "metadata": { + "name": "${APPLICATION_NAME}", + "labels": { + "app": "${APPLICATION_NAME}" + } + }, + "spec": { + "strategy": { + "type": "Recreate" + }, + "triggers": [ + { + "type": "ConfigChange" + }, + { + "type": "ImageChange", + "imageChangeParams": { + "automatic": true, + "containerNames": [ + "jupyter-notebook" + ], + "from": { + "kind": "ImageStreamTag", + "name": "${APPLICATION_NAME}:latest" } - ], - "env": [ - { - "name": "JUPYTER_NOTEBOOK_PASSWORD", - "value": "${NOTEBOOK_PASSWORD}" - } - ], - "volumeMounts": [ - { - "mountPath": "/home/jovyan/configs", - "name": "configs" - } - ] + } } ], - "securityContext": { - "supplementalGroups": [ - 100 - ] + "replicas": 1, + "selector": { + "app": "${APPLICATION_NAME}", + "deploymentconfig": "${APPLICATION_NAME}" }, - "volumes": [ - { - "configMap": { - "name": "${APPLICATION_NAME}-cfg" - }, - "name": "configs" - } - ] + "template": { + "metadata": { + "annotations": { + "alpha.image.policy.openshift.io/resolve-names": "*" + }, + "labels": { + "app": "${APPLICATION_NAME}", + "deploymentconfig": "${APPLICATION_NAME}" + } + }, + "spec": { + "containers": [ + { + "name": "jupyter-notebook", + "image": "${APPLICATION_NAME}:latest", + "command": [ + "start-notebook.sh", + "--config=/home/jovyan/configs/jupyter_notebook_config.py", + "--no-browser", + "--ip=0.0.0.0" + ], + + "ports": [ + { + "containerPort": 8888, + "protocol": "TCP" + } + ], + "env": [ + { + "name": "JUPYTER_NOTEBOOK_PASSWORD", + "value": "${NOTEBOOK_PASSWORD}" + } + ], + "volumeMounts": [ + { + "mountPath": "/home/jovyan/configs", + "name": "configs" + } + ] + } + ], + "securityContext": { + "supplementalGroups": [ + 100 + ] + }, + "volumes": [ + { + "configMap": { + "name": "${APPLICATION_NAME}-cfg" + }, + "name": "configs" + } + ] + } + } } - } - } - }, - { - "kind": "Route", - "apiVersion": "v1", - "metadata": { - "name": "${APPLICATION_NAME}", - "labels": { - "app": "${APPLICATION_NAME}" - } - }, - "spec": { - "host": "", - "to": { + }, + { + "kind": "Route", + "apiVersion": "v1", + "metadata": { + "name": "${APPLICATION_NAME}", + "labels": { + "app": "${APPLICATION_NAME}" + } + }, + "spec": { + "host": "", + "to": { + "kind": "Service", + "name": "${APPLICATION_NAME}", + "weight": 100 + }, + "port": { + "targetPort": "8888-tcp" + }, + "tls": { + "termination": "edge", + "insecureEdgeTerminationPolicy": "Redirect" + } + } + }, + { "kind": "Service", - "name": "${APPLICATION_NAME}", - "weight": 100 - }, - "port": { - "targetPort": "8888-tcp" - }, - "tls": { - "termination": "edge", - "insecureEdgeTerminationPolicy": "Redirect" - } - } - }, - { - "kind": "Service", - "apiVersion": "v1", - "metadata": { - "name": "${APPLICATION_NAME}", - "labels": { - "app": "${APPLICATION_NAME}" - } - }, - "spec": { - "ports": [ - { - "name": "8888-tcp", - "protocol": "TCP", - "port": 8888, - "targetPort": 8888 + "apiVersion": "v1", + "metadata": { + "name": "${APPLICATION_NAME}", + "labels": { + "app": "${APPLICATION_NAME}" + } + }, + "spec": { + "ports": [ + { + "name": "8888-tcp", + "protocol": "TCP", + "port": 8888, + "targetPort": 8888 + } + ], + "selector": { + "app": "${APPLICATION_NAME}", + "deploymentconfig": "${APPLICATION_NAME}" + }, + "type": "ClusterIP" } - ], - "selector": { - "app": "${APPLICATION_NAME}", - "deploymentconfig": "${APPLICATION_NAME}" - }, - "type": "ClusterIP" - } + } + ] } ] } From f0cdbd536b33d64549798e8cf3cc12233a9cb1fd Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Mon, 19 Feb 2018 13:17:54 +1100 Subject: [PATCH 06/10] Remove packages file from image when using Source-to-Image before doing a build, rather than after. --- examples/source-to-image/assemble | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/examples/source-to-image/assemble b/examples/source-to-image/assemble index 2c80b679..a8326c37 100755 --- a/examples/source-to-image/assemble +++ b/examples/source-to-image/assemble @@ -4,6 +4,16 @@ set -x set -eo pipefail +# Remove any 'environment.yml' or 'requirements.txt' files which may +# have been carried over from the base image so we don't reinstall +# packages which have already been installed. This could occur where +# an S2I build was used to create a new base image with pre-installed +# Python packages, with the new image then subsequently being used as a +# S2I builder base image. + +rm -f /home/$NB_USER/work/environment.yml +rm -f /home/$NB_USER/work/requirements.txt + # Copy injected files to correct place in 'work' directory. cp -Rf /tmp/src/. /home/$NB_USER/work @@ -25,13 +35,6 @@ else fi fi -# Remove any 'environment.yml' or 'requirements.txt' file when done in -# case we are producing an image which will in turn be used as an S2I -# builder image. - -rm -f /home/$NB_USER/work/environment.yml -rm -f /home/$NB_USER/work/requirements.txt - # Fix up permissions on home directory and Python installation so that # everything is still writable by 'users' group. From 9b78b0c13c29b5c2eb15d2fd3b83d7356018b142 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Mon, 19 Feb 2018 13:49:29 +1100 Subject: [PATCH 07/10] Add README for Source-to-Image examples directory. --- examples/source-to-image/README.md | 159 +++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 examples/source-to-image/README.md diff --git a/examples/source-to-image/README.md b/examples/source-to-image/README.md new file mode 100644 index 00000000..60b9a48a --- /dev/null +++ b/examples/source-to-image/README.md @@ -0,0 +1,159 @@ +This example provides scripts for building custom Jupyter Notebook images containing notebooks, data files, and with Python packages required by the notebooks already installed. The scripts provided work with the Source-to-Image tool and you can create the images from the command line on your own computer. It provides similar capabilities to ``mybinder.org`` but without the complexity of having to setup any special infrastructure. Templates are also provided to enable running builds in OpenShift, as well as deploying the resulting image to make it available. + +Source-to-Image Project +----------------------- + +Source-to-Image (S2I) is an open source project which provides a tool for creating container images. It works by taking a base image, injecting additional source code or files into a running container created from the base image, and running a builder script in the container to process the source code or files to prepare the new image. + +Details on the S2I tool, and executable binaries for Linux, macOS and Windows, can be found on GitHub at: + +* https://github.com/openshift/source-to-image + +The tool is standalone, and can be used on any system which provides a docker daemon for running containers. To provide an end-to-end capability to build and deploy applications in containers, support for S2I is also integrated into container platforms such as OpenShift. + +Getting Started with S2I +------------------------ + +As an example of how S2I can be used to create a custom image with a bundled set of notebooks, run: + +``` +s2i build \ + --scripts-url https://raw.githubusercontent.com/jupyter/docker-stacks/source-to-image/examples/source-to-image \ + --context-dir docs/source/examples/Notebook \ + https://github.com/jupyter/notebook \ + jupyter/minimal-notebook:latest \ + notebook-examples +``` + +This example command will pull down the Git repository ``https://github.com/jupyter/notebook`` and build the image ``notebook-examples`` using the files contained in the ``docs/source/examples/Notebook`` directory of that Git repository. The base image which the files will be combined with is ``jupyter/minimal-notebook:latest``, but you can specify any of the Jupyter Project ``docker-stacks`` images as the base image. + +The resulting image from running the command can be seen by running ``docker images``. + +``` +REPOSITORY TAG IMAGE ID CREATED SIZE +notebook-examples latest f5899ed1241d 2 minutes ago 2.59GB +``` + +You can now run the image. + +``` +$ docker run --rm -p 8888:8888 notebook-examples +Executing the command: jupyter notebook +[I 01:14:50.532 NotebookApp] Writing notebook server cookie secret to /home/jovyan/.local/share/jupyter/runtime/notebook_cookie_secret +[W 01:14:50.724 NotebookApp] WARNING: The notebook server is listening on all IP addresses and not using encryption. This is not recommended. +[I 01:14:50.747 NotebookApp] JupyterLab beta preview extension loaded from /opt/conda/lib/python3.6/site-packages/jupyterlab +[I 01:14:50.747 NotebookApp] JupyterLab application directory is /opt/conda/share/jupyter/lab +[I 01:14:50.754 NotebookApp] Serving notebooks from local directory: /home/jovyan +[I 01:14:50.754 NotebookApp] 0 active kernels +[I 01:14:50.754 NotebookApp] The Jupyter Notebook is running at: +[I 01:14:50.754 NotebookApp] http://[all ip addresses on your system]:8888/?token=04646d5c5e928da75842cd318d4a3c5aa1f942fc5964323a +[I 01:14:50.754 NotebookApp] Use Control-C to stop this server and shut down all kernels (twice to skip confirmation). +[C 01:14:50.755 NotebookApp] + + Copy/paste this URL into your browser when you connect for the first time, + to login with a token: + http://localhost:8888/?token=04646d5c5e928da75842cd318d4a3c5aa1f942fc5964323a +``` + +Open your browser on the URL displayed, browse to the ``work`` folder through the Jupyter Notebook web interface and you will find the notebooks from the Git repository and can work with them. + +The S2I Builder Scripts +----------------------- + +Normally when using S2I, the base image would be S2I enabled and contain the builder scripts needed to prepare the image and define how the application in the image should be run. As the Jupyter Project ``docker-stacks`` images are not S2I enabled (although they could be), in the above example the ``--scripts-url`` option has been used to specify that the example builder scripts contained in this directory of this Git repository should be used. + +Using the ``--scripts-url`` option, the builder scripts can be hosted on any HTTP server, or you could also use builder scripts local to your computer file using an appropriate ``file://`` format URI argument to ``--scripts-url``. + +The builder scripts in this directory of this repository are ``assemble`` and ``run`` and are provided as examples of what can be done. You can use the scripts as is, or create your own. + +The supplied ``assemble`` script performs a few key steps. These are: + +``` +cp -Rf /tmp/src/. /home/$NB_USER/work + +rm -rf /tmp/src +``` + +The first steps copy files into the location they need to be when the image is run, from the directory where they are initially placed by the ``s2i`` command. + +The next steps are: + +``` +if [ -f /home/$NB_USER/work/environment.yml ]; then + (cd /home/$NB_USER/work && conda env update --name root --file environment.yml) + conda clean -tipsy +else + if [ -f /home/$NB_USER/work/requirements.txt ]; then + (cd /home/$NB_USER/work && pip --no-cache-dir install -r requirements.txt) + fi +fi +``` + +This determines whether a ``environment.yml`` or ``requirements.txt`` file exists with the files and if so, runs the appropriate package management tool to install any Python packages listed in those files. + +This means that so long as a set of notebook files provides one of these files listing what Python packages they need, those packages will be automatically installed into the image so they are available when the image is run. + +A final step is: + +``` +fix-permissions $CONDA_DIR +fix-permissions /home/$NB_USER +``` + +This fixes up permissions on any new files created by the build. This is necessary to ensure that when the image is run, you can still install additional files. This is important for when an image is run in ``sudo`` mode, or it is hosted in a more secure container platform such as Kubernetes/OpenShift where it will be run as a set user ID that isn't known in advance. + +As long as you preserve the first and last set of steps, you can do whatever you want in the ``assemble`` script to install packages, create files etc. Do be aware though that S2I builds do not run as ``root`` and so you cannot install additional system packages. If you need to install additional system packages, use a ``Dockerfile`` and normal ``docker build`` to first create a new custom base image from the Jupyter Project ``docker-stacks`` images, with the extra system packages, and then use that image with the S2I build to combine your notebooks and have Python packages installed. + +The ``run`` script in this directory is very simple and just runs the notebook application. + +``` +exec start-notebook.sh "$@" +``` + +Integration with OpenShift +-------------------------- + +The OpenShift platform provides integrated support for S2I type builds. Templates are provided for using the S2I build mechanism with the scripts in this directory. To load the templates run: + +``` +oc create -f https://raw.githubusercontent.com/jupyter/docker-stacks/source-to-image/examples/source-to-image/templates.json +``` + +This will create the templates: + +``` +jupyter-notebook-builder +jupyter-notebook-quickstart +``` + +The templates can be used from the OpenShift web console or command line. This ``README`` is only going to explain deploying from the command line. + +To use the OpenShift command line to build into an image, and deploy, the set of notebooks used above, run: + +``` +oc new-app --template jupyter-notebook-quickstart \ + --param APPLICATION_NAME=notebook-examples \ + --param GIT_REPOSITORY_URL=https://github.com/jupyter/notebook \ + --param CONTEXT_DIR=docs/source/examples/Notebook \ + --param BUILDER_IMAGE=jupyter/minimal-notebook:latest \ + --param NOTEBOOK_PASSWORD=mypassword +``` + +You can provide a password using the ``NOTEBOOK_PASSWORD`` parameter. If you don't set that parameter, a password will be generated, with it being displayed by the ``oc new-app`` command. + +Once the image has been built, it will be deployed. To see the hostname for accessing the notebook, run ``oc get routes``. + +``` +NAME HOST/PORT PATH SERVICES PORT TERMINATION WILDCARD +notebook-examples notebook-examples-jupyter.abcd.pro-us-east-1.openshiftapps.com notebook-examples 8888-tcp edge/Redirect None +``` + +As the deployment will use a secure connection, the URL for accessing the notebook in this case would be: + +``` +https://notebook-examples-jupyter.abcd.pro-us-east-1.openshiftapps.com +``` + +If you only want to build an image but not deploy it, you can use the ``jupyter-notebook-builder`` template. You can then deploy it using the ``jupyter-notebook`` template provided with the [openshift](../openshift) examples directory. + +See the ``openshift`` examples directory for further information on customizing configuration for a Jupyter Notebook deployment and deleting a deployment. From 9e9ca7344c342a5ef8382164780d5b05af774644 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Mon, 19 Feb 2018 14:14:18 +1100 Subject: [PATCH 08/10] Fix URLs for location of Source-to-Image example directory in master. --- examples/source-to-image/README.md | 4 ++-- examples/source-to-image/templates.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/source-to-image/README.md b/examples/source-to-image/README.md index 60b9a48a..429e7462 100644 --- a/examples/source-to-image/README.md +++ b/examples/source-to-image/README.md @@ -18,7 +18,7 @@ As an example of how S2I can be used to create a custom image with a bundled set ``` s2i build \ - --scripts-url https://raw.githubusercontent.com/jupyter/docker-stacks/source-to-image/examples/source-to-image \ + --scripts-url https://raw.githubusercontent.com/jupyter/docker-stacks/master/examples/source-to-image \ --context-dir docs/source/examples/Notebook \ https://github.com/jupyter/notebook \ jupyter/minimal-notebook:latest \ @@ -116,7 +116,7 @@ Integration with OpenShift The OpenShift platform provides integrated support for S2I type builds. Templates are provided for using the S2I build mechanism with the scripts in this directory. To load the templates run: ``` -oc create -f https://raw.githubusercontent.com/jupyter/docker-stacks/source-to-image/examples/source-to-image/templates.json +oc create -f https://raw.githubusercontent.com/jupyter/docker-stacks/master/examples/source-to-image/templates.json ``` This will create the templates: diff --git a/examples/source-to-image/templates.json b/examples/source-to-image/templates.json index af5127e7..c9b4c960 100644 --- a/examples/source-to-image/templates.json +++ b/examples/source-to-image/templates.json @@ -27,7 +27,7 @@ }, { "name": "BUILDER_SCRIPTS", - "value": "https://raw.githubusercontent.com/jupyter/docker-stacks/source-to-image/examples/source-to-image", + "value": "https://raw.githubusercontent.com/jupyter/docker-stacks/master/examples/source-to-image", "required": true }, { From d2cfd3de0d6ce87c781dab1962e9987617ea1353 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Tue, 20 Feb 2018 10:52:47 +1100 Subject: [PATCH 09/10] Avoid relying on existence of work subdirectory in Source-to-Image examples. --- examples/source-to-image/README.md | 26 +++++++++++++++---------- examples/source-to-image/assemble | 20 +++++++++---------- examples/source-to-image/templates.json | 6 +++--- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/examples/source-to-image/README.md b/examples/source-to-image/README.md index 429e7462..41ea7214 100644 --- a/examples/source-to-image/README.md +++ b/examples/source-to-image/README.md @@ -1,4 +1,10 @@ -This example provides scripts for building custom Jupyter Notebook images containing notebooks, data files, and with Python packages required by the notebooks already installed. The scripts provided work with the Source-to-Image tool and you can create the images from the command line on your own computer. It provides similar capabilities to ``mybinder.org`` but without the complexity of having to setup any special infrastructure. Templates are also provided to enable running builds in OpenShift, as well as deploying the resulting image to make it available. +This example provides scripts for building custom Jupyter Notebook images containing notebooks, data files, and with Python packages required by the notebooks already installed. The scripts provided work with the Source-to-Image tool and you can create the images from the command line on your own computer. Templates are also provided to enable running builds in OpenShift, as well as deploying the resulting image to OpenShift to make it available. + +The build scripts, when used with the Source-to-Image tool, provide similar capabilities to ``repo2docker``. When builds are run under OpenShift with the supplied templates, it provides similar capabilities to ``mybinder.org``, but where notebook instances are deployed in your existing OpenShift project and JupyterHub is not required. + +For separate examples of using JupyterHub with OpenShift, see the project: + +* https://github.com/jupyter-on-openshift/jupyterhub-quickstart Source-to-Image Project ----------------------- @@ -55,7 +61,7 @@ Executing the command: jupyter notebook http://localhost:8888/?token=04646d5c5e928da75842cd318d4a3c5aa1f942fc5964323a ``` -Open your browser on the URL displayed, browse to the ``work`` folder through the Jupyter Notebook web interface and you will find the notebooks from the Git repository and can work with them. +Open your browser on the URL displayed, and you will find the notebooks from the Git repository and can work with them. The S2I Builder Scripts ----------------------- @@ -66,25 +72,25 @@ Using the ``--scripts-url`` option, the builder scripts can be hosted on any HTT The builder scripts in this directory of this repository are ``assemble`` and ``run`` and are provided as examples of what can be done. You can use the scripts as is, or create your own. -The supplied ``assemble`` script performs a few key steps. These are: +The supplied ``assemble`` script performs a few key steps. + +The first steps copy files into the location they need to be when the image is run, from the directory where they are initially placed by the ``s2i`` command. ``` -cp -Rf /tmp/src/. /home/$NB_USER/work +cp -Rf /tmp/src/. /home/$NB_USER rm -rf /tmp/src ``` -The first steps copy files into the location they need to be when the image is run, from the directory where they are initially placed by the ``s2i`` command. - The next steps are: ``` -if [ -f /home/$NB_USER/work/environment.yml ]; then - (cd /home/$NB_USER/work && conda env update --name root --file environment.yml) +if [ -f /home/$NB_USER/environment.yml ]; then + conda env update --name root --file /home/$NB_USER/environment.yml conda clean -tipsy else - if [ -f /home/$NB_USER/work/requirements.txt ]; then - (cd /home/$NB_USER/work && pip --no-cache-dir install -r requirements.txt) + if [ -f /home/$NB_USER/requirements.txt ]; then + pip --no-cache-dir install -r /home/$NB_USER/requirements.txt fi fi ``` diff --git a/examples/source-to-image/assemble b/examples/source-to-image/assemble index a8326c37..e01949ed 100755 --- a/examples/source-to-image/assemble +++ b/examples/source-to-image/assemble @@ -11,27 +11,25 @@ set -eo pipefail # Python packages, with the new image then subsequently being used as a # S2I builder base image. -rm -f /home/$NB_USER/work/environment.yml -rm -f /home/$NB_USER/work/requirements.txt +rm -f /home/$NB_USER/environment.yml +rm -f /home/$NB_USER/requirements.txt -# Copy injected files to correct place in 'work' directory. +# Copy injected files to target directory. -cp -Rf /tmp/src/. /home/$NB_USER/work +cp -Rf /tmp/src/. /home/$NB_USER rm -rf /tmp/src # Install any Python modules. If we find an 'environment.yml' file we # assume we should use 'conda' to install packages. If 'requirements.txt' -# use 'pip' instead. Ensure we are in the 'work' directory so relative -# directory paths in these files resolve okay if installing packages -# from subdirectories. +# use 'pip' instead. -if [ -f /home/$NB_USER/work/environment.yml ]; then - (cd /home/$NB_USER/work && conda env update --name root --file environment.yml) +if [ -f /home/$NB_USER/environment.yml ]; then + conda env update --name root --file /home/$NB_USER/environment.yml conda clean -tipsy else - if [ -f /home/$NB_USER/work/requirements.txt ]; then - (cd /home/$NB_USER/work && pip --no-cache-dir install -r requirements.txt) + if [ -f /home/$NB_USER/requirements.txt ]; then + pip --no-cache-dir install -r /home/$NB_USER/requirements.txt fi fi diff --git a/examples/source-to-image/templates.json b/examples/source-to-image/templates.json index c9b4c960..3213ba33 100644 --- a/examples/source-to-image/templates.json +++ b/examples/source-to-image/templates.json @@ -221,7 +221,7 @@ } }, "data": { - "jupyter_notebook_config.py": "import os\n\npassword = os.environ.get('JUPYTER_NOTEBOOK_PASSWORD')\n\nif password:\n import notebook.auth\n c.NotebookApp.password = notebook.auth.passwd(password)\n del password\n del os.environ['JUPYTER_NOTEBOOK_PASSWORD']\n\nimage_config_file = '/home/jovyan/work/.jupyter/jupyter_notebook_config.py'\n\nif os.path.exists(image_config_file):\n with open(image_config_file) as fp:\n exec(compile(fp.read(), image_config_file, 'exec'), globals())\n" + "jupyter_notebook_config.py": "import os\n\npassword = os.environ.get('JUPYTER_NOTEBOOK_PASSWORD')\n\nif password:\n import notebook.auth\n c.NotebookApp.password = notebook.auth.passwd(password)\n del password\n del os.environ['JUPYTER_NOTEBOOK_PASSWORD']\n\nimage_config_file = '/home/jovyan/.jupyter/jupyter_notebook_config.py'\n\nif os.path.exists(image_config_file):\n with open(image_config_file) as fp:\n exec(compile(fp.read(), image_config_file, 'exec'), globals())\n" } }, { @@ -277,7 +277,7 @@ "image": "${APPLICATION_NAME}:latest", "command": [ "start-notebook.sh", - "--config=/home/jovyan/configs/jupyter_notebook_config.py", + "--config=/etc/jupyter/openshift/jupyter_notebook_config.py", "--no-browser", "--ip=0.0.0.0" ], @@ -296,7 +296,7 @@ ], "volumeMounts": [ { - "mountPath": "/home/jovyan/configs", + "mountPath": "/etc/jupyter/openshift", "name": "configs" } ] From 9850b5710ede19ba8699a07a41834ea11ae65229 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Tue, 20 Feb 2018 12:21:19 +1100 Subject: [PATCH 10/10] Correct URL in Source-to-Image template to reference master branch. --- examples/source-to-image/templates.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/source-to-image/templates.json b/examples/source-to-image/templates.json index 3213ba33..b8ac7ee9 100644 --- a/examples/source-to-image/templates.json +++ b/examples/source-to-image/templates.json @@ -130,7 +130,7 @@ }, { "name": "BUILDER_SCRIPTS", - "value": "https://raw.githubusercontent.com/jupyter/docker-stacks/source-to-image/examples/source-to-image", + "value": "https://raw.githubusercontent.com/jupyter/docker-stacks/master/examples/source-to-image", "required": true }, {