hazza/docker-stacks
1
0
Fork 0
mirror of https://github.com/jupyter/docker-stacks.git synced 2025-10-18 07:22:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into asalikhov/fix_markdown

Browse Source
This commit is contained in:
Ayaz Salikhov
2021-05-04 01:08:53 +03:00
parent 451544c956 ccc1470b02
commit 16ec0852c5
25 changed files with 815 additions and 631 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/docker.yml vendored
View File

@@ -20,6 +20,8 @@ jobs:
build:
name: Build Docker Images
runs-on: ubuntu-latest
permissions:
contents: write
if: >
!contains(github.event.head_commit.message, 'ci skip') &&
!contains(github.event.pull_request.title, 'ci skip')
@@ -52,9 +54,6 @@ jobs:
BUILDKIT_PROGRESS: plain
- name: Run Post-Build Hooks
run: make -C main hook-all
env:
COMMIT_MSG: "${{github.event.head_commit.message}}"
WIKI_PATH: ../wiki
- name: Login to Docker Hub
if: github.ref == 'refs/heads/master'
run: >

2
.github/workflows/sphinx.yml vendored
View File

@@ -16,6 +16,8 @@ jobs:
build:
name: Build Sphinx Documentation
runs-on: ubuntu-latest
permissions:
contents: write
if: >
!contains(github.event.head_commit.message , 'ci skip') &&
!contains(github.event.pull_request.title, 'ci skip')

11
Makefile
View File

@@ -98,15 +98,10 @@ git-commit: ## commit outstading git changes and push to remote
git commit -m "[ci skip] Automated publish for $(GITHUB_SHA)" || exit 0
@cd $(LOCAL_PATH) && git push -u publisher master
hook/%: export COMMIT_MSG?=$(shell git log -1 --pretty=%B)
hook/%: export GITHUB_SHA?=$(shell git rev-parse HEAD)
hook/%: export WIKI_PATH?=../wiki
hook/%: WIKI_PATH?=../wiki
hook/%: ## run post-build hooks for an image
BUILD_TIMESTAMP="$$(date -u +%FT%TZ)" \
DOCKER_REPO="$(OWNER)/$(notdir $@)" \
IMAGE_NAME="$(OWNER)/$(notdir $@):latest" \
IMAGE_SHORT_NAME="$(notdir $@)" \
$(SHELL) $(notdir $@)/hooks/run_hook
python3 -m tagging.tag_image --short-image-name "$(notdir $@)" --owner "$(OWNER)" && \
python3 -m tagging.create_manifests --short-image-name "$(notdir $@)" --owner "$(OWNER)" --wiki-path "$(WIKI_PATH)"
hook-all: $(foreach I,$(ALL_IMAGES),hook/$(I) ) ## run post-build hooks for all images

61
all-spark-notebook/hooks/run_hook
View File

@@ -1,61 +0,0 @@
#!/bin/bash
set -e
# Apply tags
GIT_SHA_TAG=${GITHUB_SHA:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
# Update index
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Apache Spark
\`\`\`
$(docker run --rm ${IMAGE_NAME} bash -c '$SPARK_HOME/bin/spark-submit --version' 2>&1)
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## R Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} R --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} R --silent -e 'installed.packages(.Library)[, c(1,3)]')
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF

53
base-notebook/hooks/run_hook
View File

@@ -1,53 +0,0 @@
#!/bin/bash
set -e
# Apply tags
GIT_SHA_TAG=${GITHUB_SHA:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
PY_VERSION_TAG="python-$(docker run --rm ${IMAGE_NAME} python --version 2>&1 | awk '{print $2}')"
docker tag $IMAGE_NAME "$DOCKER_REPO:$PY_VERSION_TAG"
NB_VERSION_TAG="notebook-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyter-notebook --version | tr -d '\r')"
docker tag $IMAGE_NAME "$DOCKER_REPO:${NB_VERSION_TAG%% }"
LAB_VERSION_TAG="lab-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyter-lab --version | tr -d '\r')"
docker tag $IMAGE_NAME "$DOCKER_REPO:${LAB_VERSION_TAG%%\r}"
HUB_VERSION_TAG="hub-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyterhub --version | tr -d '\r')"
docker tag $IMAGE_NAME "$DOCKER_REPO:${HUB_VERSION_TAG%%\r}"
# Update index
INDEX_FILE="${WIKI_PATH}/Home.md"
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${PY_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${NB_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${LAB_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${HUB_VERSION_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "$INDEX_FILE"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF

13
datascience-notebook/Dockerfile
View File

@@ -5,10 +5,6 @@ FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
# Set when building on Travis so that certain long-running build steps can
# be skipped to shorten build time.
ARG TEST_ONLY_BUILD
# Fix DL4006
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -79,16 +75,13 @@ RUN conda install --quiet --yes \
fix-permissions "${CONDA_DIR}" && \
fix-permissions "/home/${NB_USER}"
# Add Julia packages. Only add HDF5 if this is not a test-only build since
# it takes roughly half the entire build time of all of the images on Travis
# to add this one package and often causes Travis to timeout.
#
# Add Julia packages.
# Install IJulia as jovyan and then move the kernelspec out
# to the system share location. Avoids problems with runtime UID change not
# taking effect properly on the .local folder in the jovyan home dir.
RUN julia -e 'import Pkg; Pkg.update()' && \
(test $TEST_ONLY_BUILD || julia -e 'import Pkg; Pkg.add("HDF5")') && \
julia -e "using Pkg; pkg\"add IJulia\"; pkg\"precompile\"" && \
julia -e 'import Pkg; Pkg.add("HDF5")' && \
julia -e 'using Pkg; pkg"add IJulia"; pkg"precompile"' && \
# move kernelspec out of home \
mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" && \
chmod -R go+rx "${CONDA_DIR}/share/jupyter" && \

78
datascience-notebook/hooks/run_hook
View File

@@ -1,78 +0,0 @@
#!/bin/bash
set -e
# Apply tags
GIT_SHA_TAG=${GITHUB_SHA:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
PY_VERSION_TAG="python-$(docker run --rm ${IMAGE_NAME} python --version 2>&1 | awk '{print $2}')"
docker tag $IMAGE_NAME "$DOCKER_REPO:$PY_VERSION_TAG"
R_VERSION_TAG="r-$(docker run --rm -a STDOUT ${IMAGE_NAME} R --version | sed -n 1p | awk '{print $3}')"
docker tag $IMAGE_NAME "$DOCKER_REPO:${R_VERSION_TAG%%\r}"
JULIA_VERSION_TAG="julia-$(docker run --rm -a STDOUT ${IMAGE_NAME} julia --version | awk '{print $3}')"
docker tag $IMAGE_NAME "$DOCKER_REPO:${JULIA_VERSION_TAG%%\r}"
NB_VERSION_TAG="notebook-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyter-notebook --version | tr -d '\r')"
docker tag $IMAGE_NAME "$DOCKER_REPO:${NB_VERSION_TAG%% }"
LAB_VERSION_TAG="lab-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyter-lab --version | tr -d '\r')"
docker tag $IMAGE_NAME "$DOCKER_REPO:${LAB_VERSION_TAG%%\r}"
HUB_VERSION_TAG="hub-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyterhub --version | tr -d '\r')"
docker tag $IMAGE_NAME "$DOCKER_REPO:${HUB_VERSION_TAG%%\r}"
# Update index
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${PY_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${JULIA_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${R_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${NB_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${LAB_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${HUB_VERSION_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Julia Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} julia -E 'using InteractiveUtils; versioninfo()')
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} julia -E 'import Pkg; Pkg.status()')
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## R Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} R --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} R --silent -e 'installed.packages(.Library)[, c(1,3)]')
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF

2
docs/contributing/lint.md
View File

@@ -1,7 +1,7 @@
# Lint
In order to enforce some rules **linters** are used in this project.
Linters can be run either during the **development phase** (by the developer) and during **integration phase** (by Travis).
Linters can be run either during the **development phase** (by the developer) and during **integration phase** (by GitHub Actions).
To integrate and enforce this process in the project lifecycle we are using **git hooks** through [pre-commit][pre-commit].
## Pre-commit hook

356
docs/locale/en/LC_MESSAGES/contributing.po
View File

File diff suppressed because it is too large Load Diff

45
minimal-notebook/hooks/run_hook
View File

@@ -1,45 +0,0 @@
#!/bin/bash
set -e
# Apply tags
GIT_SHA_TAG=${GITHUB_SHA:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
# Update index
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF

51
pyspark-notebook/hooks/run_hook
View File

@@ -1,51 +0,0 @@
#!/bin/bash
set -e
# Apply tags
GIT_SHA_TAG=${GITHUB_SHA:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
# Update index
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Apache Spark
\`\`\`
$(docker run --rm ${IMAGE_NAME} bash -c '$SPARK_HOME/bin/spark-submit --version' 2>&1)
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF

55
r-notebook/hooks/run_hook
View File

@@ -1,55 +0,0 @@
#!/bin/bash
set -e
# Apply tags
GIT_SHA_TAG=${GITHUB_SHA:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
# Update index
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## R Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} R --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} R --silent -e 'installed.packages(.Library)[, c(1,3)]')
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF

1
requirements-dev.txt
View File

@@ -1,4 +1,5 @@
docker
plumbum
pre-commit
pytest
recommonmark

45
scipy-notebook/hooks/run_hook
View File

@@ -1,45 +0,0 @@
#!/bin/bash
set -e
# Apply tags
GIT_SHA_TAG=${GITHUB_SHA:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
# Update index
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF

116
tagging/README.md Normal file
View File

@@ -0,0 +1,116 @@
# Docker stacks tagging and manifest creation
The main purpose of the source code in this folder is to properly tag all the images and to update [build manifests](https://github.com/jupyter/docker-stacks/wiki).
These two processes are closely related, so the source code is widely reused.
Basic example of a tag is a `python` version tag.
For example, an image `jupyter/base-notebook` with `python 3.8.8` will have a tag `jupyter/base-notebook:python-3.8.8`.
This tag (and all the other tags) are pushed to Docker Hub.
Manifest is a description of some important part of the image in a `markdown`.
For example, we dump all the `conda` packages including their versions.
## Main principles
- All the images are located in a hierarchical tree. More info on [image relationships](../docs/using/selecting.md#image-relationships).
- We have `tagger` and `manifest` classes, which can be run inside docker containers to obtain tags and build manifest pieces.
- These classes are inherited from the parent image to all the children images.
- Because manifests and tags might change from parent to children, `taggers` and `manifests` are reevaluated on each image. So, the values are not inherited.
- To tag an image and create a manifest, run `make hook/base-notebook` (or another image of your choice).
## Source code description
In this section we will briefly describe source code in this folder and give examples on how to use it.
### DockerRunner
`DockerRunner` is a helper class to easily run a docker container and execute commands inside this container:
```python
from .docker_runner import DockerRunner
with DockerRunner("ubuntu:bionic") as container:
DockerRunner.run_simple_command(container, cmd="env", print_result=True)
```
### GitHelper
`GitHelper` methods are run in the current `git` repo and give the information about last commit hash and commit message:
```python
from .git_helper import GitHelper
print("Git hash:", GitHelper.commit_hash())
print("Git message:", GitHelper.commit_message())
```
Prefix of commit hash (namely, 12 letters) is used as an image tag to make it easy to inherit from a fixed version of a docker image.
### Tagger
`Tagger` is a class, which can be run inside docker container to calculate some tag for an image.
All the taggers are inherited from `TaggerInterface`:
```python
class TaggerInterface:
"""Common interface for all taggers"""
@staticmethod
def tag_value(container) -> str:
raise NotImplementedError
```
So, `tag_value(container)` method gets a docker container as an input and returns some tag.
`SHATagger` example:
```python
class SHATagger(TaggerInterface):
@staticmethod
def tag_value(container):
return GitHelper.commit_hash_tag()
```
- `taggers.py` contains all the taggers.
- `tag_image.py` is a python executable which is used to tag the image.
### Manifest
`ManifestHeader` is a build manifest header.
It contains information about `Build datetime`, `Docker image size` and `Git commit` info.
All the other manifest classes are inherited from `ManifestInterface`:
```python
class ManifestInterface:
"""Common interface for all manifests"""
@staticmethod
def markdown_piece(container) -> str:
raise NotImplementedError
```
- `markdown_piece(container)` method returns piece of markdown file to be used as a part of build manifest.
`AptPackagesManifest` example:
```python
class AptPackagesManifest(ManifestInterface):
@staticmethod
def markdown_piece(container) -> str:
return "\n".join([
"## Apt Packages",
"",
quoted_output(container, "apt list --installed")
])
```
- `quoted_output` simply runs the command inside container using `DockerRunner.run_simple_command` and wraps it to triple quotes to create a valid markdown piece of file.
- `manifests.py` contains all the manifests.
- `create_manifests.py` is a python executable which is used to create the build manifest for an image.
### Images Hierarchy
All images dependencies on each other and what taggers and manifest they make use of is defined in `images_hierarchy.py`.
`get_taggers_and_manifests.py` defines a helper function to get the taggers and manifests for a specific image.

0
tagging/__init__.py Normal file
View File

91
tagging/create_manifests.py Executable file
View File

@@ -0,0 +1,91 @@
#!/usr/bin/env python3
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import argparse
import datetime
import logging
import os
from typing import List
from .docker_runner import DockerRunner
from .get_taggers_and_manifests import get_taggers_and_manifests
from .git_helper import GitHelper
from .manifests import ManifestHeader, ManifestInterface
logger = logging.getLogger(__name__)
BUILD_TIMESTAMP = datetime.datetime.utcnow().isoformat()[:-7] + "Z"
MARKDOWN_LINE_BREAK = "<br />"
def append_build_history_line(short_image_name: str, owner: str, wiki_path: str, all_tags: List[str]) -> None:
logger.info("Appending build history line")
date_column = f"`{BUILD_TIMESTAMP}`"
image_column = MARKDOWN_LINE_BREAK.join(
f"`{owner}/{short_image_name}:{tag_value}`" for tag_value in all_tags
)
commit_hash = GitHelper.commit_hash()
commit_hash_tag = GitHelper.commit_hash_tag()
links_column = MARKDOWN_LINE_BREAK.join([
f"[Git diff](https://github.com/jupyter/docker-stacks/commit/{commit_hash})",
f"[Dockerfile](https://github.com/jupyter/docker-stacks/blob/{commit_hash}/{short_image_name}/Dockerfile)"
f"[Build manifest](./{short_image_name}-{commit_hash_tag})"
])
build_history_line = "|".join([date_column, image_column, links_column]) + "|"
home_wiki_file = os.path.join(wiki_path, "Home.md")
with open(home_wiki_file, "r") as f:
file = f.read()
TABLE_BEGINNING = "|-|-|-|\n"
file = file.replace(TABLE_BEGINNING, TABLE_BEGINNING + build_history_line + "\n")
with open(home_wiki_file, "w") as f:
f.write(file)
def create_manifest_file(
short_image_name: str,
owner: str,
wiki_path: str,
manifests: List[ManifestInterface],
container
) -> None:
manifest_names = [manifest.__name__ for manifest in manifests]
logger.info(f"Using manifests: {manifest_names}")
commit_hash_tag = GitHelper.commit_hash_tag()
manifest_file = os.path.join(wiki_path, f"manifests/{short_image_name}-{commit_hash_tag}.md")
markdown_pieces = [ManifestHeader.create_header(short_image_name, owner, BUILD_TIMESTAMP)] + \
[manifest.markdown_piece(container) for manifest in manifests]
markdown_content = "\n\n".join(markdown_pieces) + "\n"
with open(manifest_file, "w") as f:
f.write(markdown_content)
def create_manifests(short_image_name: str, owner: str, wiki_path: str) -> None:
logger.info(f"Creating manifests for image: {short_image_name}")
taggers, manifests = get_taggers_and_manifests(short_image_name)
image = f"{owner}/{short_image_name}:latest"
with DockerRunner(image) as container:
all_tags = [tagger.tag_value(container) for tagger in taggers]
append_build_history_line(short_image_name, owner, wiki_path, all_tags)
create_manifest_file(short_image_name, owner, wiki_path, manifests, container)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("--short-image-name", required=True, help="Short image name to apply tags for")
arg_parser.add_argument("--owner", required=True, help="Owner of the image")
arg_parser.add_argument("--wiki-path", required=True, help="Path to the wiki pages")
args = arg_parser.parse_args()
logger.info(f"Current build timestamp: {BUILD_TIMESTAMP}")
create_manifests(args.short_image_name, args.owner, args.wiki_path)

39
tagging/docker_runner.py Normal file
View File

@@ -0,0 +1,39 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import docker
import logging
logger = logging.getLogger(__name__)
class DockerRunner:
def __init__(self, image_name: str, docker_client=docker.from_env(), command: str = "sleep infinity"):
self.container = None
self.image_name = image_name
self.command = command
self.docker_client = docker_client
def __enter__(self):
logger.info(f"Creating container for image {self.image_name} ...")
self.container = self.docker_client.containers.run(
image=self.image_name, command=self.command, detach=True,
)
logger.info(f"Container {self.container.name} created")
return self.container
def __exit__(self, exc_type, exc_value, traceback):
logger.info(f"Removing container {self.container.name} ...")
if self.container:
self.container.remove(force=True)
logger.info(f"Container {self.container.name} removed")
@staticmethod
def run_simple_command(container, cmd: str, print_result: bool = True):
logger.info(f"Running cmd: '{cmd}' on container: {container}")
out = container.exec_run(cmd)
assert out.exit_code == 0, f"Command: {cmd} failed"
result = out.output.decode("utf-8").rstrip()
if print_result:
logger.info(f"Command result: {result}")
return result

19
tagging/get_taggers_and_manifests.py Normal file
View File

@@ -0,0 +1,19 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
from typing import List, Tuple
from .images_hierarchy import ALL_IMAGES
from .manifests import ManifestInterface
from .taggers import TaggerInterface
def get_taggers_and_manifests(short_image_name: str) -> Tuple[List[TaggerInterface], List[ManifestInterface]]:
taggers: List[TaggerInterface] = []
manifests: List[ManifestInterface] = []
while short_image_name is not None:
image_description = ALL_IMAGES[short_image_name]
taggers = image_description.taggers + taggers
manifests = image_description.manifests + manifests
short_image_name = image_description.parent_image
return taggers, manifests

23
tagging/git_helper.py Executable file
View File

@@ -0,0 +1,23 @@
#!/usr/bin/env python3
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
from plumbum.cmd import git
class GitHelper:
@staticmethod
def commit_hash() -> str:
return git["rev-parse", "HEAD"]().strip()
@staticmethod
def commit_hash_tag() -> str:
return GitHelper.commit_hash()[:12]
@staticmethod
def commit_message() -> str:
return git["log", -1, "--pretty=%B"]().strip()
if __name__ == "__main__":
print("Git hash:", GitHelper.commit_hash())
print("Git message:", GitHelper.commit_message())

65
tagging/images_hierarchy.py Normal file
View File

@@ -0,0 +1,65 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
from dataclasses import dataclass, field
from typing import Optional, List
from .taggers import TaggerInterface, \
SHATagger, \
UbuntuVersionTagger, PythonVersionTagger, \
JupyterNotebookVersionTagger, JupyterLabVersionTagger, JupyterHubVersionTagger, \
RVersionTagger, TensorflowVersionTagger, JuliaVersionTagger, \
SparkVersionTagger, HadoopVersionTagger, JavaVersionTagger
from .manifests import ManifestInterface, \
CondaEnvironmentManifest, AptPackagesManifest, \
RPackagesManifest, JuliaPackagesManifest, SparkInfoManifest
@dataclass
class ImageDescription:
parent_image: Optional[str]
taggers: List[TaggerInterface] = field(default_factory=list)
manifests: List[ManifestInterface] = field(default_factory=list)
ALL_IMAGES = {
"base-notebook": ImageDescription(
parent_image=None,
taggers=[
SHATagger,
UbuntuVersionTagger, PythonVersionTagger,
JupyterNotebookVersionTagger, JupyterLabVersionTagger, JupyterHubVersionTagger
],
manifests=[
CondaEnvironmentManifest, AptPackagesManifest
]
),
"minimal-notebook": ImageDescription(
parent_image="base-notebook"
),
"scipy-notebook": ImageDescription(
parent_image="minimal-notebook"
),
"r-notebook": ImageDescription(
parent_image="minimal-notebook",
taggers=[RVersionTagger],
manifests=[RPackagesManifest]
),
"tensorflow-notebook": ImageDescription(
parent_image="scipy-notebook",
taggers=[TensorflowVersionTagger]
),
"datascience-notebook": ImageDescription(
parent_image="scipy-notebook",
taggers=[RVersionTagger, JuliaVersionTagger],
manifests=[RPackagesManifest, JuliaPackagesManifest]
),
"pyspark-notebook": ImageDescription(
parent_image="scipy-notebook",
taggers=[SparkVersionTagger, HadoopVersionTagger, JavaVersionTagger],
manifests=[SparkInfoManifest]
),
"all-spark-notebook": ImageDescription(
parent_image="pyspark-notebook",
taggers=[RVersionTagger],
manifests=[RPackagesManifest]
)
}

108
tagging/manifests.py Normal file
View File

@@ -0,0 +1,108 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import logging
from plumbum.cmd import docker
from .docker_runner import DockerRunner
from .git_helper import GitHelper
logger = logging.getLogger(__name__)
def quoted_output(container, cmd: str) -> str:
return "\n".join([
"```",
DockerRunner.run_simple_command(container, cmd, print_result=False),
"```"
])
class ManifestHeader:
"""ManifestHeader doesn't fall under common interface and we run it separately"""
@staticmethod
def create_header(short_image_name: str, owner: str, build_timestamp: str) -> str:
commit_hash = GitHelper.commit_hash()
commit_hash_tag = GitHelper.commit_hash_tag()
commit_message = GitHelper.commit_message()
image_size = docker["images", f"{owner}/{short_image_name}:latest", "--format", "{{.Size}}"]().rstrip()
return "\n".join([
f"# Build manifest for image: {short_image_name}:{commit_hash_tag}",
"",
"## Build Info",
"",
f"* Build datetime: {build_timestamp}",
f"* Docker image: {owner}/{short_image_name}:{commit_hash_tag}",
f"* Docker image size: {image_size}",
f"* Git commit SHA: [{commit_hash}](https://github.com/jupyter/docker-stacks/commit/{commit_hash})",
"* Git commit message:",
"```",
f"{commit_message}",
"```"
])
class ManifestInterface:
"""Common interface for all manifests"""
@staticmethod
def markdown_piece(container) -> str:
raise NotImplementedError
class CondaEnvironmentManifest(ManifestInterface):
@staticmethod
def markdown_piece(container) -> str:
return "\n".join([
"## Python Packages",
"",
quoted_output(container, "python --version"),
"",
quoted_output(container, "conda info"),
"",
quoted_output(container, "conda list")
])
class AptPackagesManifest(ManifestInterface):
@staticmethod
def markdown_piece(container) -> str:
return "\n".join([
"## Apt Packages",
"",
quoted_output(container, "apt list --installed")
])
class RPackagesManifest(ManifestInterface):
@staticmethod
def markdown_piece(container) -> str:
return "\n".join([
"## R Packages",
"",
quoted_output(container, "R --version"),
"",
quoted_output(container, "R --silent -e 'installed.packages(.Library)[, c(1,3)]'")
])
class JuliaPackagesManifest(ManifestInterface):
@staticmethod
def markdown_piece(container) -> str:
return "\n".join([
"## Julia Packages",
"",
quoted_output(container, "julia -E 'using InteractiveUtils; versioninfo()'"),
"",
quoted_output(container, "julia -E 'import Pkg; Pkg.status()'")
])
class SparkInfoManifest(ManifestInterface):
@staticmethod
def markdown_piece(container) -> str:
return "\n".join([
"## Apache Spark",
"",
quoted_output(container, "/usr/local/spark/bin/spark-submit --version"),
])

36
tagging/tag_image.py Executable file
View File

@@ -0,0 +1,36 @@
#!/usr/bin/env python3
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import argparse
import logging
from plumbum.cmd import docker
from .docker_runner import DockerRunner
from .get_taggers_and_manifests import get_taggers_and_manifests
logger = logging.getLogger(__name__)
def tag_image(short_image_name: str, owner: str) -> None:
logger.info(f"Tagging image: {short_image_name}")
taggers, _ = get_taggers_and_manifests(short_image_name)
image = f"{owner}/{short_image_name}:latest"
with DockerRunner(image) as container:
for tagger in taggers:
tagger_name = tagger.__name__
tag_value = tagger.tag_value(container)
logger.info(f"Applying tag tagger_name: {tagger_name} tag_value: {tag_value}")
docker["tag", image, f"{owner}/{short_image_name}:{tag_value}"]()
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("--short-image-name", required=True, help="Short image name to apply tags for")
arg_parser.add_argument("--owner", required=True, help="Owner of the image")
args = arg_parser.parse_args()
tag_image(args.short_image_name, args.owner)

118
tagging/taggers.py Normal file
View File

@@ -0,0 +1,118 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import logging
from .git_helper import GitHelper
from .docker_runner import DockerRunner
logger = logging.getLogger(__name__)
def _get_program_version(container, program: str) -> str:
return DockerRunner.run_simple_command(container, cmd=f"{program} --version")
def _get_env_variable(container, variable: str) -> str:
env = DockerRunner.run_simple_command(
container,
cmd="env",
print_result=False
).split()
for env_entry in env:
if env_entry.startswith(variable):
return env_entry[len(variable) + 1:]
raise KeyError(variable)
def _get_pip_package_version(container, package: str) -> str:
VERSION_PREFIX = "Version: "
package_info = DockerRunner.run_simple_command(
container,
cmd=f"pip show {package}",
print_result=False
)
version_line = package_info.split("\n")[1]
assert version_line.startswith(VERSION_PREFIX)
return version_line[len(VERSION_PREFIX):]
class TaggerInterface:
"""Common interface for all taggers"""
@staticmethod
def tag_value(container) -> str:
raise NotImplementedError
class SHATagger(TaggerInterface):
@staticmethod
def tag_value(container) -> str:
return GitHelper.commit_hash_tag()
class UbuntuVersionTagger(TaggerInterface):
@staticmethod
def tag_value(container) -> str:
os_release = DockerRunner.run_simple_command(container, "cat /etc/os-release").split("\n")
for line in os_release:
if line.startswith("VERSION_ID"):
return "ubuntu-" + line.split("=")[1].strip('"')
class PythonVersionTagger(TaggerInterface):
@staticmethod
def tag_value(container) -> str:
return "python-" + _get_program_version(container, "python").split()[1]
class JupyterNotebookVersionTagger(TaggerInterface):
@staticmethod
def tag_value(container) -> str:
return "notebook-" + _get_program_version(container, "jupyter-notebook")
class JupyterLabVersionTagger(TaggerInterface):
@staticmethod
def tag_value(container) -> str:
return "lab-" + _get_program_version(container, "jupyter-lab")
class JupyterHubVersionTagger(TaggerInterface):
@staticmethod
def tag_value(container) -> str:
return "hub-" + _get_program_version(container, "jupyterhub")
class RVersionTagger(TaggerInterface):
@staticmethod
def tag_value(container) -> str:
return "r-" + _get_program_version(container, "R").split()[2]
class TensorflowVersionTagger(TaggerInterface):
@staticmethod
def tag_value(container) -> str:
return "tensorflow-" + _get_pip_package_version(container, "tensorflow")
class JuliaVersionTagger(TaggerInterface):
@staticmethod
def tag_value(container) -> str:
return "julia-" + _get_program_version(container, "julia").split()[2]
class SparkVersionTagger(TaggerInterface):
@staticmethod
def tag_value(container) -> str:
return "spark-" + _get_env_variable(container, "APACHE_SPARK_VERSION")
class HadoopVersionTagger(TaggerInterface):
@staticmethod
def tag_value(container) -> str:
return "hadoop-" + _get_env_variable(container, "HADOOP_VERSION")
class JavaVersionTagger(TaggerInterface):
@staticmethod
def tag_value(container) -> str:
return "java-" + _get_program_version(container, "java").split()[1]

45
tensorflow-notebook/hooks/run_hook
View File

@@ -1,45 +0,0 @@
#!/bin/bash
set -e
# Apply tags
GIT_SHA_TAG=${GITHUB_SHA:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
# Update index
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF