Add an ability to specify registry when using docker images (#2008)

* Add an ability to specify registry when using docker images

* Fix typo

* [TMP] Speedup workflow

* Revert "[TMP] Speedup workflow"

This reverts commit 3af0055ccf.
This commit is contained in:
Ayaz Salikhov
2023-10-19 21:15:10 +02:00
committed by GitHub
parent bceaead5d2
commit f8cd90ade1
33 changed files with 119 additions and 56 deletions

View File

@@ -1,6 +1,7 @@
name: Download parent image, build a new one and test it; then upload the image, tags and manifests to GitHub artifacts
env:
REGISTRY: docker.io
OWNER: ${{ github.repository_owner }}
on:
@@ -60,7 +61,7 @@ jobs:
shell: bash
- name: Build image 🛠
run: docker build --rm --force-rm --tag ${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/ --build-arg OWNER=${{ env.OWNER }}
run: docker build --rm --force-rm --tag ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/ --build-arg REGISTRY=${{ env.REGISTRY }} --build-arg OWNER=${{ env.OWNER }}
env:
DOCKER_BUILDKIT: 1
# Full logs for CI build
@@ -68,12 +69,12 @@ jobs:
shell: bash
- name: Run tests ✅
run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --owner ${{ env.OWNER }}
run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
shell: bash
- name: Write tags file 🏷
run: |
python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --owner ${{ env.OWNER }}
python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
shell: bash
- name: Upload tags file 💾
uses: actions/upload-artifact@v3
@@ -83,7 +84,7 @@ jobs:
retention-days: 3
- name: Write manifest and build history file 🏷
run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --owner ${{ env.OWNER }}
run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
shell: bash
- name: Upload manifest file 💾
uses: actions/upload-artifact@v3

View File

@@ -1,6 +1,7 @@
name: Download Docker image and its tags from GitHub artifacts, apply them and push the image to Docker Hub
env:
REGISTRY: docker.io
OWNER: ${{ github.repository_owner }}
on:
@@ -50,7 +51,7 @@ jobs:
name: ${{ inputs.image }}-${{ inputs.platform }}-tags
path: /tmp/jupyter/tags/
- name: Apply tags to the loaded image 🏷
run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --platform ${{ inputs.platform }} --owner ${{ env.OWNER }}
run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --platform ${{ inputs.platform }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
- name: Push Images to Docker Hub 📤
if: github.repository == 'jupyter/docker-stacks' && (github.ref == 'refs/heads/main' || github.event_name == 'schedule')

View File

@@ -4,6 +4,7 @@
# Use bash for inline if-statements in arch_patch target
SHELL:=bash
REGISTRY?=docker.io
OWNER?=jupyter
# Need to list the images in build dependency order
@@ -37,9 +38,9 @@ help:
build/%: DOCKER_BUILD_ARGS?=
build/%: ## build the latest image for a stack using the system's architecture
docker build $(DOCKER_BUILD_ARGS) --rm --force-rm --tag "$(OWNER)/$(notdir $@):latest" "./images/$(notdir $@)" --build-arg OWNER="$(OWNER)"
docker build $(DOCKER_BUILD_ARGS) --rm --force-rm --tag "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" "./images/$(notdir $@)" --build-arg REGISTRY="$(REGISTRY)" --build-arg OWNER="$(OWNER)"
@echo -n "Built image size: "
@docker images "$(OWNER)/$(notdir $@):latest" --format "{{.Size}}"
@docker images "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" --format "{{.Size}}"
build-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks
@@ -68,9 +69,9 @@ linkcheck-docs: ## check broken links
hook/%: ## run post-build hooks for an image
python3 -m tagging.write_tags_file --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --owner "$(OWNER)" && \
python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --owner "$(OWNER)" && \
python3 -m tagging.apply_tags --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --platform "$(shell uname -m)" --owner "$(OWNER)"
python3 -m tagging.write_tags_file --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --registry "$(REGISTRY)" --owner "$(OWNER)" && \
python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --registry "$(REGISTRY)" --owner "$(OWNER)" && \
python3 -m tagging.apply_tags --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --platform "$(shell uname -m)" --registry "$(REGISTRY)" --owner "$(OWNER)"
hook-all: $(foreach I, $(ALL_IMAGES), hook/$(I)) ## run post-build hooks for all images
@@ -105,5 +106,5 @@ run-sudo-shell/%: ## run a bash in interactive mode as root in a stack
test/%: ## run tests against a stack
python3 -m tests.run_tests --short-image-name "$(notdir $@)" --owner "$(OWNER)"
python3 -m tests.run_tests --short-image-name "$(notdir $@)" --registry "$(REGISTRY)" --owner "$(OWNER)"
test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks

View File

@@ -2,8 +2,9 @@
# Distributed under the terms of the Modified BSD License.
# https://hub.docker.com/r/jupyter/base-notebook/tags
ARG REGISTRY=docker.io
ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/base-notebook:2023-09-25
ARG BASE_CONTAINER=$REGISTRY/$OWNER/base-notebook:2023-09-25
FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook
FROM docker.io/jupyter/base-notebook
# Name your environment and choose the python version
ARG env_name=python310

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook
FROM docker.io/jupyter/base-notebook
# Install the Dask dashboard
RUN mamba install --yes 'dask-labextension' && \

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook
FROM docker.io/jupyter/base-notebook
RUN mamba install --yes 'jupyterhub==4.0.1' && \
mamba clean --all -f -y && \

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook
FROM docker.io/jupyter/base-notebook
RUN mamba install --yes 'flake8' && \
mamba clean --all -f -y && \

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook
FROM docker.io/jupyter/base-notebook
# Fix: https://github.com/hadolint/hadolint/wiki/DL4006
# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook
FROM docker.io/jupyter/base-notebook
# Fix: https://github.com/hadolint/hadolint/wiki/DL4006
# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook
FROM docker.io/jupyter/base-notebook
USER root

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook
FROM docker.io/jupyter/base-notebook
# Install in the default python3 environment
RUN pip install --no-cache-dir 'flake8' && \

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook
FROM docker.io/jupyter/base-notebook
RUN mamba install --yes 'jupyterlab_rise' && \
mamba clean --all -f -y && \

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook:notebook-6.5.4
FROM docker.io/jupyter/base-notebook:notebook-6.5.4
RUN pip install --no-cache-dir 'jupyter_contrib_nbextensions' && \
jupyter contrib nbextension install --user && \

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook
FROM docker.io/jupyter/base-notebook
RUN mamba install --yes 'py-xgboost' && \
mamba clean --all -f -y && \

View File

@@ -298,7 +298,7 @@ This recipe is not tested and might be broken.
```
```dockerfile
FROM jupyter/all-spark-notebook
FROM docker.io/jupyter/all-spark-notebook
# Set env vars for pydoop
ENV HADOOP_HOME /usr/local/hadoop-2.7.3
@@ -415,7 +415,7 @@ Please note that the [Delta Lake](https://delta.io/) packages are only available
By adding the properties to `spark-defaults.conf`, the user no longer needs to enable Delta support in each notebook.
```dockerfile
FROM jupyter/pyspark-notebook
FROM docker.io/jupyter/pyspark-notebook
RUN mamba install --yes 'delta-spark' && \
mamba clean --all -f -y && \
@@ -446,7 +446,7 @@ This recipe is not tested and might be broken.
The example below is a Dockerfile to load Source Han Sans with normal weight, usually used for the web.
```dockerfile
FROM jupyter/scipy-notebook
FROM docker.io/jupyter/scipy-notebook
RUN PYV=$(ls "${CONDA_DIR}/lib" | grep ^python) && \
MPL_DATA="${CONDA_DIR}/lib/${PYV}/site-packages/matplotlib/mpl-data" && \
@@ -489,7 +489,7 @@ This recipe is not tested and might be broken.
The example below is a Dockerfile to install the [ijavascript kernel](https://github.com/n-riesco/ijavascript).
```dockerfile
FROM jupyter/scipy-notebook
FROM docker.io/jupyter/scipy-notebook
# install ijavascript
RUN npm install -g ijavascript

View File

@@ -42,7 +42,7 @@ You can customize the docker-stack notebook image to deploy by modifying the `no
For example, you can build and deploy a `jupyter/all-spark-notebook` by modifying the Dockerfile like so:
```dockerfile
FROM jupyter/all-spark-notebook
FROM docker.io/jupyter/all-spark-notebook
# Your RUN commands and so on
```

View File

@@ -2,7 +2,7 @@
# Distributed under the terms of the Modified BSD License.
# Pick your favorite docker-stacks image
FROM jupyter/minimal-notebook
FROM docker.io/jupyter/minimal-notebook
USER root

View File

@@ -2,7 +2,7 @@
# Distributed under the terms of the Modified BSD License.
# Pick your favorite docker-stacks image
FROM jupyter/minimal-notebook
FROM docker.io/jupyter/minimal-notebook
USER root

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/pyspark-notebook
ARG BASE_CONTAINER=$REGISTRY/$OWNER/pyspark-notebook
FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/docker-stacks-foundation
ARG BASE_CONTAINER=$REGISTRY/$OWNER/docker-stacks-foundation
FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/scipy-notebook
ARG BASE_CONTAINER=$REGISTRY/$OWNER/scipy-notebook
FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/minimal-notebook
ARG BASE_CONTAINER=$REGISTRY/$OWNER/minimal-notebook
FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/base-notebook
ARG BASE_CONTAINER=$REGISTRY/$OWNER/base-notebook
FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/scipy-notebook
ARG BASE_CONTAINER=$REGISTRY/$OWNER/scipy-notebook
FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/minimal-notebook
ARG BASE_CONTAINER=$REGISTRY/$OWNER/minimal-notebook
FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/minimal-notebook
ARG BASE_CONTAINER=$REGISTRY/$OWNER/minimal-notebook
FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/scipy-notebook
ARG BASE_CONTAINER=$REGISTRY/$OWNER/scipy-notebook
FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -16,6 +16,7 @@ LOGGER = logging.getLogger(__name__)
def apply_tags(
short_image_name: str,
registry: str,
owner: str,
tags_dir: Path,
platform: str,
@@ -26,7 +27,7 @@ def apply_tags(
"""
LOGGER.info(f"Tagging image: {short_image_name}")
image = f"{owner}/{short_image_name}:latest"
image = f"{registry}/{owner}/{short_image_name}:latest"
filename = f"{platform}-{short_image_name}.txt"
tags = (tags_dir / filename).read_text().splitlines()
@@ -60,6 +61,13 @@ if __name__ == "__main__":
choices=["x86_64", "aarch64", "arm64"],
help="Image platform",
)
arg_parser.add_argument(
"--registry",
required=True,
type=str,
choices=["docker.io", "quay.io"],
help="Image registry",
)
arg_parser.add_argument(
"--owner",
required=True,
@@ -68,4 +76,6 @@ if __name__ == "__main__":
args = arg_parser.parse_args()
args.platform = unify_aarch64(args.platform)
apply_tags(args.short_image_name, args.owner, args.tags_dir, args.platform)
apply_tags(
args.short_image_name, args.registry, args.owner, args.tags_dir, args.platform
)

View File

@@ -23,13 +23,18 @@ class ManifestHeader:
"""ManifestHeader doesn't fall under common interface, and we run it separately"""
@staticmethod
def create_header(short_image_name: str, owner: str, build_timestamp: str) -> str:
def create_header(
short_image_name: str, registry: str, owner: str, build_timestamp: str
) -> str:
commit_hash = GitHelper.commit_hash()
commit_hash_tag = GitHelper.commit_hash_tag()
commit_message = GitHelper.commit_message()
image_size = docker[
"images", f"{owner}/{short_image_name}:latest", "--format", "{{.Size}}"
"images",
f"{registry}/{owner}/{short_image_name}:latest",
"--format",
"{{.Size}}",
]().rstrip()
return "\n".join(
@@ -39,7 +44,7 @@ class ManifestHeader:
"## Build Info",
"",
f"* Build datetime: {build_timestamp}",
f"* Docker image: {owner}/{short_image_name}:{commit_hash_tag}",
f"* Docker image: {registry}/{owner}/{short_image_name}:{commit_hash_tag}",
f"* Docker image size: {image_size}",
f"* Git commit SHA: [{commit_hash}](https://github.com/jupyter/docker-stacks/commit/{commit_hash})",
"* Git commit message:",

View File

@@ -23,6 +23,7 @@ MARKDOWN_LINE_BREAK = "<br />"
def write_build_history_line(
short_image_name: str,
registry: str,
owner: str,
hist_line_dir: Path,
filename: str,
@@ -32,7 +33,7 @@ def write_build_history_line(
date_column = f"`{BUILD_TIMESTAMP}`"
image_column = MARKDOWN_LINE_BREAK.join(
f"`{owner}/{short_image_name}:{tag_value}`" for tag_value in all_tags
f"`{registry}/{owner}/{short_image_name}:{tag_value}`" for tag_value in all_tags
)
commit_hash = GitHelper.commit_hash()
links_column = MARKDOWN_LINE_BREAK.join(
@@ -49,6 +50,7 @@ def write_build_history_line(
def write_manifest_file(
short_image_name: str,
registry: str,
owner: str,
manifest_dir: Path,
filename: str,
@@ -59,7 +61,7 @@ def write_manifest_file(
LOGGER.info(f"Using manifests: {manifest_names}")
markdown_pieces = [
ManifestHeader.create_header(short_image_name, owner, BUILD_TIMESTAMP)
ManifestHeader.create_header(short_image_name, registry, owner, BUILD_TIMESTAMP)
] + [manifest.markdown_piece(container) for manifest in manifests]
markdown_content = "\n\n".join(markdown_pieces) + "\n"
@@ -69,6 +71,7 @@ def write_manifest_file(
def write_manifest(
short_image_name: str,
registry: str,
owner: str,
hist_line_dir: Path,
manifest_dir: Path,
@@ -76,7 +79,7 @@ def write_manifest(
LOGGER.info(f"Creating manifests for image: {short_image_name}")
taggers, manifests = get_taggers_and_manifests(short_image_name)
image = f"{owner}/{short_image_name}:latest"
image = f"{registry}/{owner}/{short_image_name}:latest"
file_prefix = get_platform()
commit_hash_tag = GitHelper.commit_hash_tag()
@@ -88,10 +91,16 @@ def write_manifest(
tags_prefix + "-" + tagger.tag_value(container) for tagger in taggers
]
write_build_history_line(
short_image_name, owner, hist_line_dir, filename, all_tags
short_image_name, registry, owner, hist_line_dir, filename, all_tags
)
write_manifest_file(
short_image_name, owner, manifest_dir, filename, manifests, container
short_image_name,
registry,
owner,
manifest_dir,
filename,
manifests,
container,
)
@@ -116,6 +125,13 @@ if __name__ == "__main__":
type=Path,
help="Directory to save manifest file",
)
arg_parser.add_argument(
"--registry",
required=True,
type=str,
choices=["docker.io", "quay.io"],
help="Image registry",
)
arg_parser.add_argument(
"--owner",
required=True,
@@ -126,5 +142,9 @@ if __name__ == "__main__":
LOGGER.info(f"Current build timestamp: {BUILD_TIMESTAMP}")
write_manifest(
args.short_image_name, args.owner, args.hist_line_dir, args.manifest_dir
args.short_image_name,
args.registry,
args.owner,
args.hist_line_dir,
args.manifest_dir,
)

View File

@@ -14,6 +14,7 @@ LOGGER = logging.getLogger(__name__)
def write_tags_file(
short_image_name: str,
registry: str,
owner: str,
tags_dir: Path,
) -> None:
@@ -23,11 +24,11 @@ def write_tags_file(
LOGGER.info(f"Tagging image: {short_image_name}")
taggers, _ = get_taggers_and_manifests(short_image_name)
image = f"{owner}/{short_image_name}:latest"
image = f"{registry}/{owner}/{short_image_name}:latest"
tags_prefix = get_platform()
filename = f"{tags_prefix}-{short_image_name}.txt"
tags = [f"{owner}/{short_image_name}:{tags_prefix}-latest"]
tags = [f"{registry}/{owner}/{short_image_name}:{tags_prefix}-latest"]
with DockerRunner(image) as container:
for tagger in taggers:
tagger_name = tagger.__class__.__name__
@@ -55,6 +56,13 @@ if __name__ == "__main__":
type=Path,
help="Directory to save tags file",
)
arg_parser.add_argument(
"--registry",
required=True,
type=str,
choices=["docker.io", "quay.io"],
help="Image registry",
)
arg_parser.add_argument(
"--owner",
required=True,
@@ -62,4 +70,4 @@ if __name__ == "__main__":
)
args = arg_parser.parse_args()
write_tags_file(args.short_image_name, args.owner, args.tags_dir)
write_tags_file(args.short_image_name, args.registry, args.owner, args.tags_dir)

View File

@@ -13,11 +13,11 @@ python3 = plumbum.local["python3"]
LOGGER = logging.getLogger(__name__)
def test_image(short_image_name: str, owner: str) -> None:
def test_image(short_image_name: str, registry: str, owner: str) -> None:
LOGGER.info(f"Testing image: {short_image_name}")
test_dirs = get_test_dirs(short_image_name)
LOGGER.info(f"Test dirs to be run: {test_dirs}")
with plumbum.local.env(TEST_IMAGE=f"{owner}/{short_image_name}"):
with plumbum.local.env(TEST_IMAGE=f"{registry}/{owner}/{short_image_name}"):
(
python3[
"-m",
@@ -41,6 +41,13 @@ if __name__ == "__main__":
required=True,
help="Short image name to run test on",
)
arg_parser.add_argument(
"--registry",
required=True,
type=str,
choices=["docker.io", "quay.io"],
help="Image registry",
)
arg_parser.add_argument(
"--owner",
required=True,
@@ -49,4 +56,4 @@ if __name__ == "__main__":
args = arg_parser.parse_args()
test_image(args.short_image_name, args.owner)
test_image(args.short_image_name, args.registry, args.owner)