Add an ability to specify registry when using docker images (#2008)

* Add an ability to specify registry when using docker images

* Fix typo

* [TMP] Speedup workflow

* Revert "[TMP] Speedup workflow"

This reverts commit 3af0055ccf.
This commit is contained in:
Ayaz Salikhov
2023-10-19 21:15:10 +02:00
committed by GitHub
parent bceaead5d2
commit f8cd90ade1
33 changed files with 119 additions and 56 deletions

View File

@@ -1,6 +1,7 @@
name: Download parent image, build a new one and test it; then upload the image, tags and manifests to GitHub artifacts name: Download parent image, build a new one and test it; then upload the image, tags and manifests to GitHub artifacts
env: env:
REGISTRY: docker.io
OWNER: ${{ github.repository_owner }} OWNER: ${{ github.repository_owner }}
on: on:
@@ -60,7 +61,7 @@ jobs:
shell: bash shell: bash
- name: Build image 🛠 - name: Build image 🛠
run: docker build --rm --force-rm --tag ${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/ --build-arg OWNER=${{ env.OWNER }} run: docker build --rm --force-rm --tag ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/ --build-arg REGISTRY=${{ env.REGISTRY }} --build-arg OWNER=${{ env.OWNER }}
env: env:
DOCKER_BUILDKIT: 1 DOCKER_BUILDKIT: 1
# Full logs for CI build # Full logs for CI build
@@ -68,12 +69,12 @@ jobs:
shell: bash shell: bash
- name: Run tests ✅ - name: Run tests ✅
run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --owner ${{ env.OWNER }} run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
shell: bash shell: bash
- name: Write tags file 🏷 - name: Write tags file 🏷
run: | run: |
python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --owner ${{ env.OWNER }} python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
shell: bash shell: bash
- name: Upload tags file 💾 - name: Upload tags file 💾
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
@@ -83,7 +84,7 @@ jobs:
retention-days: 3 retention-days: 3
- name: Write manifest and build history file 🏷 - name: Write manifest and build history file 🏷
run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --owner ${{ env.OWNER }} run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
shell: bash shell: bash
- name: Upload manifest file 💾 - name: Upload manifest file 💾
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3

View File

@@ -1,6 +1,7 @@
name: Download Docker image and its tags from GitHub artifacts, apply them and push the image to Docker Hub name: Download Docker image and its tags from GitHub artifacts, apply them and push the image to Docker Hub
env: env:
REGISTRY: docker.io
OWNER: ${{ github.repository_owner }} OWNER: ${{ github.repository_owner }}
on: on:
@@ -50,7 +51,7 @@ jobs:
name: ${{ inputs.image }}-${{ inputs.platform }}-tags name: ${{ inputs.image }}-${{ inputs.platform }}-tags
path: /tmp/jupyter/tags/ path: /tmp/jupyter/tags/
- name: Apply tags to the loaded image 🏷 - name: Apply tags to the loaded image 🏷
run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --platform ${{ inputs.platform }} --owner ${{ env.OWNER }} run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --platform ${{ inputs.platform }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
- name: Push Images to Docker Hub 📤 - name: Push Images to Docker Hub 📤
if: github.repository == 'jupyter/docker-stacks' && (github.ref == 'refs/heads/main' || github.event_name == 'schedule') if: github.repository == 'jupyter/docker-stacks' && (github.ref == 'refs/heads/main' || github.event_name == 'schedule')

View File

@@ -4,6 +4,7 @@
# Use bash for inline if-statements in arch_patch target # Use bash for inline if-statements in arch_patch target
SHELL:=bash SHELL:=bash
REGISTRY?=docker.io
OWNER?=jupyter OWNER?=jupyter
# Need to list the images in build dependency order # Need to list the images in build dependency order
@@ -37,9 +38,9 @@ help:
build/%: DOCKER_BUILD_ARGS?= build/%: DOCKER_BUILD_ARGS?=
build/%: ## build the latest image for a stack using the system's architecture build/%: ## build the latest image for a stack using the system's architecture
docker build $(DOCKER_BUILD_ARGS) --rm --force-rm --tag "$(OWNER)/$(notdir $@):latest" "./images/$(notdir $@)" --build-arg OWNER="$(OWNER)" docker build $(DOCKER_BUILD_ARGS) --rm --force-rm --tag "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" "./images/$(notdir $@)" --build-arg REGISTRY="$(REGISTRY)" --build-arg OWNER="$(OWNER)"
@echo -n "Built image size: " @echo -n "Built image size: "
@docker images "$(OWNER)/$(notdir $@):latest" --format "{{.Size}}" @docker images "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" --format "{{.Size}}"
build-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks build-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks
@@ -68,9 +69,9 @@ linkcheck-docs: ## check broken links
hook/%: ## run post-build hooks for an image hook/%: ## run post-build hooks for an image
python3 -m tagging.write_tags_file --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --owner "$(OWNER)" && \ python3 -m tagging.write_tags_file --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --registry "$(REGISTRY)" --owner "$(OWNER)" && \
python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --owner "$(OWNER)" && \ python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --registry "$(REGISTRY)" --owner "$(OWNER)" && \
python3 -m tagging.apply_tags --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --platform "$(shell uname -m)" --owner "$(OWNER)" python3 -m tagging.apply_tags --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --platform "$(shell uname -m)" --registry "$(REGISTRY)" --owner "$(OWNER)"
hook-all: $(foreach I, $(ALL_IMAGES), hook/$(I)) ## run post-build hooks for all images hook-all: $(foreach I, $(ALL_IMAGES), hook/$(I)) ## run post-build hooks for all images
@@ -105,5 +106,5 @@ run-sudo-shell/%: ## run a bash in interactive mode as root in a stack
test/%: ## run tests against a stack test/%: ## run tests against a stack
python3 -m tests.run_tests --short-image-name "$(notdir $@)" --owner "$(OWNER)" python3 -m tests.run_tests --short-image-name "$(notdir $@)" --registry "$(REGISTRY)" --owner "$(OWNER)"
test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks

View File

@@ -2,8 +2,9 @@
# Distributed under the terms of the Modified BSD License. # Distributed under the terms of the Modified BSD License.
# https://hub.docker.com/r/jupyter/base-notebook/tags # https://hub.docker.com/r/jupyter/base-notebook/tags
ARG REGISTRY=docker.io
ARG OWNER=jupyter ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/base-notebook:2023-09-25 ARG BASE_CONTAINER=$REGISTRY/$OWNER/base-notebook:2023-09-25
FROM $BASE_CONTAINER FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook FROM docker.io/jupyter/base-notebook
# Name your environment and choose the python version # Name your environment and choose the python version
ARG env_name=python310 ARG env_name=python310

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook FROM docker.io/jupyter/base-notebook
# Install the Dask dashboard # Install the Dask dashboard
RUN mamba install --yes 'dask-labextension' && \ RUN mamba install --yes 'dask-labextension' && \

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook FROM docker.io/jupyter/base-notebook
RUN mamba install --yes 'jupyterhub==4.0.1' && \ RUN mamba install --yes 'jupyterhub==4.0.1' && \
mamba clean --all -f -y && \ mamba clean --all -f -y && \

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook FROM docker.io/jupyter/base-notebook
RUN mamba install --yes 'flake8' && \ RUN mamba install --yes 'flake8' && \
mamba clean --all -f -y && \ mamba clean --all -f -y && \

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook FROM docker.io/jupyter/base-notebook
# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 # Fix: https://github.com/hadolint/hadolint/wiki/DL4006
# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 # Fix: https://github.com/koalaman/shellcheck/wiki/SC3014

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook FROM docker.io/jupyter/base-notebook
# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 # Fix: https://github.com/hadolint/hadolint/wiki/DL4006
# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 # Fix: https://github.com/koalaman/shellcheck/wiki/SC3014

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook FROM docker.io/jupyter/base-notebook
USER root USER root

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook FROM docker.io/jupyter/base-notebook
# Install in the default python3 environment # Install in the default python3 environment
RUN pip install --no-cache-dir 'flake8' && \ RUN pip install --no-cache-dir 'flake8' && \

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook FROM docker.io/jupyter/base-notebook
RUN mamba install --yes 'jupyterlab_rise' && \ RUN mamba install --yes 'jupyterlab_rise' && \
mamba clean --all -f -y && \ mamba clean --all -f -y && \

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook:notebook-6.5.4 FROM docker.io/jupyter/base-notebook:notebook-6.5.4
RUN pip install --no-cache-dir 'jupyter_contrib_nbextensions' && \ RUN pip install --no-cache-dir 'jupyter_contrib_nbextensions' && \
jupyter contrib nbextension install --user && \ jupyter contrib nbextension install --user && \

View File

@@ -1,4 +1,4 @@
FROM jupyter/base-notebook FROM docker.io/jupyter/base-notebook
RUN mamba install --yes 'py-xgboost' && \ RUN mamba install --yes 'py-xgboost' && \
mamba clean --all -f -y && \ mamba clean --all -f -y && \

View File

@@ -298,7 +298,7 @@ This recipe is not tested and might be broken.
``` ```
```dockerfile ```dockerfile
FROM jupyter/all-spark-notebook FROM docker.io/jupyter/all-spark-notebook
# Set env vars for pydoop # Set env vars for pydoop
ENV HADOOP_HOME /usr/local/hadoop-2.7.3 ENV HADOOP_HOME /usr/local/hadoop-2.7.3
@@ -415,7 +415,7 @@ Please note that the [Delta Lake](https://delta.io/) packages are only available
By adding the properties to `spark-defaults.conf`, the user no longer needs to enable Delta support in each notebook. By adding the properties to `spark-defaults.conf`, the user no longer needs to enable Delta support in each notebook.
```dockerfile ```dockerfile
FROM jupyter/pyspark-notebook FROM docker.io/jupyter/pyspark-notebook
RUN mamba install --yes 'delta-spark' && \ RUN mamba install --yes 'delta-spark' && \
mamba clean --all -f -y && \ mamba clean --all -f -y && \
@@ -446,7 +446,7 @@ This recipe is not tested and might be broken.
The example below is a Dockerfile to load Source Han Sans with normal weight, usually used for the web. The example below is a Dockerfile to load Source Han Sans with normal weight, usually used for the web.
```dockerfile ```dockerfile
FROM jupyter/scipy-notebook FROM docker.io/jupyter/scipy-notebook
RUN PYV=$(ls "${CONDA_DIR}/lib" | grep ^python) && \ RUN PYV=$(ls "${CONDA_DIR}/lib" | grep ^python) && \
MPL_DATA="${CONDA_DIR}/lib/${PYV}/site-packages/matplotlib/mpl-data" && \ MPL_DATA="${CONDA_DIR}/lib/${PYV}/site-packages/matplotlib/mpl-data" && \
@@ -489,7 +489,7 @@ This recipe is not tested and might be broken.
The example below is a Dockerfile to install the [ijavascript kernel](https://github.com/n-riesco/ijavascript). The example below is a Dockerfile to install the [ijavascript kernel](https://github.com/n-riesco/ijavascript).
```dockerfile ```dockerfile
FROM jupyter/scipy-notebook FROM docker.io/jupyter/scipy-notebook
# install ijavascript # install ijavascript
RUN npm install -g ijavascript RUN npm install -g ijavascript

View File

@@ -42,7 +42,7 @@ You can customize the docker-stack notebook image to deploy by modifying the `no
For example, you can build and deploy a `jupyter/all-spark-notebook` by modifying the Dockerfile like so: For example, you can build and deploy a `jupyter/all-spark-notebook` by modifying the Dockerfile like so:
```dockerfile ```dockerfile
FROM jupyter/all-spark-notebook FROM docker.io/jupyter/all-spark-notebook
# Your RUN commands and so on # Your RUN commands and so on
``` ```

View File

@@ -2,7 +2,7 @@
# Distributed under the terms of the Modified BSD License. # Distributed under the terms of the Modified BSD License.
# Pick your favorite docker-stacks image # Pick your favorite docker-stacks image
FROM jupyter/minimal-notebook FROM docker.io/jupyter/minimal-notebook
USER root USER root

View File

@@ -2,7 +2,7 @@
# Distributed under the terms of the Modified BSD License. # Distributed under the terms of the Modified BSD License.
# Pick your favorite docker-stacks image # Pick your favorite docker-stacks image
FROM jupyter/minimal-notebook FROM docker.io/jupyter/minimal-notebook
USER root USER root

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team. # Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License. # Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/pyspark-notebook ARG BASE_CONTAINER=$REGISTRY/$OWNER/pyspark-notebook
FROM $BASE_CONTAINER FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team. # Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License. # Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/docker-stacks-foundation ARG BASE_CONTAINER=$REGISTRY/$OWNER/docker-stacks-foundation
FROM $BASE_CONTAINER FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team. # Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License. # Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/scipy-notebook ARG BASE_CONTAINER=$REGISTRY/$OWNER/scipy-notebook
FROM $BASE_CONTAINER FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team. # Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License. # Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/minimal-notebook ARG BASE_CONTAINER=$REGISTRY/$OWNER/minimal-notebook
FROM $BASE_CONTAINER FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team. # Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License. # Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/base-notebook ARG BASE_CONTAINER=$REGISTRY/$OWNER/base-notebook
FROM $BASE_CONTAINER FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team. # Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License. # Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/scipy-notebook ARG BASE_CONTAINER=$REGISTRY/$OWNER/scipy-notebook
FROM $BASE_CONTAINER FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team. # Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License. # Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/minimal-notebook ARG BASE_CONTAINER=$REGISTRY/$OWNER/minimal-notebook
FROM $BASE_CONTAINER FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team. # Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License. # Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/minimal-notebook ARG BASE_CONTAINER=$REGISTRY/$OWNER/minimal-notebook
FROM $BASE_CONTAINER FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -1,7 +1,8 @@
# Copyright (c) Jupyter Development Team. # Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License. # Distributed under the terms of the Modified BSD License.
ARG REGISTRY=docker.io
ARG OWNER=jupyter ARG OWNER=jupyter
ARG BASE_CONTAINER=$OWNER/scipy-notebook ARG BASE_CONTAINER=$REGISTRY/$OWNER/scipy-notebook
FROM $BASE_CONTAINER FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"

View File

@@ -16,6 +16,7 @@ LOGGER = logging.getLogger(__name__)
def apply_tags( def apply_tags(
short_image_name: str, short_image_name: str,
registry: str,
owner: str, owner: str,
tags_dir: Path, tags_dir: Path,
platform: str, platform: str,
@@ -26,7 +27,7 @@ def apply_tags(
""" """
LOGGER.info(f"Tagging image: {short_image_name}") LOGGER.info(f"Tagging image: {short_image_name}")
image = f"{owner}/{short_image_name}:latest" image = f"{registry}/{owner}/{short_image_name}:latest"
filename = f"{platform}-{short_image_name}.txt" filename = f"{platform}-{short_image_name}.txt"
tags = (tags_dir / filename).read_text().splitlines() tags = (tags_dir / filename).read_text().splitlines()
@@ -60,6 +61,13 @@ if __name__ == "__main__":
choices=["x86_64", "aarch64", "arm64"], choices=["x86_64", "aarch64", "arm64"],
help="Image platform", help="Image platform",
) )
arg_parser.add_argument(
"--registry",
required=True,
type=str,
choices=["docker.io", "quay.io"],
help="Image registry",
)
arg_parser.add_argument( arg_parser.add_argument(
"--owner", "--owner",
required=True, required=True,
@@ -68,4 +76,6 @@ if __name__ == "__main__":
args = arg_parser.parse_args() args = arg_parser.parse_args()
args.platform = unify_aarch64(args.platform) args.platform = unify_aarch64(args.platform)
apply_tags(args.short_image_name, args.owner, args.tags_dir, args.platform) apply_tags(
args.short_image_name, args.registry, args.owner, args.tags_dir, args.platform
)

View File

@@ -23,13 +23,18 @@ class ManifestHeader:
"""ManifestHeader doesn't fall under common interface, and we run it separately""" """ManifestHeader doesn't fall under common interface, and we run it separately"""
@staticmethod @staticmethod
def create_header(short_image_name: str, owner: str, build_timestamp: str) -> str: def create_header(
short_image_name: str, registry: str, owner: str, build_timestamp: str
) -> str:
commit_hash = GitHelper.commit_hash() commit_hash = GitHelper.commit_hash()
commit_hash_tag = GitHelper.commit_hash_tag() commit_hash_tag = GitHelper.commit_hash_tag()
commit_message = GitHelper.commit_message() commit_message = GitHelper.commit_message()
image_size = docker[ image_size = docker[
"images", f"{owner}/{short_image_name}:latest", "--format", "{{.Size}}" "images",
f"{registry}/{owner}/{short_image_name}:latest",
"--format",
"{{.Size}}",
]().rstrip() ]().rstrip()
return "\n".join( return "\n".join(
@@ -39,7 +44,7 @@ class ManifestHeader:
"## Build Info", "## Build Info",
"", "",
f"* Build datetime: {build_timestamp}", f"* Build datetime: {build_timestamp}",
f"* Docker image: {owner}/{short_image_name}:{commit_hash_tag}", f"* Docker image: {registry}/{owner}/{short_image_name}:{commit_hash_tag}",
f"* Docker image size: {image_size}", f"* Docker image size: {image_size}",
f"* Git commit SHA: [{commit_hash}](https://github.com/jupyter/docker-stacks/commit/{commit_hash})", f"* Git commit SHA: [{commit_hash}](https://github.com/jupyter/docker-stacks/commit/{commit_hash})",
"* Git commit message:", "* Git commit message:",

View File

@@ -23,6 +23,7 @@ MARKDOWN_LINE_BREAK = "<br />"
def write_build_history_line( def write_build_history_line(
short_image_name: str, short_image_name: str,
registry: str,
owner: str, owner: str,
hist_line_dir: Path, hist_line_dir: Path,
filename: str, filename: str,
@@ -32,7 +33,7 @@ def write_build_history_line(
date_column = f"`{BUILD_TIMESTAMP}`" date_column = f"`{BUILD_TIMESTAMP}`"
image_column = MARKDOWN_LINE_BREAK.join( image_column = MARKDOWN_LINE_BREAK.join(
f"`{owner}/{short_image_name}:{tag_value}`" for tag_value in all_tags f"`{registry}/{owner}/{short_image_name}:{tag_value}`" for tag_value in all_tags
) )
commit_hash = GitHelper.commit_hash() commit_hash = GitHelper.commit_hash()
links_column = MARKDOWN_LINE_BREAK.join( links_column = MARKDOWN_LINE_BREAK.join(
@@ -49,6 +50,7 @@ def write_build_history_line(
def write_manifest_file( def write_manifest_file(
short_image_name: str, short_image_name: str,
registry: str,
owner: str, owner: str,
manifest_dir: Path, manifest_dir: Path,
filename: str, filename: str,
@@ -59,7 +61,7 @@ def write_manifest_file(
LOGGER.info(f"Using manifests: {manifest_names}") LOGGER.info(f"Using manifests: {manifest_names}")
markdown_pieces = [ markdown_pieces = [
ManifestHeader.create_header(short_image_name, owner, BUILD_TIMESTAMP) ManifestHeader.create_header(short_image_name, registry, owner, BUILD_TIMESTAMP)
] + [manifest.markdown_piece(container) for manifest in manifests] ] + [manifest.markdown_piece(container) for manifest in manifests]
markdown_content = "\n\n".join(markdown_pieces) + "\n" markdown_content = "\n\n".join(markdown_pieces) + "\n"
@@ -69,6 +71,7 @@ def write_manifest_file(
def write_manifest( def write_manifest(
short_image_name: str, short_image_name: str,
registry: str,
owner: str, owner: str,
hist_line_dir: Path, hist_line_dir: Path,
manifest_dir: Path, manifest_dir: Path,
@@ -76,7 +79,7 @@ def write_manifest(
LOGGER.info(f"Creating manifests for image: {short_image_name}") LOGGER.info(f"Creating manifests for image: {short_image_name}")
taggers, manifests = get_taggers_and_manifests(short_image_name) taggers, manifests = get_taggers_and_manifests(short_image_name)
image = f"{owner}/{short_image_name}:latest" image = f"{registry}/{owner}/{short_image_name}:latest"
file_prefix = get_platform() file_prefix = get_platform()
commit_hash_tag = GitHelper.commit_hash_tag() commit_hash_tag = GitHelper.commit_hash_tag()
@@ -88,10 +91,16 @@ def write_manifest(
tags_prefix + "-" + tagger.tag_value(container) for tagger in taggers tags_prefix + "-" + tagger.tag_value(container) for tagger in taggers
] ]
write_build_history_line( write_build_history_line(
short_image_name, owner, hist_line_dir, filename, all_tags short_image_name, registry, owner, hist_line_dir, filename, all_tags
) )
write_manifest_file( write_manifest_file(
short_image_name, owner, manifest_dir, filename, manifests, container short_image_name,
registry,
owner,
manifest_dir,
filename,
manifests,
container,
) )
@@ -116,6 +125,13 @@ if __name__ == "__main__":
type=Path, type=Path,
help="Directory to save manifest file", help="Directory to save manifest file",
) )
arg_parser.add_argument(
"--registry",
required=True,
type=str,
choices=["docker.io", "quay.io"],
help="Image registry",
)
arg_parser.add_argument( arg_parser.add_argument(
"--owner", "--owner",
required=True, required=True,
@@ -126,5 +142,9 @@ if __name__ == "__main__":
LOGGER.info(f"Current build timestamp: {BUILD_TIMESTAMP}") LOGGER.info(f"Current build timestamp: {BUILD_TIMESTAMP}")
write_manifest( write_manifest(
args.short_image_name, args.owner, args.hist_line_dir, args.manifest_dir args.short_image_name,
args.registry,
args.owner,
args.hist_line_dir,
args.manifest_dir,
) )

View File

@@ -14,6 +14,7 @@ LOGGER = logging.getLogger(__name__)
def write_tags_file( def write_tags_file(
short_image_name: str, short_image_name: str,
registry: str,
owner: str, owner: str,
tags_dir: Path, tags_dir: Path,
) -> None: ) -> None:
@@ -23,11 +24,11 @@ def write_tags_file(
LOGGER.info(f"Tagging image: {short_image_name}") LOGGER.info(f"Tagging image: {short_image_name}")
taggers, _ = get_taggers_and_manifests(short_image_name) taggers, _ = get_taggers_and_manifests(short_image_name)
image = f"{owner}/{short_image_name}:latest" image = f"{registry}/{owner}/{short_image_name}:latest"
tags_prefix = get_platform() tags_prefix = get_platform()
filename = f"{tags_prefix}-{short_image_name}.txt" filename = f"{tags_prefix}-{short_image_name}.txt"
tags = [f"{owner}/{short_image_name}:{tags_prefix}-latest"] tags = [f"{registry}/{owner}/{short_image_name}:{tags_prefix}-latest"]
with DockerRunner(image) as container: with DockerRunner(image) as container:
for tagger in taggers: for tagger in taggers:
tagger_name = tagger.__class__.__name__ tagger_name = tagger.__class__.__name__
@@ -55,6 +56,13 @@ if __name__ == "__main__":
type=Path, type=Path,
help="Directory to save tags file", help="Directory to save tags file",
) )
arg_parser.add_argument(
"--registry",
required=True,
type=str,
choices=["docker.io", "quay.io"],
help="Image registry",
)
arg_parser.add_argument( arg_parser.add_argument(
"--owner", "--owner",
required=True, required=True,
@@ -62,4 +70,4 @@ if __name__ == "__main__":
) )
args = arg_parser.parse_args() args = arg_parser.parse_args()
write_tags_file(args.short_image_name, args.owner, args.tags_dir) write_tags_file(args.short_image_name, args.registry, args.owner, args.tags_dir)

View File

@@ -13,11 +13,11 @@ python3 = plumbum.local["python3"]
LOGGER = logging.getLogger(__name__) LOGGER = logging.getLogger(__name__)
def test_image(short_image_name: str, owner: str) -> None: def test_image(short_image_name: str, registry: str, owner: str) -> None:
LOGGER.info(f"Testing image: {short_image_name}") LOGGER.info(f"Testing image: {short_image_name}")
test_dirs = get_test_dirs(short_image_name) test_dirs = get_test_dirs(short_image_name)
LOGGER.info(f"Test dirs to be run: {test_dirs}") LOGGER.info(f"Test dirs to be run: {test_dirs}")
with plumbum.local.env(TEST_IMAGE=f"{owner}/{short_image_name}"): with plumbum.local.env(TEST_IMAGE=f"{registry}/{owner}/{short_image_name}"):
( (
python3[ python3[
"-m", "-m",
@@ -41,6 +41,13 @@ if __name__ == "__main__":
required=True, required=True,
help="Short image name to run test on", help="Short image name to run test on",
) )
arg_parser.add_argument(
"--registry",
required=True,
type=str,
choices=["docker.io", "quay.io"],
help="Image registry",
)
arg_parser.add_argument( arg_parser.add_argument(
"--owner", "--owner",
required=True, required=True,
@@ -49,4 +56,4 @@ if __name__ == "__main__":
args = arg_parser.parse_args() args = arg_parser.parse_args()
test_image(args.short_image_name, args.owner) test_image(args.short_image_name, args.registry, args.owner)