diff --git a/Makefile b/Makefile index b6c79967..c256ccfe 100644 --- a/Makefile +++ b/Makefile @@ -102,11 +102,8 @@ hook/%: export COMMIT_MSG?=$(shell git log -1 --pretty=%B) hook/%: export GITHUB_SHA?=$(shell git rev-parse HEAD) hook/%: export WIKI_PATH?=../wiki hook/%: ## run post-build hooks for an image - BUILD_TIMESTAMP="$$(date -u +%FT%TZ)" \ - DOCKER_REPO="$(OWNER)/$(notdir $@)" \ - IMAGE_NAME="$(OWNER)/$(notdir $@):latest" \ - IMAGE_SHORT_NAME="$(notdir $@)" \ - ./tagging/apply_tags.py --short-image-name "$(notdir $@)" --owner "$(OWNER)" + ./tagging/tag_image.py --short-image-name "$(notdir $@)" --owner "$(OWNER)" && \ + ./tagging/create_manifests.py --short-image-name "$(notdir $@)" --owner "$(OWNER)" hook-all: $(foreach I,$(ALL_IMAGES),hook/$(I) ) ## run post-build hooks for all images diff --git a/tagging/create_manifests.py b/tagging/create_manifests.py new file mode 100755 index 00000000..16fc7559 --- /dev/null +++ b/tagging/create_manifests.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import argparse +import datetime +import logging +import os +from docker_runner import DockerRunner +from get_taggers_and_manifests import get_taggers_and_manifests +from git_helper import GitHelper +from taggers import SHATagger + + +logger = logging.getLogger(__name__) + + +BUILD_TIMESTAMP = datetime.datetime.utcnow().isoformat()[:-7] + "Z" +MARKDOWN_NEWLINE = "
" + + +def append_build_history_line(short_image_name, owner, wiki_path, all_tags, container): + date_column = f"`{BUILD_TIMESTAMP}`" + image_column = MARKDOWN_NEWLINE.join( + f"`{owner}/{short_image_name}:{tag_value}`" for tag_value in all_tags + ) + commit_sha_tag = SHATagger.tag_value(container) # first 12 letters of commit hash + commit_hash = GitHelper.commit_hash() # full commit hash + links_column = MARKDOWN_NEWLINE.join([ + f"[Git diff](https://github.com/jupyter/docker-stacks/commit/{commit_hash})", + f"[Dockerfile](https://github.com/jupyter/docker-stacks/blob/{commit_hash}/{short_image_name}/Dockerfile)" + f"[Build manifest](./{short_image_name}-{commit_sha_tag})" + ]) + build_history_line = "|".join([date_column, image_column, links_column]) + "|" + + home_wiki_file = os.path.join(wiki_path, 'Home.md') + with open(home_wiki_file, "r") as f: + file = f.read() + file.replace("|-|-|-|", "|-|-|-|\n" + build_history_line) + with open(home_wiki_file, "w") as f: + f.write(file) + + +def create_manifests(short_image_name, owner, wiki_path): + logger.info(f"Creating manifests for image: {short_image_name}") + taggers, manifests = get_taggers_and_manifests(short_image_name) + + image = f"{owner}/{short_image_name}:latest" + + with DockerRunner(image) as container: + all_tags = [tagger.tag_value(container) for tagger in taggers] + append_build_history_line(short_image_name, owner, wiki_path, all_tags, container) + + manifest_names = [manifest.__name__ for manifest in manifests] + logger.info(f"Using manifests: {manifest_names}") + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument("--short-image-name", required=True, help="Short image name to apply tags for") + arg_parser.add_argument("--owner", required=True, help="Owner of the image") + arg_parser.add_argument("--wiki-path", required=True, help="Path to the wiki pages") + args = arg_parser.parse_args() + + logger.info(f"Calculated build BUILD_TIMESTAMP: {BUILD_TIMESTAMP}") + + create_manifests(args.short_image_name, args.owner, args.wiki_path) diff --git a/tagging/get_taggers_and_manifests.py b/tagging/get_taggers_and_manifests.py new file mode 100644 index 00000000..2867601b --- /dev/null +++ b/tagging/get_taggers_and_manifests.py @@ -0,0 +1,16 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +from images_hierarchy import ALL_IMAGES + + +def get_taggers_and_manifests(short_image_name): + taggers = [] + manifests = [] + while short_image_name is not None: + image_description = ALL_IMAGES[short_image_name] + + taggers = image_description.taggers + taggers + manifests = image_description.manifests + manifests + + short_image_name = image_description.parent_image + return taggers, manifests diff --git a/tagging/images_hierarchy.py b/tagging/images_hierarchy.py index 195867b9..43d4725f 100644 --- a/tagging/images_hierarchy.py +++ b/tagging/images_hierarchy.py @@ -8,12 +8,15 @@ from taggers import TaggerInterface, \ JupyterNotebookVersionTagger, JupyterLabVersionTagger, JupyterHubVersionTagger, \ RVersionTagger, TensorflowVersionTagger, JuliaVersionTagger, \ SparkVersionTagger, HadoopVersionTagger, JavaVersionTagger +from manifests import ManifestInterface, \ + BuildInfoManifest, CondaEnvironmentManifest, AptPackagesManifest @dataclass class ImageDescription: parent_image: Optional[str] taggers: List[TaggerInterface] = field(default_factory=list) + manifests: List[ManifestInterface] = field(default_factory=list) ALL_IMAGES = { @@ -23,6 +26,9 @@ ALL_IMAGES = { SHATagger, UbuntuVersionTagger, PythonVersionTagger, JupyterNotebookVersionTagger, JupyterLabVersionTagger, JupyterHubVersionTagger + ], + manifests=[ + BuildInfoManifest, CondaEnvironmentManifest, AptPackagesManifest ] ), "minimal-notebook": ImageDescription( diff --git a/tagging/manifests.py b/tagging/manifests.py new file mode 100644 index 00000000..db9e5fa1 --- /dev/null +++ b/tagging/manifests.py @@ -0,0 +1,31 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import logging + + +logger = logging.getLogger(__name__) + + +class ManifestInterface: + """Common interface for all manifests""" + @staticmethod + def manifest_piece(container): + raise NotImplementedError + + +class BuildInfoManifest(ManifestInterface): + @staticmethod + def manifest_piece(container): + return None + + +class CondaEnvironmentManifest(ManifestInterface): + @staticmethod + def manifest_piece(container): + return None + + +class AptPackagesManifest(ManifestInterface): + @staticmethod + def manifest_piece(container): + return None diff --git a/tagging/apply_tags.py b/tagging/tag_image.py similarity index 53% rename from tagging/apply_tags.py rename to tagging/tag_image.py index fee455d1..6af74b3f 100755 --- a/tagging/apply_tags.py +++ b/tagging/tag_image.py @@ -5,24 +5,15 @@ import argparse import logging from plumbum.cmd import docker from docker_runner import DockerRunner -from images_hierarchy import ALL_IMAGES +from get_taggers_and_manifests import get_taggers_and_manifests logger = logging.getLogger(__name__) -def get_all_taggers(short_image_name): - taggers = [] - while short_image_name is not None: - image_description = ALL_IMAGES[short_image_name] - taggers = image_description.taggers + taggers - short_image_name = image_description.parent_image - return taggers - - -def apply_tags(short_image_name, owner): - logger.info(f"Applying tags for image: {short_image_name}") - taggers = get_all_taggers(short_image_name) +def tag_image(short_image_name, owner): + logger.info(f"Tagging image: {short_image_name}") + taggers, _ = get_taggers_and_manifests(short_image_name) image = f"{owner}/{short_image_name}:latest" @@ -31,7 +22,7 @@ def apply_tags(short_image_name, owner): tagger_name = tagger.__name__ tag_value = tagger.tag_value(container) logger.info(f"Applying tag tagger_name: {tagger_name} tag_value: {tag_value}") - docker["tag", f"{owner}/{short_image_name}:latest", f"{owner}/{short_image_name}:{tag_value}"]() + docker["tag", image, f"{owner}/{short_image_name}:{tag_value}"]() if __name__ == "__main__": @@ -42,9 +33,4 @@ if __name__ == "__main__": arg_parser.add_argument("--owner", required=True, help="Owner of the image") args = arg_parser.parse_args() - short_image_name = args.short_image_name - owner = args.owner - - assert short_image_name in ALL_IMAGES, f"Did not found {short_image_name} image description" - - apply_tags(short_image_name, owner) + tag_image(args.short_image_name, args.owner) diff --git a/tagging/taggers.py b/tagging/taggers.py index 5ac168ed..85ca7858 100644 --- a/tagging/taggers.py +++ b/tagging/taggers.py @@ -29,7 +29,7 @@ def _get_pip_package_version(container, package): class TaggerInterface: - """HooksInterface for all hooks common interface""" + """Common interface for all taggers""" @staticmethod def tag_value(container): raise NotImplementedError