[WIP] Implement draft version of manifests (build_history_line)

Makefile

@@ -102,11 +102,8 @@ hook/%: export COMMIT_MSG?=$(shell git log -1 --pretty=%B)
 hook/%: export GITHUB_SHA?=$(shell git rev-parse HEAD)
 hook/%: export WIKI_PATH?=../wiki
 hook/%: ## run post-build hooks for an image
-	BUILD_TIMESTAMP="$$(date -u +%FT%TZ)" \
+	./tagging/tag_image.py --short-image-name "$(notdir $@)" --owner "$(OWNER)" && \
-	DOCKER_REPO="$(OWNER)/$(notdir $@)" \
+	./tagging/create_manifests.py --short-image-name "$(notdir $@)" --owner "$(OWNER)"
-	IMAGE_NAME="$(OWNER)/$(notdir $@):latest" \
-	IMAGE_SHORT_NAME="$(notdir $@)" \
-	./tagging/apply_tags.py --short-image-name "$(notdir $@)" --owner "$(OWNER)"
 
 hook-all: $(foreach I,$(ALL_IMAGES),hook/$(I) ) ## run post-build hooks for all images
 

tagging/create_manifests.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import argparse
+import datetime
+import logging
+import os
+from docker_runner import DockerRunner
+from get_taggers_and_manifests import get_taggers_and_manifests
+from git_helper import GitHelper
+from taggers import SHATagger
+
+
+logger = logging.getLogger(__name__)
+
+
+BUILD_TIMESTAMP = datetime.datetime.utcnow().isoformat()[:-7] + "Z"
+MARKDOWN_NEWLINE = "<br />"
+
+
+def append_build_history_line(short_image_name, owner, wiki_path, all_tags, container):
+    date_column = f"`{BUILD_TIMESTAMP}`"
+    image_column = MARKDOWN_NEWLINE.join(
+        f"`{owner}/{short_image_name}:{tag_value}`" for tag_value in all_tags
+    )
+    commit_sha_tag = SHATagger.tag_value(container)  # first 12 letters of commit hash
+    commit_hash = GitHelper.commit_hash()  # full commit hash
+    links_column = MARKDOWN_NEWLINE.join([
+        f"[Git diff](https://github.com/jupyter/docker-stacks/commit/{commit_hash})",
+        f"[Dockerfile](https://github.com/jupyter/docker-stacks/blob/{commit_hash}/{short_image_name}/Dockerfile)"
+        f"[Build manifest](./{short_image_name}-{commit_sha_tag})"
+    ])
+    build_history_line = "|".join([date_column, image_column, links_column]) + "|"
+
+    home_wiki_file = os.path.join(wiki_path, 'Home.md')
+    with open(home_wiki_file, "r") as f:
+        file = f.read()
+    file.replace("|-|-|-|", "|-|-|-|\n" + build_history_line)
+    with open(home_wiki_file, "w") as f:
+        f.write(file)
+
+
+def create_manifests(short_image_name, owner, wiki_path):
+    logger.info(f"Creating manifests for image: {short_image_name}")
+    taggers, manifests = get_taggers_and_manifests(short_image_name)
+
+    image = f"{owner}/{short_image_name}:latest"
+
+    with DockerRunner(image) as container:
+        all_tags = [tagger.tag_value(container) for tagger in taggers]
+        append_build_history_line(short_image_name, owner, wiki_path, all_tags, container)
+
+        manifest_names = [manifest.__name__ for manifest in manifests]
+        logger.info(f"Using manifests: {manifest_names}")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG)
+
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument("--short-image-name", required=True, help="Short image name to apply tags for")
+    arg_parser.add_argument("--owner", required=True, help="Owner of the image")
+    arg_parser.add_argument("--wiki-path", required=True, help="Path to the wiki pages")
+    args = arg_parser.parse_args()
+
+    logger.info(f"Calculated build BUILD_TIMESTAMP: {BUILD_TIMESTAMP}")
+
+    create_manifests(args.short_image_name, args.owner, args.wiki_path)

tagging/get_taggers_and_manifests.py

@@ -0,0 +1,16 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+from images_hierarchy import ALL_IMAGES
+
+
+def get_taggers_and_manifests(short_image_name):
+    taggers = []
+    manifests = []
+    while short_image_name is not None:
+        image_description = ALL_IMAGES[short_image_name]
+
+        taggers = image_description.taggers + taggers
+        manifests = image_description.manifests + manifests
+
+        short_image_name = image_description.parent_image
+    return taggers, manifests

tagging/images_hierarchy.py

@@ -8,12 +8,15 @@ from taggers import TaggerInterface, \
     JupyterNotebookVersionTagger, JupyterLabVersionTagger, JupyterHubVersionTagger, \
     RVersionTagger, TensorflowVersionTagger, JuliaVersionTagger, \
     SparkVersionTagger, HadoopVersionTagger, JavaVersionTagger
+from manifests import ManifestInterface, \
+    BuildInfoManifest, CondaEnvironmentManifest, AptPackagesManifest
 
 
 @dataclass
 class ImageDescription:
     parent_image: Optional[str]
     taggers: List[TaggerInterface] = field(default_factory=list)
+    manifests: List[ManifestInterface] = field(default_factory=list)
 
 
 ALL_IMAGES = {
@@ -23,6 +26,9 @@ ALL_IMAGES = {
             SHATagger,
             UbuntuVersionTagger, PythonVersionTagger,
             JupyterNotebookVersionTagger, JupyterLabVersionTagger, JupyterHubVersionTagger
+        ],
+        manifests=[
+            BuildInfoManifest, CondaEnvironmentManifest, AptPackagesManifest
         ]
     ),
     "minimal-notebook": ImageDescription(

tagging/manifests.py

@@ -0,0 +1,31 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class ManifestInterface:
+    """Common interface for all manifests"""
+    @staticmethod
+    def manifest_piece(container):
+        raise NotImplementedError
+
+
+class BuildInfoManifest(ManifestInterface):
+    @staticmethod
+    def manifest_piece(container):
+        return None
+
+
+class CondaEnvironmentManifest(ManifestInterface):
+    @staticmethod
+    def manifest_piece(container):
+        return None
+
+
+class AptPackagesManifest(ManifestInterface):
+    @staticmethod
+    def manifest_piece(container):
+        return None

tagging/tag_image.py

@@ -5,24 +5,15 @@ import argparse
 import logging
 from plumbum.cmd import docker
 from docker_runner import DockerRunner
-from images_hierarchy import ALL_IMAGES
+from get_taggers_and_manifests import get_taggers_and_manifests
 
 
 logger = logging.getLogger(__name__)
 
 
-def get_all_taggers(short_image_name):
+def tag_image(short_image_name, owner):
-    taggers = []
+    logger.info(f"Tagging image: {short_image_name}")
-    while short_image_name is not None:
+    taggers, _ = get_taggers_and_manifests(short_image_name)
-        image_description = ALL_IMAGES[short_image_name]
-        taggers = image_description.taggers + taggers
-        short_image_name = image_description.parent_image
-    return taggers
-
-
-def apply_tags(short_image_name, owner):
-    logger.info(f"Applying tags for image: {short_image_name}")
-    taggers = get_all_taggers(short_image_name)
 
     image = f"{owner}/{short_image_name}:latest"
 
@@ -31,7 +22,7 @@ def apply_tags(short_image_name, owner):
             tagger_name = tagger.__name__
             tag_value = tagger.tag_value(container)
             logger.info(f"Applying tag tagger_name: {tagger_name} tag_value: {tag_value}")
-            docker["tag", f"{owner}/{short_image_name}:latest", f"{owner}/{short_image_name}:{tag_value}"]()
+            docker["tag", image, f"{owner}/{short_image_name}:{tag_value}"]()
 
 
 if __name__ == "__main__":
@@ -42,9 +33,4 @@ if __name__ == "__main__":
     arg_parser.add_argument("--owner", required=True, help="Owner of the image")
     args = arg_parser.parse_args()
 
-    short_image_name = args.short_image_name
+    tag_image(args.short_image_name, args.owner)
-    owner = args.owner
-
-    assert short_image_name in ALL_IMAGES, f"Did not found {short_image_name} image description"
-
-    apply_tags(short_image_name, owner)

tagging/taggers.py

@@ -29,7 +29,7 @@ def _get_pip_package_version(container, package):
 
 
 class TaggerInterface:
-    """HooksInterface for all hooks common interface"""
+    """Common interface for all taggers"""
     @staticmethod
     def tag_value(container):
         raise NotImplementedError