Upload manifests in workflow

.github/workflows/docker-tag-manifest-push.yml

@@ -40,6 +40,20 @@ jobs:
         run: python3 -m tagging.tag_image --short-image-name ${{ matrix.image }}
         shell: bash
 
+      - name: Write manifest files 🏷
+        run: python3 -m tagging.write_manifests --short-image-name ${{ matrix.image }}
+        shell: bash
+      - name: Upload manifest file 💾
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.image }}-${{ inputs.platform }}-manifest
+          path: /tmp/manifests/*.md
+      - name: Upload build history line 💾
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.image }}-${{ inputs.platform }}-history_line
+          path: /tmp/build_history_lines/*.txt
+
       - name: Login to Docker Hub 🔐
         if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.event_name == 'schedule'
         uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # dependabot updates to latest release

.github/workflows/docker.yml

@@ -9,7 +9,7 @@ on:
       - ".github/workflows/docker.yml"
       - ".github/workflows/docker-build-upload.yml"
       - ".github/workflows/docker-test.yml"
-      - ".github/workflows/docker-tag-push.yml"
+      - ".github/workflows/docker-tag-manifest-push.yml"
 
       - ".github/actions/create-dev-env/action.yml"
       - ".github/actions/load-image/action.yml"
@@ -34,7 +34,7 @@ on:
       - ".github/workflows/docker.yml"
       - ".github/workflows/docker-build-upload.yml"
       - ".github/workflows/docker-test.yml"
-      - ".github/workflows/docker-tag-push.yml"
+      - ".github/workflows/docker-tag-manifest-push.yml"
 
       - ".github/actions/create-dev-env/action.yml"
       - ".github/actions/load-image/action.yml"
@@ -94,7 +94,7 @@ jobs:
 
   amd64-images-tag-push:
     needs: [test-amd64-base, test-amd64-minimal]
-    uses: ./.github/workflows/docker-tag-push.yml
+    uses: ./.github/workflows/docker-tag-manifest-push.yml
     with:
       platform: amd64
       runsOn: ubuntu-latest

tagging/tag_image.py

@@ -3,23 +3,18 @@
 # Distributed under the terms of the Modified BSD License.
 import argparse
 import logging
-import platform
 
 import plumbum
 
 from tagging.docker_runner import DockerRunner
 from tagging.get_taggers_and_manifests import get_taggers_and_manifests
+from tagging.tags_prefix import get_tags_prefix
 
 docker = plumbum.local["docker"]
 
 LOGGER = logging.getLogger(__name__)
 
 
-def get_tags_prefix() -> str:
-    machine = platform.machine()
-    return "" if machine == "x86_64" else f"{machine}-"
-
-
 def tag_image(short_image_name: str, owner: str, tags_prefix: str) -> None:
     """
     Tags <owner>/<short_image_name>:latest with the tags reported by all taggers
@@ -62,6 +57,5 @@ if __name__ == "__main__":
     args = arg_parser.parse_args()
 
     tags_prefix = get_tags_prefix()
-    LOGGER.info(f"Using: {tags_prefix=}")
 
     tag_image(args.short_image_name, args.owner, tags_prefix)

tagging/tags_prefix.py

@@ -0,0 +1,8 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import platform
+
+
+def get_tags_prefix() -> str:
+    machine = platform.machine()
+    return "" if machine == "x86_64" else f"{machine}-"

tagging/write_manifests.py

@@ -4,7 +4,7 @@
 import argparse
 import datetime
 import logging
-import os
+from pathlib import Path
 
 from docker.models.containers import Container
 
@@ -12,6 +12,7 @@ from tagging.docker_runner import DockerRunner
 from tagging.get_taggers_and_manifests import get_taggers_and_manifests
 from tagging.git_helper import GitHelper
 from tagging.manifests import ManifestHeader, ManifestInterface
+from tagging.tags_prefix import get_tags_prefix
 
 LOGGER = logging.getLogger(__name__)
 
@@ -20,10 +21,10 @@ BUILD_TIMESTAMP = datetime.datetime.utcnow().isoformat()[:-7] + "Z"
 MARKDOWN_LINE_BREAK = "<br />"
 
 
-def append_build_history_line(
+def write_build_history_line(
     short_image_name: str,
     owner: str,
-    wiki_path: str,
+    manifest_filename: str,
     all_tags: list[str],
 ) -> None:
     LOGGER.info("Appending build history line")
@@ -33,60 +34,55 @@ def append_build_history_line(
         f"`{owner}/{short_image_name}:{tag_value}`" for tag_value in all_tags
     )
     commit_hash = GitHelper.commit_hash()
-    commit_hash_tag = GitHelper.commit_hash_tag()
     links_column = MARKDOWN_LINE_BREAK.join(
         [
             f"[Git diff](https://github.com/jupyter/docker-stacks/commit/{commit_hash})",
             f"[Dockerfile](https://github.com/jupyter/docker-stacks/blob/{commit_hash}/{short_image_name}/Dockerfile)",
-            f"[Build manifest](./{short_image_name}-{commit_hash_tag})",
+            f"[Build manifest](./{manifest_filename.removesuffix('.md')})",
         ]
     )
     build_history_line = "|".join([date_column, image_column, links_column]) + "|"
-
+    build_history_filename = manifest_filename.replace(".md", ".txt")
-    home_wiki_file = os.path.join(wiki_path, "Home.md")
+    Path(f"/tmp/build_history_lines/{build_history_filename}").write_text(
-    with open(home_wiki_file) as f:
+        build_history_line
-        file = f.read()
+    )
-    TABLE_BEGINNING = "|-|-|-|\n"
-    file = file.replace(TABLE_BEGINNING, TABLE_BEGINNING + build_history_line + "\n")
-    with open(home_wiki_file, "w") as f:
-        f.write(file)
 
 
-def create_manifest_file(
+def write_manifest_file(
     short_image_name: str,
     owner: str,
-    wiki_path: str,
+    manifest_filename: str,
     manifests: list[ManifestInterface],
     container: Container,
 ) -> None:
     manifest_names = [manifest.__class__.__name__ for manifest in manifests]
     LOGGER.info(f"Using manifests: {manifest_names}")
 
-    commit_hash_tag = GitHelper.commit_hash_tag()
-    manifest_file = os.path.join(
-        wiki_path,
-        f"manifests/{short_image_name}-{commit_hash_tag}.md",
-    )
-
     markdown_pieces = [
         ManifestHeader.create_header(short_image_name, owner, BUILD_TIMESTAMP)
     ] + [manifest.markdown_piece(container) for manifest in manifests]
     markdown_content = "\n\n".join(markdown_pieces) + "\n"
 
-    with open(manifest_file, "w") as f:
+    Path(f"/tmp/manifests/{manifest_filename}").write_text(markdown_content)
-        f.write(markdown_content)
 
 
-def create_manifests(short_image_name: str, owner: str, wiki_path: str) -> None:
+def write_manifests(short_image_name: str, owner: str) -> None:
     LOGGER.info(f"Creating manifests for image: {short_image_name}")
     taggers, manifests = get_taggers_and_manifests(short_image_name)
 
     image = f"{owner}/{short_image_name}:latest"
 
+    tags_prefix = get_tags_prefix()
+    commit_hash_tag = GitHelper.commit_hash_tag()
+    filename = f"{tags_prefix}{short_image_name}-{commit_hash_tag}.md"
+    manifest_filename = f"{filename}.md"
+
     with DockerRunner(image) as container:
-        all_tags = [tagger.tag_value(container) for tagger in taggers]
+        all_tags = [tags_prefix + tagger.tag_value(container) for tagger in taggers]
-        append_build_history_line(short_image_name, owner, wiki_path, all_tags)
+        write_build_history_line(short_image_name, owner, all_tags)
-        create_manifest_file(short_image_name, owner, wiki_path, manifests, container)
+        write_manifest_file(
+            short_image_name, owner, manifest_filename, manifests, container
+        )
 
 
 if __name__ == "__main__":
@@ -96,12 +92,11 @@ if __name__ == "__main__":
     arg_parser.add_argument(
         "--short-image-name",
         required=True,
-        help="Short image name to apply tags for",
+        help="Short image name to create manifests for",
     )
-    arg_parser.add_argument("--owner", required=True, help="Owner of the image")
+    arg_parser.add_argument("--owner", default="jupyter", help="Owner of the image")
-    arg_parser.add_argument("--wiki-path", required=True, help="Path to the wiki pages")
     args = arg_parser.parse_args()
 
     LOGGER.info(f"Current build timestamp: {BUILD_TIMESTAMP}")
 
-    create_manifests(args.short_image_name, args.owner, args.wiki_path)
+    write_manifests(args.short_image_name, args.owner)