Improve artifacts handling and writing manifest

.github/workflows/docker-build-test-upload.yml

@@ -70,6 +70,5 @@ jobs:
       - name: Cleanup artifacts 🗑️
         run: |
           rm -f /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
-          docker system prune --all --force
         shell: bash
         if: always()

.github/workflows/docker-tag-manifest-push.yml

@@ -38,9 +38,12 @@ jobs:
           platform: ${{ inputs.platform }}
 
       # Self-hosted runners share a state (whole VM) between runs
-      - name: Reset docker state 🗑️
+      - name: Reset docker state and cleanup artifacts 🗑️
         if: ${{ inputs.platform != 'amd64' }}
-        run: docker system prune --all --force
+        run: |
+          docker system prune --all --force
+          rm -rf /tmp/hist_lines/
+          rm -rf /tmp/manifests/
         shell: bash
 
       - name: Load image to Docker 📥
@@ -55,8 +58,8 @@ jobs:
           docker image ls -a
         shell: bash
 
-      - name: Write manifest files 🏷
+      - name: Write manifest and build history file 🏷
-        run: python3 -m tagging.write_manifests --short-image-name ${{ matrix.image }} --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/
+        run: python3 -m tagging.write_manifest --short-image-name ${{ matrix.image }} --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/
         shell: bash
       - name: Upload manifest file 💾
         uses: actions/upload-artifact@v3
@@ -71,6 +74,11 @@ jobs:
           path: /tmp/hist_lines/${{ inputs.platform }}-${{ matrix.image }}-*.txt
           retention-days: 3
 
+      - name: Remove aarch64 latest tag
+        if: ${{ inputs.platform != 'amd64' }}
+        run: docker rmi jupyter/${{ matrix.image }}
+        shell: bash
+
       - name: Login to Docker Hub 🔐
         if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.event_name == 'schedule'
         uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # dependabot updates to latest release
@@ -82,12 +90,3 @@ jobs:
         if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.event_name == 'schedule'
         run: docker push --all-tags jupyter/${{ matrix.image }}
         shell: bash
-
-      # Self-hosted runners share a state (whole VM) between runs
-      - name: Cleanup artifacts 🗑️
-        run: |
-          rm -rf /tmp/hist_lines/
-          rm -rf /tmp/manifests/
-          docker system prune --all --force
-        shell: bash
-        if: always()

Makefile

@@ -70,7 +70,7 @@ linkcheck-docs: ## check broken links
 hook/%: WIKI_PATH?=wiki
 hook/%: ## run post-build hooks for an image
 	python3 -m tagging.tag_image --short-image-name "$(notdir $@)" --owner "$(OWNER)" && \
-	python3 -m tagging.create_manifests --short-image-name "$(notdir $@)" --owner "$(OWNER)" --wiki-path "$(WIKI_PATH)"
+	python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --owner "$(OWNER)" --wiki-path "$(WIKI_PATH)"
 hook-all: $(foreach I, $(ALL_IMAGES), hook/$(I)) ## run post-build hooks for all images
 
 

tagging/README.md

@@ -113,7 +113,7 @@ class AptPackagesManifest(ManifestInterface):
 
 - `quoted_output` simply runs the command inside container using `DockerRunner.run_simple_command` and wraps it to triple quotes to create a valid markdown piece of file.
 - `manifests.py` contains all the manifests.
-- `create_manifests.py` is a python executable which is used to create the build manifest for an image.
+- `write_manifest.py` is a python executable which is used to create the build manifest and history line for an image.
 
 ### Images Hierarchy
 

tagging/tag_image.py

@@ -37,9 +37,8 @@ def tag_image(short_image_name: str, owner: str) -> None:
                 "tag", image, f"{owner}/{short_image_name}:{tags_prefix}{tag_value}"
             ]()
     if tags_prefix != "":
-        LOGGER.info(f"Changing :latest tag to include {tags_prefix=}")
+        LOGGER.info(f"Adding {tags_prefix}latest tag")
         docker["tag", image, f"{owner}/{short_image_name}:{tags_prefix}latest"]()
-        docker["rmi", image]()
 
 
 if __name__ == "__main__":

tagging/write_manifest.py

@@ -73,7 +73,7 @@ def get_file_prefix() -> str:
     return "amd64" if machine == "x86_64" else "aarch64"
 
 
-def write_manifests(
+def write_manifest(
     short_image_name: str,
     owner: str,
     hist_line_dir: Path,
@@ -82,14 +82,14 @@ def write_manifests(
     LOGGER.info(f"Creating manifests for image: {short_image_name}")
     taggers, manifests = get_taggers_and_manifests(short_image_name)
 
-    tags_prefix = get_tags_prefix()
+    image = f"{owner}/{short_image_name}:latest"
-    image = f"{owner}/{short_image_name}:{tags_prefix}latest"
 
     file_prefix = get_file_prefix()
     commit_hash_tag = GitHelper.commit_hash_tag()
     filename = f"{file_prefix}-{short_image_name}-{commit_hash_tag}"
 
     with DockerRunner(image) as container:
+        tags_prefix = get_tags_prefix()
         all_tags = [tags_prefix + tagger.tag_value(container) for tagger in taggers]
         write_build_history_line(
             short_image_name, owner, hist_line_dir, filename, all_tags
@@ -125,6 +125,6 @@ if __name__ == "__main__":
 
     LOGGER.info(f"Current build timestamp: {BUILD_TIMESTAMP}")
 
-    write_manifests(
+    write_manifest(
         args.short_image_name, args.owner, args.hist_line_dir, args.manifest_dir
     )