Merge tags and create multi-platform images (#1789)

* Merge tags and create multi-platform images

* Add new files :)

* Provide images

* Fix

* Fix

* Use the same naming style as python for the images

* Fix docker image rmi command

* Use plumbum.FG

* Fix

* Add missing path

* Few merging fixes

* Add pushing manifests

* Add missing path

* Add arch to apply tags

* Use platform word

* Remove unused step

* Merge tags for tensorflow as well

* Do not use print

* Make merge_tags work when image doesn't exist

* Add logs

* Add icon

* Add tensorflow-notebook workaround

* Unify platform usage

* Remove unused function

* Fix

* Update docs

.github/actions/create-dev-env/action.yml

@@ -17,7 +17,7 @@ runs:
       uses: actions/setup-python@v4
       with:
         python-version: 3.x
-      if: ${{ inputs.platform == 'amd64' }}
+      if: ${{ inputs.platform == 'x86_64' }}
 
     - name: Install Dev Dependencies 📦
       run: |

.github/actions/download-manifests/action.yml

@@ -26,7 +26,7 @@ runs:
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: base-notebook-amd64-history_line
+        name: base-notebook-x86_64-history_line
         path: ${{ inputs.histLineDir }}
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
@@ -36,7 +36,7 @@ runs:
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: minimal-notebook-amd64-history_line
+        name: minimal-notebook-x86_64-history_line
         path: ${{ inputs.histLineDir }}
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
@@ -46,7 +46,7 @@ runs:
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: scipy-notebook-amd64-history_line
+        name: scipy-notebook-x86_64-history_line
         path: ${{ inputs.histLineDir }}
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
@@ -56,12 +56,12 @@ runs:
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: r-notebook-amd64-history_line
+        name: r-notebook-x86_64-history_line
         path: ${{ inputs.histLineDir }}
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: tensorflow-notebook-amd64-history_line
+        name: tensorflow-notebook-x86_64-history_line
         path: ${{ inputs.histLineDir }}
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
@@ -71,7 +71,7 @@ runs:
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: datascience-notebook-amd64-history_line
+        name: datascience-notebook-x86_64-history_line
         path: ${{ inputs.histLineDir }}
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
@@ -81,7 +81,7 @@ runs:
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: pyspark-notebook-amd64-history_line
+        name: pyspark-notebook-x86_64-history_line
         path: ${{ inputs.histLineDir }}
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
@@ -91,7 +91,7 @@ runs:
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: all-spark-notebook-amd64-history_line
+        name: all-spark-notebook-x86_64-history_line
         path: ${{ inputs.histLineDir }}
 
     - name: Download artifact 📥
@@ -102,7 +102,7 @@ runs:
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: base-notebook-amd64-manifest
+        name: base-notebook-x86_64-manifest
         path: ${{ inputs.manifestDir }}
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
@@ -112,7 +112,7 @@ runs:
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: minimal-notebook-amd64-manifest
+        name: minimal-notebook-x86_64-manifest
         path: ${{ inputs.manifestDir }}
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
@@ -122,7 +122,7 @@ runs:
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: scipy-notebook-amd64-manifest
+        name: scipy-notebook-x86_64-manifest
         path: ${{ inputs.manifestDir }}
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
@@ -132,12 +132,12 @@ runs:
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: r-notebook-amd64-manifest
+        name: r-notebook-x86_64-manifest
         path: ${{ inputs.manifestDir }}
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: tensorflow-notebook-amd64-manifest
+        name: tensorflow-notebook-x86_64-manifest
         path: ${{ inputs.manifestDir }}
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
@@ -147,7 +147,7 @@ runs:
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: datascience-notebook-amd64-manifest
+        name: datascience-notebook-x86_64-manifest
         path: ${{ inputs.manifestDir }}
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
@@ -157,7 +157,7 @@ runs:
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: pyspark-notebook-amd64-manifest
+        name: pyspark-notebook-x86_64-manifest
         path: ${{ inputs.manifestDir }}
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
@@ -167,5 +167,5 @@ runs:
     - name: Download artifact 📥
       uses: actions/download-artifact@v3
       with:
-        name: all-spark-notebook-amd64-manifest
+        name: all-spark-notebook-x86_64-manifest
         path: ${{ inputs.manifestDir }}

.github/workflows/docker-build-test-upload.yml

@@ -1,4 +1,4 @@
-name: Download parent image, build new one, test it and upload to GitHub artifacts
+name: Download parent image, build new one, test and upload it and tags and manifests to GitHub artifacts
 
 env:
   OWNER: ${{ github.repository_owner }}
@@ -35,9 +35,13 @@ jobs:
           platform: ${{ inputs.platform }}
 
       # Self-hosted runners share a state (whole VM) between runs
-      - name: Reset docker state 🗑️
+      - name: Reset docker state and cleanup artifacts 🗑️
-        if: ${{ inputs.platform != 'amd64' }}
+        if: ${{ inputs.platform != 'x86_64' }}
-        run: docker system prune --all --force
+        run: |
+          docker system prune --all --force
+          rm -rf /tmp/tags/
+          rm -rf /tmp/hist_lines/
+          rm -rf /tmp/manifests/
         shell: bash
 
       - name: Load parent built image to Docker 📥
@@ -59,6 +63,33 @@ jobs:
         run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --owner ${{ env.OWNER }}
         shell: bash
 
+      - name: Write tags file 🏷
+        run: |
+          python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/tags/ --owner ${{ env.OWNER }}
+        shell: bash
+      - name: Upload tags file 💾
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.image }}-${{ inputs.platform }}-tags
+          path: /tmp/tags/${{ inputs.platform }}-${{ inputs.image }}.txt
+          retention-days: 3
+
+      - name: Write manifest and build history file 🏷
+        run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/ --owner ${{ env.OWNER }}
+        shell: bash
+      - name: Upload manifest file 💾
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.image }}-${{ inputs.platform }}-manifest
+          path: /tmp/manifests/${{ inputs.platform }}-${{ inputs.image }}-*.md
+          retention-days: 3
+      - name: Upload build history line 💾
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.image }}-${{ inputs.platform }}-history_line
+          path: /tmp/hist_lines/${{ inputs.platform }}-${{ inputs.image }}-*.txt
+          retention-days: 3
+
       - name: Save image as a tar for later use 💾
         run: docker save ${{ env.OWNER }}/${{ inputs.image }} -o /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
         shell: bash

.github/workflows/docker-merge-tags.yml

@@ -0,0 +1,63 @@
+name: Download Docker images from GitHub artifacts and merge tags
+
+env:
+  OWNER: ${{ github.repository_owner }}
+
+on:
+  workflow_call:
+    inputs:
+      images:
+        description: Stringified JSON object listing image names
+        required: true
+        type: string
+    secrets:
+      DOCKERHUB_USERNAME:
+        required: true
+      DOCKERHUB_TOKEN:
+        required: true
+
+jobs:
+  tag-push:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        image: ${{ fromJson(inputs.images) }}
+
+    steps:
+      - name: Checkout Repo ⚡️
+        uses: actions/checkout@v3
+      - name: Create dev environment 📦
+        uses: ./.github/actions/create-dev-env
+        with:
+          platform: x86_64
+
+      - name: Download x86_64 tags file 📥
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.image }}-x86_64-tags
+          path: /tmp/tags/
+
+      - name: Download aarch64 tags file 📥
+        if: matrix.image != 'tensorflow-notebook'
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.image }}-aarch64-tags
+          path: /tmp/tags/
+
+      - name: Create an empty aarch64 tags file for tensorflow-notebook 💩
+        if: matrix.image == 'tensorflow-notebook'
+        run: touch /tmp/tags/aarch64-tensorflow-notebook.txt
+        shell: bash
+
+      - name: Login to Docker Hub 🔐
+        if: github.ref == 'refs/heads/main' || github.event_name == 'schedule'
+        uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # dependabot updates to latest release
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Merge tags for the images 🔀
+        if: github.ref == 'refs/heads/main' || github.event_name == 'schedule'
+        run: python3 -m tagging.merge_tags --short-image-name ${{ matrix.image }} --tags-dir /tmp/tags/
+        shell: bash

.github/workflows/docker-tag-manifest-push.yml

@@ -14,10 +14,6 @@ on:
         description: Image platform
         required: true
         type: string
-      runsOn:
-        description: GitHub Actions Runner image
-        required: true
-        type: string
     secrets:
       DOCKERHUB_USERNAME:
         required: true
@@ -26,7 +22,7 @@ on:
 
 jobs:
   tag-push:
-    runs-on: ${{ inputs.runsOn }}
+    runs-on: ubuntu-latest
 
     strategy:
       matrix:
@@ -38,50 +34,13 @@ jobs:
       - name: Create dev environment 📦
         uses: ./.github/actions/create-dev-env
         with:
-          platform: ${{ inputs.platform }}
+          platform: x86_64
-
-      # Self-hosted runners share a state (whole VM) between runs
-      - name: Reset docker state and cleanup artifacts 🗑️
-        if: ${{ inputs.platform != 'amd64' }}
-        run: |
-          docker system prune --all --force
-          rm -rf /tmp/hist_lines/
-          rm -rf /tmp/manifests/
-        shell: bash
-
       - name: Load image to Docker 📥
         uses: ./.github/actions/load-image
         with:
           image: ${{ matrix.image }}
           platform: ${{ inputs.platform }}
 
-      - name: Create tags 🏷
-        run: |
-          python3 -m tagging.tag_image --short-image-name ${{ matrix.image }} --owner ${{ env.OWNER }}
-          docker image ls -a
-        shell: bash
-
-      - name: Write manifest and build history file 🏷
-        run: python3 -m tagging.write_manifest --short-image-name ${{ matrix.image }} --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/ --owner ${{ env.OWNER }}
-        shell: bash
-      - name: Upload manifest file 💾
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.image }}-${{ inputs.platform }}-manifest
-          path: /tmp/manifests/${{ inputs.platform }}-${{ matrix.image }}-*.md
-          retention-days: 3
-      - name: Upload build history line 💾
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.image }}-${{ inputs.platform }}-history_line
-          path: /tmp/hist_lines/${{ inputs.platform }}-${{ matrix.image }}-*.txt
-          retention-days: 3
-
-      - name: Remove aarch64 latest tag 🗑️
-        if: ${{ inputs.platform != 'amd64' }}
-        run: docker rmi ${{ env.OWNER }}/${{ matrix.image }}
-        shell: bash
-
       - name: Login to Docker Hub 🔐
         if: github.ref == 'refs/heads/main' || github.event_name == 'schedule'
         uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # dependabot updates to latest release
@@ -89,6 +48,14 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Download tags file 📥
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.image }}-${{ inputs.platform }}-tags
+          path: /tmp/tags/
+      - name: Apply tags to the loaded image 🏷
+        run: python3 -m tagging.apply_tags --short-image-name ${{ matrix.image }} --tags-dir /tmp/tags/ --platform ${{ inputs.platform }} --owner ${{ env.OWNER }}
+
       - name: Push Images to Docker Hub 📤
         if: github.ref == 'refs/heads/main' || github.event_name == 'schedule'
         run: docker push --all-tags ${{ env.OWNER }}/${{ matrix.image }}

.github/workflows/docker-wiki-update.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Create dev environment 📦
         uses: ./.github/actions/create-dev-env
         with:
-          platform: amd64
+          platform: x86_64
 
       - name: Download all manifests and history lines 📥
         uses: ./.github/actions/download-manifests

.github/workflows/docker.yml

@@ -10,6 +10,7 @@ on:
       # We use local reusable workflows to make architecture clean an simple
       # https://docs.github.com/en/actions/using-workflows/reusing-workflows
       - ".github/workflows/docker-build-test-upload.yml"
+      - ".github/workflows/docker-merge-tags.yml"
       - ".github/workflows/docker-tag-manifest-push.yml"
       - ".github/workflows/docker-wiki-update.yml"
 
@@ -37,6 +38,7 @@ on:
     paths:
       - ".github/workflows/docker.yml"
       - ".github/workflows/docker-build-test-upload.yml"
+      - ".github/workflows/docker-merge-tags.yml"
       - ".github/workflows/docker-tag-manifest-push.yml"
       - ".github/workflows/docker-wiki-update.yml"
 
@@ -73,12 +75,12 @@ jobs:
       platform: aarch64
       runsOn: ARM64
 
-  amd64-base:
+  x86_64-base:
     uses: ./.github/workflows/docker-build-test-upload.yml
     with:
       parentImage: ""
       image: base-notebook
-      platform: amd64
+      platform: x86_64
       runsOn: ubuntu-latest
 
   aarch64-minimal:
@@ -90,13 +92,13 @@ jobs:
       platform: aarch64
       runsOn: ARM64
 
-  amd64-minimal:
+  x86_64-minimal:
-    needs: [amd64-base]
+    needs: [x86_64-base]
     uses: ./.github/workflows/docker-build-test-upload.yml
     with:
       parentImage: base-notebook
       image: minimal-notebook
-      platform: amd64
+      platform: x86_64
       runsOn: ubuntu-latest
 
   aarch64-scipy:
@@ -108,13 +110,13 @@ jobs:
       platform: aarch64
       runsOn: ARM64
 
-  amd64-scipy:
+  x86_64-scipy:
-    needs: [amd64-minimal]
+    needs: [x86_64-minimal]
     uses: ./.github/workflows/docker-build-test-upload.yml
     with:
       parentImage: minimal-notebook
       image: scipy-notebook
-      platform: amd64
+      platform: x86_64
       runsOn: ubuntu-latest
 
   aarch64-r:
@@ -126,22 +128,22 @@ jobs:
       platform: aarch64
       runsOn: ARM64
 
-  amd64-r:
+  x86_64-r:
-    needs: [amd64-minimal]
+    needs: [x86_64-minimal]
     uses: ./.github/workflows/docker-build-test-upload.yml
     with:
       parentImage: minimal-notebook
       image: r-notebook
-      platform: amd64
+      platform: x86_64
       runsOn: ubuntu-latest
 
-  amd64-tensorflow:
+  x86_64-tensorflow:
-    needs: [amd64-scipy]
+    needs: [x86_64-scipy]
     uses: ./.github/workflows/docker-build-test-upload.yml
     with:
       parentImage: scipy-notebook
       image: tensorflow-notebook
-      platform: amd64
+      platform: x86_64
       runsOn: ubuntu-latest
 
   aarch64-datascience:
@@ -153,13 +155,13 @@ jobs:
       platform: aarch64
       runsOn: ARM64
 
-  amd64-datascience:
+  x86_64-datascience:
-    needs: [amd64-scipy]
+    needs: [x86_64-scipy]
     uses: ./.github/workflows/docker-build-test-upload.yml
     with:
       parentImage: scipy-notebook
       image: datascience-notebook
-      platform: amd64
+      platform: x86_64
       runsOn: ubuntu-latest
 
   aarch64-pyspark:
@@ -171,13 +173,13 @@ jobs:
       platform: aarch64
       runsOn: ARM64
 
-  amd64-pyspark:
+  x86_64-pyspark:
-    needs: [amd64-scipy]
+    needs: [x86_64-scipy]
     uses: ./.github/workflows/docker-build-test-upload.yml
     with:
       parentImage: scipy-notebook
       image: pyspark-notebook
-      platform: amd64
+      platform: x86_64
       runsOn: ubuntu-latest
 
   aarch64-all-spark:
@@ -189,13 +191,13 @@ jobs:
       platform: aarch64
       runsOn: ARM64
 
-  amd64-all-spark:
+  x86_64-all-spark:
-    needs: [amd64-pyspark]
+    needs: [x86_64-pyspark]
     uses: ./.github/workflows/docker-build-test-upload.yml
     with:
       parentImage: pyspark-notebook
       image: all-spark-notebook
-      platform: amd64
+      platform: x86_64
       runsOn: ubuntu-latest
 
   aarch64-images-tag-push:
@@ -215,7 +217,6 @@ jobs:
     uses: ./.github/workflows/docker-tag-manifest-push.yml
     with:
       platform: aarch64
-      runsOn: ARM64
       # https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations
       # The strategy property is not supported in any job that calls a reusable workflow
       # Using the workaround: https://github.community/t/reusable-workflow-with-strategy-matrix/205676/2
@@ -230,25 +231,43 @@ jobs:
           "all-spark-notebook"
         ]
 
-  amd64-images-tag-push:
+  x86_64-images-tag-push:
     secrets:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
     needs:
       [
-        amd64-base,
+        x86_64-base,
-        amd64-minimal,
+        x86_64-minimal,
-        amd64-scipy,
+        x86_64-scipy,
-        amd64-r,
+        x86_64-r,
-        amd64-tensorflow,
+        x86_64-tensorflow,
-        amd64-datascience,
+        x86_64-datascience,
-        amd64-pyspark,
+        x86_64-pyspark,
-        amd64-all-spark,
+        x86_64-all-spark,
       ]
     uses: ./.github/workflows/docker-tag-manifest-push.yml
     with:
-      platform: amd64
+      platform: x86_64
-      runsOn: ubuntu-latest
+      images: >-
+        [
+          "base-notebook",
+          "minimal-notebook",
+          "scipy-notebook",
+          "r-notebook",
+          "tensorflow-notebook",
+          "datascience-notebook",
+          "pyspark-notebook",
+          "all-spark-notebook"
+        ]
+
+  merge-tags:
+    secrets:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+    needs: [aarch64-images-tag-push, x86_64-images-tag-push]
+    uses: ./.github/workflows/docker-merge-tags.yml
+    with:
       images: >-
         [
           "base-notebook",
@@ -264,5 +283,5 @@ jobs:
   wiki-update:
     permissions:
       contents: write
-    needs: [aarch64-images-tag-push, amd64-images-tag-push]
+    needs: [aarch64-images-tag-push, x86_64-images-tag-push]
     uses: ./.github/workflows/docker-wiki-update.yml

README.md

@@ -115,6 +115,6 @@ This change is tracked in the issue [#1217](https://github.com/jupyter/docker-st
 
 ## CPU Architectures
 
-- We publish containers for both `amd64` (`x86_64`) and `aarch64` platforms, except for `tensorflow-notebook`, which only supports `amd64` for now
+- We publish containers for both `x86_64` and `aarch64` platforms, except for `tensorflow-notebook`, which only supports `x86_64` for now
-- We do not create multi-platform images
+- Single-platform images have either `aarch64` or `x86_64` tag prefixes, for example `jupyter/base-notebook:aarch64-python-3.10.5`
-- Instead, all `arm64` images have _aarch64-_ tag prefix, for example `jupyter/base-notebook:aarch64-python-3.10.5`
+- Starting from `2022-09-21`, we create multi-platform images

base-notebook/Dockerfile

@@ -32,7 +32,7 @@ RUN apt-get update --yes && \
     fonts-liberation \
     locales \
     # - pandoc is used to convert notebooks to html files
-    #   it's not present in arm64 ubuntu image, so we install it here
+    #   it's not present in aarch64 ubuntu image, so we install it here
     pandoc \
     # - run-one - a wrapper script that runs no more
     #   than one unique  instance  of  some  command with a unique set of arguments,

docs/using/selecting.md

@@ -208,7 +208,7 @@ Whenever a docker image is pushed to the container registry, it is tagged with:
 
 ```{warning}
 - Tags before `2022-07-05` were sometimes incorrect. Please, do not rely on them.
-- All `arm64` images have _aarch64-_ tag prefix, for example `aarch64-python-3.10.5`.
+- Single-platform images have either `aarch64` or `x86_64` tag prefixes, for example `jupyter/base-notebook:aarch64-python-3.10.5`
 ```
 
 For stability and reproducibility, you should either reference a date formatted

tagging/apply_tags.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import argparse
+import logging
+from pathlib import Path
+
+import plumbum
+
+docker = plumbum.local["docker"]
+
+LOGGER = logging.getLogger(__name__)
+
+
+def apply_tags(
+    short_image_name: str,
+    owner: str,
+    tags_dir: Path,
+    platform: str,
+) -> None:
+    """
+    Tags <owner>/<short_image_name>:latest with the tags
+    reported by all taggers for the given image.
+    """
+    LOGGER.info(f"Tagging image: {short_image_name}")
+
+    image = f"{owner}/{short_image_name}:latest"
+    filename = f"{platform}-{short_image_name}.txt"
+    tags = (tags_dir / filename).read_text().splitlines()
+
+    for tag in tags:
+        LOGGER.info(f"Applying tag: {tag}")
+        docker["tag", image, tag] & plumbum.FG
+
+    LOGGER.info("Removing latest tag from the image")
+    docker["image", "rmi", image] & plumbum.FG
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument(
+        "--short-image-name",
+        required=True,
+        help="Short image name to apply tags for",
+    )
+    arg_parser.add_argument(
+        "--tags-dir",
+        required=True,
+        type=Path,
+        help="Directory with saved tags file",
+    )
+    arg_parser.add_argument(
+        "--platform",
+        required=True,
+        type=str,
+        choices=["x86_64", "aarch64"],
+        help="Image platform",
+    )
+    arg_parser.add_argument(
+        "--owner",
+        required=True,
+        help="Owner of the image",
+    )
+    args = arg_parser.parse_args()
+
+    apply_tags(args.short_image_name, args.owner, args.tags_dir, args.platform)

tagging/get_platform.py

@@ -0,0 +1,14 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import platform
+
+ALL_PLATFORMS = {"x86_64", "aarch64"}
+
+
+def get_platform() -> str:
+    machine = platform.machine()
+    return {
+        "aarch64": "aarch64",
+        "arm64": "aarch64",  # To support local building on aarch64 Macs
+        "x86_64": "x86_64",
+    }[machine]

tagging/get_tags_prefix.py

@@ -1,8 +0,0 @@
-# Copyright (c) Jupyter Development Team.
-# Distributed under the terms of the Modified BSD License.
-import platform
-
-
-def get_tags_prefix() -> str:
-    machine = platform.machine()
-    return "" if machine == "x86_64" else f"{machine}-"

tagging/merge_tags.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import argparse
+import logging
+from pathlib import Path
+
+import plumbum
+
+from tagging.get_platform import ALL_PLATFORMS
+
+docker = plumbum.local["docker"]
+
+LOGGER = logging.getLogger(__name__)
+
+
+def merge_tags(
+    short_image_name: str,
+    tags_dir: Path,
+) -> None:
+    """
+    Merge tags for x86_64 and aarch64 images when possible.
+    """
+    LOGGER.info(f"Merging tags for image: {short_image_name}")
+
+    all_tags: set[str] = set()
+
+    for platform in ALL_PLATFORMS:
+        filename = f"{platform}-{short_image_name}.txt"
+        tags = (tags_dir / filename).read_text().splitlines()
+        all_tags.update(tag.replace(platform + "-", "") for tag in tags)
+
+    LOGGER.info(f"Got tags: {all_tags}")
+
+    for tag in all_tags:
+        LOGGER.info(f"Trying to merge tag: {tag}")
+        existing_images = []
+        for platform in ALL_PLATFORMS:
+            image_with_platform = tag.replace(":", f":{platform}-")
+            LOGGER.info(f"Trying to pull: {image_with_platform}")
+            try:
+                docker["pull", image_with_platform] & plumbum.FG
+                existing_images.append(image_with_platform)
+                LOGGER.info("Pull success")
+            except plumbum.ProcessExecutionError:
+                LOGGER.info(
+                    "Pull failed, image with this tag and platform doesn't exist"
+                )
+
+        LOGGER.info(f"Found images: {existing_images}")
+        (
+            docker[
+                "manifest",
+                "create",
+                tag,
+            ][existing_images]
+            & plumbum.FG
+        )
+        docker["manifest", "push", tag] & plumbum.FG
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument(
+        "--short-image-name",
+        required=True,
+        help="Short image name to apply tags for",
+    )
+    arg_parser.add_argument(
+        "--tags-dir",
+        required=True,
+        type=Path,
+        help="Directory with saved tags file",
+    )
+    args = arg_parser.parse_args()
+
+    merge_tags(args.short_image_name, args.tags_dir)

tagging/write_manifest.py

@@ -4,14 +4,13 @@
 import argparse
 import datetime
 import logging
-import platform
 from pathlib import Path
 
 from docker.models.containers import Container
 
 from tagging.docker_runner import DockerRunner
+from tagging.get_platform import get_platform
 from tagging.get_taggers_and_manifests import get_taggers_and_manifests
-from tagging.get_tags_prefix import get_tags_prefix
 from tagging.git_helper import GitHelper
 from tagging.manifests import ManifestHeader, ManifestInterface
 
@@ -68,11 +67,6 @@ def write_manifest_file(
     (manifest_dir / f"{filename}.md").write_text(markdown_content)
 
 
-def get_file_prefix() -> str:
-    machine = platform.machine()
-    return "amd64" if machine == "x86_64" else "aarch64"
-
-
 def write_manifest(
     short_image_name: str,
     owner: str,
@@ -84,13 +78,15 @@ def write_manifest(
 
     image = f"{owner}/{short_image_name}:latest"
 
-    file_prefix = get_file_prefix()
+    file_prefix = get_platform()
     commit_hash_tag = GitHelper.commit_hash_tag()
     filename = f"{file_prefix}-{short_image_name}-{commit_hash_tag}"
 
     with DockerRunner(image) as container:
-        tags_prefix = get_tags_prefix()
+        tags_prefix = get_platform()
-        all_tags = [tags_prefix + tagger.tag_value(container) for tagger in taggers]
+        all_tags = [
+            tags_prefix + "-" + tagger.tag_value(container) for tagger in taggers
+        ]
         write_build_history_line(
             short_image_name, owner, hist_line_dir, filename, all_tags
         )

tagging/write_tags_file.py

@@ -3,42 +3,41 @@
 # Distributed under the terms of the Modified BSD License.
 import argparse
 import logging
-
+from pathlib import Path
-import plumbum
 
 from tagging.docker_runner import DockerRunner
+from tagging.get_platform import get_platform
 from tagging.get_taggers_and_manifests import get_taggers_and_manifests
-from tagging.get_tags_prefix import get_tags_prefix
-
-docker = plumbum.local["docker"]
 
 LOGGER = logging.getLogger(__name__)
 
 
-def tag_image(short_image_name: str, owner: str) -> None:
+def write_tags_file(
+    short_image_name: str,
+    owner: str,
+    tags_dir: Path,
+) -> None:
     """
-    Tags <owner>/<short_image_name>:latest with the tags reported by all taggers
+    Writes tags file for the image <owner>/<short_image_name>:latest
-    for the given image.
     """
     LOGGER.info(f"Tagging image: {short_image_name}")
     taggers, _ = get_taggers_and_manifests(short_image_name)
 
     image = f"{owner}/{short_image_name}:latest"
-    tags_prefix = get_tags_prefix()
+    tags_prefix = get_platform()
+    filename = f"{tags_prefix}-{short_image_name}.txt"
 
+    tags = [f"{owner}/{short_image_name}:{tags_prefix}-latest"]
     with DockerRunner(image) as container:
         for tagger in taggers:
             tagger_name = tagger.__class__.__name__
             tag_value = tagger.tag_value(container)
             LOGGER.info(
-                f"Applying tag, tagger_name: {tagger_name} tag_value: {tag_value}"
+                f"Calculated tag, tagger_name: {tagger_name} tag_value: {tag_value}"
             )
-            docker[
+            tags.append(f"{owner}/{short_image_name}:{tags_prefix}-{tag_value}")
-                "tag", image, f"{owner}/{short_image_name}:{tags_prefix}{tag_value}"
+    tags_dir.mkdir(parents=True, exist_ok=True)
-            ]()
+    (tags_dir / filename).write_text("\n".join(tags))
-    if tags_prefix != "":
-        LOGGER.info(f"Adding {tags_prefix}latest tag")
-        docker["tag", image, f"{owner}/{short_image_name}:{tags_prefix}latest"]()
 
 
 if __name__ == "__main__":
@@ -48,7 +47,13 @@ if __name__ == "__main__":
     arg_parser.add_argument(
         "--short-image-name",
         required=True,
-        help="Short image name to apply tags for",
+        help="Short image name to write tags for",
+    )
+    arg_parser.add_argument(
+        "--tags-dir",
+        required=True,
+        type=Path,
+        help="Directory to save tags file",
     )
     arg_parser.add_argument(
         "--owner",
@@ -57,4 +62,4 @@ if __name__ == "__main__":
     )
     args = arg_parser.parse_args()
 
-    tag_image(args.short_image_name, args.owner)
+    write_tags_file(args.short_image_name, args.owner, args.tags_dir)