diff --git a/.github/actions/create-dev-env/action.yml b/.github/actions/create-dev-env/action.yml index ea774d5a..cdf7130e 100644 --- a/.github/actions/create-dev-env/action.yml +++ b/.github/actions/create-dev-env/action.yml @@ -17,7 +17,7 @@ runs: uses: actions/setup-python@v4 with: python-version: 3.x - if: ${{ inputs.platform == 'amd64' }} + if: ${{ inputs.platform == 'x86_64' }} - name: Install Dev Dependencies 📦 run: | diff --git a/.github/actions/download-manifests/action.yml b/.github/actions/download-manifests/action.yml index 25ab386b..79ac0682 100644 --- a/.github/actions/download-manifests/action.yml +++ b/.github/actions/download-manifests/action.yml @@ -26,7 +26,7 @@ runs: - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: base-notebook-amd64-history_line + name: base-notebook-x86_64-history_line path: ${{ inputs.histLineDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 @@ -36,7 +36,7 @@ runs: - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: minimal-notebook-amd64-history_line + name: minimal-notebook-x86_64-history_line path: ${{ inputs.histLineDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 @@ -46,7 +46,7 @@ runs: - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: scipy-notebook-amd64-history_line + name: scipy-notebook-x86_64-history_line path: ${{ inputs.histLineDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 @@ -56,12 +56,12 @@ runs: - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: r-notebook-amd64-history_line + name: r-notebook-x86_64-history_line path: ${{ inputs.histLineDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: tensorflow-notebook-amd64-history_line + name: tensorflow-notebook-x86_64-history_line path: ${{ inputs.histLineDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 @@ -71,7 +71,7 @@ runs: - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: datascience-notebook-amd64-history_line + name: datascience-notebook-x86_64-history_line path: ${{ inputs.histLineDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 @@ -81,7 +81,7 @@ runs: - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: pyspark-notebook-amd64-history_line + name: pyspark-notebook-x86_64-history_line path: ${{ inputs.histLineDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 @@ -91,7 +91,7 @@ runs: - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: all-spark-notebook-amd64-history_line + name: all-spark-notebook-x86_64-history_line path: ${{ inputs.histLineDir }} - name: Download artifact 📥 @@ -102,7 +102,7 @@ runs: - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: base-notebook-amd64-manifest + name: base-notebook-x86_64-manifest path: ${{ inputs.manifestDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 @@ -112,7 +112,7 @@ runs: - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: minimal-notebook-amd64-manifest + name: minimal-notebook-x86_64-manifest path: ${{ inputs.manifestDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 @@ -122,7 +122,7 @@ runs: - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: scipy-notebook-amd64-manifest + name: scipy-notebook-x86_64-manifest path: ${{ inputs.manifestDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 @@ -132,12 +132,12 @@ runs: - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: r-notebook-amd64-manifest + name: r-notebook-x86_64-manifest path: ${{ inputs.manifestDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: tensorflow-notebook-amd64-manifest + name: tensorflow-notebook-x86_64-manifest path: ${{ inputs.manifestDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 @@ -147,7 +147,7 @@ runs: - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: datascience-notebook-amd64-manifest + name: datascience-notebook-x86_64-manifest path: ${{ inputs.manifestDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 @@ -157,7 +157,7 @@ runs: - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: pyspark-notebook-amd64-manifest + name: pyspark-notebook-x86_64-manifest path: ${{ inputs.manifestDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 @@ -167,5 +167,5 @@ runs: - name: Download artifact 📥 uses: actions/download-artifact@v3 with: - name: all-spark-notebook-amd64-manifest + name: all-spark-notebook-x86_64-manifest path: ${{ inputs.manifestDir }} diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml index a96059cd..33f1ce6d 100644 --- a/.github/workflows/docker-build-test-upload.yml +++ b/.github/workflows/docker-build-test-upload.yml @@ -1,4 +1,4 @@ -name: Download parent image, build new one, test it and upload to GitHub artifacts +name: Download parent image, build new one, test and upload it and tags and manifests to GitHub artifacts env: OWNER: ${{ github.repository_owner }} @@ -35,9 +35,13 @@ jobs: platform: ${{ inputs.platform }} # Self-hosted runners share a state (whole VM) between runs - - name: Reset docker state 🗑️ - if: ${{ inputs.platform != 'amd64' }} - run: docker system prune --all --force + - name: Reset docker state and cleanup artifacts 🗑️ + if: ${{ inputs.platform != 'x86_64' }} + run: | + docker system prune --all --force + rm -rf /tmp/tags/ + rm -rf /tmp/hist_lines/ + rm -rf /tmp/manifests/ shell: bash - name: Load parent built image to Docker 📥 @@ -59,6 +63,33 @@ jobs: run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --owner ${{ env.OWNER }} shell: bash + - name: Write tags file 🏷 + run: | + python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/tags/ --owner ${{ env.OWNER }} + shell: bash + - name: Upload tags file 💾 + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.image }}-${{ inputs.platform }}-tags + path: /tmp/tags/${{ inputs.platform }}-${{ inputs.image }}.txt + retention-days: 3 + + - name: Write manifest and build history file 🏷 + run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/ --owner ${{ env.OWNER }} + shell: bash + - name: Upload manifest file 💾 + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.image }}-${{ inputs.platform }}-manifest + path: /tmp/manifests/${{ inputs.platform }}-${{ inputs.image }}-*.md + retention-days: 3 + - name: Upload build history line 💾 + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.image }}-${{ inputs.platform }}-history_line + path: /tmp/hist_lines/${{ inputs.platform }}-${{ inputs.image }}-*.txt + retention-days: 3 + - name: Save image as a tar for later use 💾 run: docker save ${{ env.OWNER }}/${{ inputs.image }} -o /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar shell: bash diff --git a/.github/workflows/docker-merge-tags.yml b/.github/workflows/docker-merge-tags.yml new file mode 100644 index 00000000..113d8095 --- /dev/null +++ b/.github/workflows/docker-merge-tags.yml @@ -0,0 +1,63 @@ +name: Download Docker images from GitHub artifacts and merge tags + +env: + OWNER: ${{ github.repository_owner }} + +on: + workflow_call: + inputs: + images: + description: Stringified JSON object listing image names + required: true + type: string + secrets: + DOCKERHUB_USERNAME: + required: true + DOCKERHUB_TOKEN: + required: true + +jobs: + tag-push: + runs-on: ubuntu-latest + + strategy: + matrix: + image: ${{ fromJson(inputs.images) }} + + steps: + - name: Checkout Repo ⚡️ + uses: actions/checkout@v3 + - name: Create dev environment 📦 + uses: ./.github/actions/create-dev-env + with: + platform: x86_64 + + - name: Download x86_64 tags file 📥 + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.image }}-x86_64-tags + path: /tmp/tags/ + + - name: Download aarch64 tags file 📥 + if: matrix.image != 'tensorflow-notebook' + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.image }}-aarch64-tags + path: /tmp/tags/ + + - name: Create an empty aarch64 tags file for tensorflow-notebook 💩 + if: matrix.image == 'tensorflow-notebook' + run: touch /tmp/tags/aarch64-tensorflow-notebook.txt + shell: bash + + - name: Login to Docker Hub 🔐 + if: github.ref == 'refs/heads/main' || github.event_name == 'schedule' + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # dependabot updates to latest release + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Merge tags for the images 🔀 + if: github.ref == 'refs/heads/main' || github.event_name == 'schedule' + run: python3 -m tagging.merge_tags --short-image-name ${{ matrix.image }} --tags-dir /tmp/tags/ + shell: bash diff --git a/.github/workflows/docker-tag-manifest-push.yml b/.github/workflows/docker-tag-manifest-push.yml index a7d676c6..e58afced 100644 --- a/.github/workflows/docker-tag-manifest-push.yml +++ b/.github/workflows/docker-tag-manifest-push.yml @@ -14,10 +14,6 @@ on: description: Image platform required: true type: string - runsOn: - description: GitHub Actions Runner image - required: true - type: string secrets: DOCKERHUB_USERNAME: required: true @@ -26,7 +22,7 @@ on: jobs: tag-push: - runs-on: ${{ inputs.runsOn }} + runs-on: ubuntu-latest strategy: matrix: @@ -38,50 +34,13 @@ jobs: - name: Create dev environment 📦 uses: ./.github/actions/create-dev-env with: - platform: ${{ inputs.platform }} - - # Self-hosted runners share a state (whole VM) between runs - - name: Reset docker state and cleanup artifacts 🗑️ - if: ${{ inputs.platform != 'amd64' }} - run: | - docker system prune --all --force - rm -rf /tmp/hist_lines/ - rm -rf /tmp/manifests/ - shell: bash - + platform: x86_64 - name: Load image to Docker 📥 uses: ./.github/actions/load-image with: image: ${{ matrix.image }} platform: ${{ inputs.platform }} - - name: Create tags 🏷 - run: | - python3 -m tagging.tag_image --short-image-name ${{ matrix.image }} --owner ${{ env.OWNER }} - docker image ls -a - shell: bash - - - name: Write manifest and build history file 🏷 - run: python3 -m tagging.write_manifest --short-image-name ${{ matrix.image }} --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/ --owner ${{ env.OWNER }} - shell: bash - - name: Upload manifest file 💾 - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.image }}-${{ inputs.platform }}-manifest - path: /tmp/manifests/${{ inputs.platform }}-${{ matrix.image }}-*.md - retention-days: 3 - - name: Upload build history line 💾 - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.image }}-${{ inputs.platform }}-history_line - path: /tmp/hist_lines/${{ inputs.platform }}-${{ matrix.image }}-*.txt - retention-days: 3 - - - name: Remove aarch64 latest tag 🗑️ - if: ${{ inputs.platform != 'amd64' }} - run: docker rmi ${{ env.OWNER }}/${{ matrix.image }} - shell: bash - - name: Login to Docker Hub 🔐 if: github.ref == 'refs/heads/main' || github.event_name == 'schedule' uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # dependabot updates to latest release @@ -89,6 +48,14 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Download tags file 📥 + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.image }}-${{ inputs.platform }}-tags + path: /tmp/tags/ + - name: Apply tags to the loaded image 🏷 + run: python3 -m tagging.apply_tags --short-image-name ${{ matrix.image }} --tags-dir /tmp/tags/ --platform ${{ inputs.platform }} --owner ${{ env.OWNER }} + - name: Push Images to Docker Hub 📤 if: github.ref == 'refs/heads/main' || github.event_name == 'schedule' run: docker push --all-tags ${{ env.OWNER }}/${{ matrix.image }} diff --git a/.github/workflows/docker-wiki-update.yml b/.github/workflows/docker-wiki-update.yml index 77563248..cee794c3 100644 --- a/.github/workflows/docker-wiki-update.yml +++ b/.github/workflows/docker-wiki-update.yml @@ -15,7 +15,7 @@ jobs: - name: Create dev environment 📦 uses: ./.github/actions/create-dev-env with: - platform: amd64 + platform: x86_64 - name: Download all manifests and history lines 📥 uses: ./.github/actions/download-manifests diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2b6daf1e..87e66b5d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -10,6 +10,7 @@ on: # We use local reusable workflows to make architecture clean an simple # https://docs.github.com/en/actions/using-workflows/reusing-workflows - ".github/workflows/docker-build-test-upload.yml" + - ".github/workflows/docker-merge-tags.yml" - ".github/workflows/docker-tag-manifest-push.yml" - ".github/workflows/docker-wiki-update.yml" @@ -37,6 +38,7 @@ on: paths: - ".github/workflows/docker.yml" - ".github/workflows/docker-build-test-upload.yml" + - ".github/workflows/docker-merge-tags.yml" - ".github/workflows/docker-tag-manifest-push.yml" - ".github/workflows/docker-wiki-update.yml" @@ -73,12 +75,12 @@ jobs: platform: aarch64 runsOn: ARM64 - amd64-base: + x86_64-base: uses: ./.github/workflows/docker-build-test-upload.yml with: parentImage: "" image: base-notebook - platform: amd64 + platform: x86_64 runsOn: ubuntu-latest aarch64-minimal: @@ -90,13 +92,13 @@ jobs: platform: aarch64 runsOn: ARM64 - amd64-minimal: - needs: [amd64-base] + x86_64-minimal: + needs: [x86_64-base] uses: ./.github/workflows/docker-build-test-upload.yml with: parentImage: base-notebook image: minimal-notebook - platform: amd64 + platform: x86_64 runsOn: ubuntu-latest aarch64-scipy: @@ -108,13 +110,13 @@ jobs: platform: aarch64 runsOn: ARM64 - amd64-scipy: - needs: [amd64-minimal] + x86_64-scipy: + needs: [x86_64-minimal] uses: ./.github/workflows/docker-build-test-upload.yml with: parentImage: minimal-notebook image: scipy-notebook - platform: amd64 + platform: x86_64 runsOn: ubuntu-latest aarch64-r: @@ -126,22 +128,22 @@ jobs: platform: aarch64 runsOn: ARM64 - amd64-r: - needs: [amd64-minimal] + x86_64-r: + needs: [x86_64-minimal] uses: ./.github/workflows/docker-build-test-upload.yml with: parentImage: minimal-notebook image: r-notebook - platform: amd64 + platform: x86_64 runsOn: ubuntu-latest - amd64-tensorflow: - needs: [amd64-scipy] + x86_64-tensorflow: + needs: [x86_64-scipy] uses: ./.github/workflows/docker-build-test-upload.yml with: parentImage: scipy-notebook image: tensorflow-notebook - platform: amd64 + platform: x86_64 runsOn: ubuntu-latest aarch64-datascience: @@ -153,13 +155,13 @@ jobs: platform: aarch64 runsOn: ARM64 - amd64-datascience: - needs: [amd64-scipy] + x86_64-datascience: + needs: [x86_64-scipy] uses: ./.github/workflows/docker-build-test-upload.yml with: parentImage: scipy-notebook image: datascience-notebook - platform: amd64 + platform: x86_64 runsOn: ubuntu-latest aarch64-pyspark: @@ -171,13 +173,13 @@ jobs: platform: aarch64 runsOn: ARM64 - amd64-pyspark: - needs: [amd64-scipy] + x86_64-pyspark: + needs: [x86_64-scipy] uses: ./.github/workflows/docker-build-test-upload.yml with: parentImage: scipy-notebook image: pyspark-notebook - platform: amd64 + platform: x86_64 runsOn: ubuntu-latest aarch64-all-spark: @@ -189,13 +191,13 @@ jobs: platform: aarch64 runsOn: ARM64 - amd64-all-spark: - needs: [amd64-pyspark] + x86_64-all-spark: + needs: [x86_64-pyspark] uses: ./.github/workflows/docker-build-test-upload.yml with: parentImage: pyspark-notebook image: all-spark-notebook - platform: amd64 + platform: x86_64 runsOn: ubuntu-latest aarch64-images-tag-push: @@ -215,7 +217,6 @@ jobs: uses: ./.github/workflows/docker-tag-manifest-push.yml with: platform: aarch64 - runsOn: ARM64 # https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations # The strategy property is not supported in any job that calls a reusable workflow # Using the workaround: https://github.community/t/reusable-workflow-with-strategy-matrix/205676/2 @@ -230,25 +231,43 @@ jobs: "all-spark-notebook" ] - amd64-images-tag-push: + x86_64-images-tag-push: secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} needs: [ - amd64-base, - amd64-minimal, - amd64-scipy, - amd64-r, - amd64-tensorflow, - amd64-datascience, - amd64-pyspark, - amd64-all-spark, + x86_64-base, + x86_64-minimal, + x86_64-scipy, + x86_64-r, + x86_64-tensorflow, + x86_64-datascience, + x86_64-pyspark, + x86_64-all-spark, ] uses: ./.github/workflows/docker-tag-manifest-push.yml with: - platform: amd64 - runsOn: ubuntu-latest + platform: x86_64 + images: >- + [ + "base-notebook", + "minimal-notebook", + "scipy-notebook", + "r-notebook", + "tensorflow-notebook", + "datascience-notebook", + "pyspark-notebook", + "all-spark-notebook" + ] + + merge-tags: + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + needs: [aarch64-images-tag-push, x86_64-images-tag-push] + uses: ./.github/workflows/docker-merge-tags.yml + with: images: >- [ "base-notebook", @@ -264,5 +283,5 @@ jobs: wiki-update: permissions: contents: write - needs: [aarch64-images-tag-push, amd64-images-tag-push] + needs: [aarch64-images-tag-push, x86_64-images-tag-push] uses: ./.github/workflows/docker-wiki-update.yml diff --git a/README.md b/README.md index fa17e414..1b437064 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,6 @@ This change is tracked in the issue [#1217](https://github.com/jupyter/docker-st ## CPU Architectures -- We publish containers for both `amd64` (`x86_64`) and `aarch64` platforms, except for `tensorflow-notebook`, which only supports `amd64` for now -- We do not create multi-platform images -- Instead, all `arm64` images have _aarch64-_ tag prefix, for example `jupyter/base-notebook:aarch64-python-3.10.5` +- We publish containers for both `x86_64` and `aarch64` platforms, except for `tensorflow-notebook`, which only supports `x86_64` for now +- Single-platform images have either `aarch64` or `x86_64` tag prefixes, for example `jupyter/base-notebook:aarch64-python-3.10.5` +- Starting from `2022-09-21`, we create multi-platform images diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 8f6e8044..6c0e89b5 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -32,7 +32,7 @@ RUN apt-get update --yes && \ fonts-liberation \ locales \ # - pandoc is used to convert notebooks to html files - # it's not present in arm64 ubuntu image, so we install it here + # it's not present in aarch64 ubuntu image, so we install it here pandoc \ # - run-one - a wrapper script that runs no more # than one unique instance of some command with a unique set of arguments, diff --git a/docs/using/selecting.md b/docs/using/selecting.md index 5b3a9c72..eaa9ea10 100644 --- a/docs/using/selecting.md +++ b/docs/using/selecting.md @@ -208,7 +208,7 @@ Whenever a docker image is pushed to the container registry, it is tagged with: ```{warning} - Tags before `2022-07-05` were sometimes incorrect. Please, do not rely on them. -- All `arm64` images have _aarch64-_ tag prefix, for example `aarch64-python-3.10.5`. +- Single-platform images have either `aarch64` or `x86_64` tag prefixes, for example `jupyter/base-notebook:aarch64-python-3.10.5` ``` For stability and reproducibility, you should either reference a date formatted diff --git a/tagging/apply_tags.py b/tagging/apply_tags.py new file mode 100755 index 00000000..04148216 --- /dev/null +++ b/tagging/apply_tags.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import argparse +import logging +from pathlib import Path + +import plumbum + +docker = plumbum.local["docker"] + +LOGGER = logging.getLogger(__name__) + + +def apply_tags( + short_image_name: str, + owner: str, + tags_dir: Path, + platform: str, +) -> None: + """ + Tags /:latest with the tags + reported by all taggers for the given image. + """ + LOGGER.info(f"Tagging image: {short_image_name}") + + image = f"{owner}/{short_image_name}:latest" + filename = f"{platform}-{short_image_name}.txt" + tags = (tags_dir / filename).read_text().splitlines() + + for tag in tags: + LOGGER.info(f"Applying tag: {tag}") + docker["tag", image, tag] & plumbum.FG + + LOGGER.info("Removing latest tag from the image") + docker["image", "rmi", image] & plumbum.FG + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + "--short-image-name", + required=True, + help="Short image name to apply tags for", + ) + arg_parser.add_argument( + "--tags-dir", + required=True, + type=Path, + help="Directory with saved tags file", + ) + arg_parser.add_argument( + "--platform", + required=True, + type=str, + choices=["x86_64", "aarch64"], + help="Image platform", + ) + arg_parser.add_argument( + "--owner", + required=True, + help="Owner of the image", + ) + args = arg_parser.parse_args() + + apply_tags(args.short_image_name, args.owner, args.tags_dir, args.platform) diff --git a/tagging/get_platform.py b/tagging/get_platform.py new file mode 100644 index 00000000..139e3697 --- /dev/null +++ b/tagging/get_platform.py @@ -0,0 +1,14 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import platform + +ALL_PLATFORMS = {"x86_64", "aarch64"} + + +def get_platform() -> str: + machine = platform.machine() + return { + "aarch64": "aarch64", + "arm64": "aarch64", # To support local building on aarch64 Macs + "x86_64": "x86_64", + }[machine] diff --git a/tagging/get_tags_prefix.py b/tagging/get_tags_prefix.py deleted file mode 100644 index beb8507d..00000000 --- a/tagging/get_tags_prefix.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import platform - - -def get_tags_prefix() -> str: - machine = platform.machine() - return "" if machine == "x86_64" else f"{machine}-" diff --git a/tagging/merge_tags.py b/tagging/merge_tags.py new file mode 100755 index 00000000..a5c37174 --- /dev/null +++ b/tagging/merge_tags.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import argparse +import logging +from pathlib import Path + +import plumbum + +from tagging.get_platform import ALL_PLATFORMS + +docker = plumbum.local["docker"] + +LOGGER = logging.getLogger(__name__) + + +def merge_tags( + short_image_name: str, + tags_dir: Path, +) -> None: + """ + Merge tags for x86_64 and aarch64 images when possible. + """ + LOGGER.info(f"Merging tags for image: {short_image_name}") + + all_tags: set[str] = set() + + for platform in ALL_PLATFORMS: + filename = f"{platform}-{short_image_name}.txt" + tags = (tags_dir / filename).read_text().splitlines() + all_tags.update(tag.replace(platform + "-", "") for tag in tags) + + LOGGER.info(f"Got tags: {all_tags}") + + for tag in all_tags: + LOGGER.info(f"Trying to merge tag: {tag}") + existing_images = [] + for platform in ALL_PLATFORMS: + image_with_platform = tag.replace(":", f":{platform}-") + LOGGER.info(f"Trying to pull: {image_with_platform}") + try: + docker["pull", image_with_platform] & plumbum.FG + existing_images.append(image_with_platform) + LOGGER.info("Pull success") + except plumbum.ProcessExecutionError: + LOGGER.info( + "Pull failed, image with this tag and platform doesn't exist" + ) + + LOGGER.info(f"Found images: {existing_images}") + ( + docker[ + "manifest", + "create", + tag, + ][existing_images] + & plumbum.FG + ) + docker["manifest", "push", tag] & plumbum.FG + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + "--short-image-name", + required=True, + help="Short image name to apply tags for", + ) + arg_parser.add_argument( + "--tags-dir", + required=True, + type=Path, + help="Directory with saved tags file", + ) + args = arg_parser.parse_args() + + merge_tags(args.short_image_name, args.tags_dir) diff --git a/tagging/write_manifest.py b/tagging/write_manifest.py index 06e890ce..f81e0213 100755 --- a/tagging/write_manifest.py +++ b/tagging/write_manifest.py @@ -4,14 +4,13 @@ import argparse import datetime import logging -import platform from pathlib import Path from docker.models.containers import Container from tagging.docker_runner import DockerRunner +from tagging.get_platform import get_platform from tagging.get_taggers_and_manifests import get_taggers_and_manifests -from tagging.get_tags_prefix import get_tags_prefix from tagging.git_helper import GitHelper from tagging.manifests import ManifestHeader, ManifestInterface @@ -68,11 +67,6 @@ def write_manifest_file( (manifest_dir / f"{filename}.md").write_text(markdown_content) -def get_file_prefix() -> str: - machine = platform.machine() - return "amd64" if machine == "x86_64" else "aarch64" - - def write_manifest( short_image_name: str, owner: str, @@ -84,13 +78,15 @@ def write_manifest( image = f"{owner}/{short_image_name}:latest" - file_prefix = get_file_prefix() + file_prefix = get_platform() commit_hash_tag = GitHelper.commit_hash_tag() filename = f"{file_prefix}-{short_image_name}-{commit_hash_tag}" with DockerRunner(image) as container: - tags_prefix = get_tags_prefix() - all_tags = [tags_prefix + tagger.tag_value(container) for tagger in taggers] + tags_prefix = get_platform() + all_tags = [ + tags_prefix + "-" + tagger.tag_value(container) for tagger in taggers + ] write_build_history_line( short_image_name, owner, hist_line_dir, filename, all_tags ) diff --git a/tagging/tag_image.py b/tagging/write_tags_file.py similarity index 53% rename from tagging/tag_image.py rename to tagging/write_tags_file.py index 1df2d84a..55cbfab7 100755 --- a/tagging/tag_image.py +++ b/tagging/write_tags_file.py @@ -3,42 +3,41 @@ # Distributed under the terms of the Modified BSD License. import argparse import logging - -import plumbum +from pathlib import Path from tagging.docker_runner import DockerRunner +from tagging.get_platform import get_platform from tagging.get_taggers_and_manifests import get_taggers_and_manifests -from tagging.get_tags_prefix import get_tags_prefix - -docker = plumbum.local["docker"] LOGGER = logging.getLogger(__name__) -def tag_image(short_image_name: str, owner: str) -> None: +def write_tags_file( + short_image_name: str, + owner: str, + tags_dir: Path, +) -> None: """ - Tags /:latest with the tags reported by all taggers - for the given image. + Writes tags file for the image /:latest """ LOGGER.info(f"Tagging image: {short_image_name}") taggers, _ = get_taggers_and_manifests(short_image_name) image = f"{owner}/{short_image_name}:latest" - tags_prefix = get_tags_prefix() + tags_prefix = get_platform() + filename = f"{tags_prefix}-{short_image_name}.txt" + tags = [f"{owner}/{short_image_name}:{tags_prefix}-latest"] with DockerRunner(image) as container: for tagger in taggers: tagger_name = tagger.__class__.__name__ tag_value = tagger.tag_value(container) LOGGER.info( - f"Applying tag, tagger_name: {tagger_name} tag_value: {tag_value}" + f"Calculated tag, tagger_name: {tagger_name} tag_value: {tag_value}" ) - docker[ - "tag", image, f"{owner}/{short_image_name}:{tags_prefix}{tag_value}" - ]() - if tags_prefix != "": - LOGGER.info(f"Adding {tags_prefix}latest tag") - docker["tag", image, f"{owner}/{short_image_name}:{tags_prefix}latest"]() + tags.append(f"{owner}/{short_image_name}:{tags_prefix}-{tag_value}") + tags_dir.mkdir(parents=True, exist_ok=True) + (tags_dir / filename).write_text("\n".join(tags)) if __name__ == "__main__": @@ -48,7 +47,13 @@ if __name__ == "__main__": arg_parser.add_argument( "--short-image-name", required=True, - help="Short image name to apply tags for", + help="Short image name to write tags for", + ) + arg_parser.add_argument( + "--tags-dir", + required=True, + type=Path, + help="Directory to save tags file", ) arg_parser.add_argument( "--owner", @@ -57,4 +62,4 @@ if __name__ == "__main__": ) args = arg_parser.parse_args() - tag_image(args.short_image_name, args.owner) + write_tags_file(args.short_image_name, args.owner, args.tags_dir)