hazza/docker-stacks
1
0
Fork 0
mirror of https://github.com/jupyter/docker-stacks.git synced 2025-10-08 18:44:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge tags and create multi-platform images (#1789)

Browse Source 
* Merge tags and create multi-platform images

* Add new files :)

* Provide images

* Fix

* Fix

* Use the same naming style as python for the images

* Fix docker image rmi command

* Use plumbum.FG

* Fix

* Add missing path

* Few merging fixes

* Add pushing manifests

* Add missing path

* Add arch to apply tags

* Use platform word

* Remove unused step

* Merge tags for tensorflow as well

* Do not use print

* Make merge_tags work when image doesn't exist

* Add logs

* Add icon

* Add tensorflow-notebook workaround

* Unify platform usage

* Remove unused function

* Fix

* Update docs
This commit is contained in:
Ayaz Salikhov
2022-09-21 09:58:57 +04:00
committed by GitHub
parent 494fc9c977
commit a85bfad8ca
16 changed files with 376 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/actions/create-dev-env/action.yml vendored
View File

@@ -17,7 +17,7 @@ runs:
uses: actions/setup-python@v4
with:
python-version: 3.x
if: ${{ inputs.platform == 'amd64' }}
if: ${{ inputs.platform == 'x86_64' }}
- name: Install Dev Dependencies 📦
run: |

32
.github/actions/download-manifests/action.yml vendored
View File

@@ -26,7 +26,7 @@ runs:
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: base-notebook-amd64-history_line
name: base-notebook-x86_64-history_line
path: ${{ inputs.histLineDir }}
- name: Download artifact 📥
uses: actions/download-artifact@v3
@@ -36,7 +36,7 @@ runs:
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: minimal-notebook-amd64-history_line
name: minimal-notebook-x86_64-history_line
path: ${{ inputs.histLineDir }}
- name: Download artifact 📥
uses: actions/download-artifact@v3
@@ -46,7 +46,7 @@ runs:
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: scipy-notebook-amd64-history_line
name: scipy-notebook-x86_64-history_line
path: ${{ inputs.histLineDir }}
- name: Download artifact 📥
uses: actions/download-artifact@v3
@@ -56,12 +56,12 @@ runs:
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: r-notebook-amd64-history_line
name: r-notebook-x86_64-history_line
path: ${{ inputs.histLineDir }}
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: tensorflow-notebook-amd64-history_line
name: tensorflow-notebook-x86_64-history_line
path: ${{ inputs.histLineDir }}
- name: Download artifact 📥
uses: actions/download-artifact@v3
@@ -71,7 +71,7 @@ runs:
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: datascience-notebook-amd64-history_line
name: datascience-notebook-x86_64-history_line
path: ${{ inputs.histLineDir }}
- name: Download artifact 📥
uses: actions/download-artifact@v3
@@ -81,7 +81,7 @@ runs:
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: pyspark-notebook-amd64-history_line
name: pyspark-notebook-x86_64-history_line
path: ${{ inputs.histLineDir }}
- name: Download artifact 📥
uses: actions/download-artifact@v3
@@ -91,7 +91,7 @@ runs:
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: all-spark-notebook-amd64-history_line
name: all-spark-notebook-x86_64-history_line
path: ${{ inputs.histLineDir }}
- name: Download artifact 📥
@@ -102,7 +102,7 @@ runs:
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: base-notebook-amd64-manifest
name: base-notebook-x86_64-manifest
path: ${{ inputs.manifestDir }}
- name: Download artifact 📥
uses: actions/download-artifact@v3
@@ -112,7 +112,7 @@ runs:
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: minimal-notebook-amd64-manifest
name: minimal-notebook-x86_64-manifest
path: ${{ inputs.manifestDir }}
- name: Download artifact 📥
uses: actions/download-artifact@v3
@@ -122,7 +122,7 @@ runs:
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: scipy-notebook-amd64-manifest
name: scipy-notebook-x86_64-manifest
path: ${{ inputs.manifestDir }}
- name: Download artifact 📥
uses: actions/download-artifact@v3
@@ -132,12 +132,12 @@ runs:
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: r-notebook-amd64-manifest
name: r-notebook-x86_64-manifest
path: ${{ inputs.manifestDir }}
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: tensorflow-notebook-amd64-manifest
name: tensorflow-notebook-x86_64-manifest
path: ${{ inputs.manifestDir }}
- name: Download artifact 📥
uses: actions/download-artifact@v3
@@ -147,7 +147,7 @@ runs:
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: datascience-notebook-amd64-manifest
name: datascience-notebook-x86_64-manifest
path: ${{ inputs.manifestDir }}
- name: Download artifact 📥
uses: actions/download-artifact@v3
@@ -157,7 +157,7 @@ runs:
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: pyspark-notebook-amd64-manifest
name: pyspark-notebook-x86_64-manifest
path: ${{ inputs.manifestDir }}
- name: Download artifact 📥
uses: actions/download-artifact@v3
@@ -167,5 +167,5 @@ runs:
- name: Download artifact 📥
uses: actions/download-artifact@v3
with:
name: all-spark-notebook-amd64-manifest
name: all-spark-notebook-x86_64-manifest
path: ${{ inputs.manifestDir }}

39
.github/workflows/docker-build-test-upload.yml vendored
View File

@@ -1,4 +1,4 @@
name: Download parent image, build new one, test it and upload to GitHub artifacts
name: Download parent image, build new one, test and upload it and tags and manifests to GitHub artifacts
env:
OWNER: ${{ github.repository_owner }}
@@ -35,9 +35,13 @@ jobs:
platform: ${{ inputs.platform }}
# Self-hosted runners share a state (whole VM) between runs
- name: Reset docker state 🗑️
if: ${{ inputs.platform != 'amd64' }}
run: docker system prune --all --force
- name: Reset docker state and cleanup artifacts 🗑️
if: ${{ inputs.platform != 'x86_64' }}
run: |
docker system prune --all --force
rm -rf /tmp/tags/
rm -rf /tmp/hist_lines/
rm -rf /tmp/manifests/
shell: bash
- name: Load parent built image to Docker 📥
@@ -59,6 +63,33 @@ jobs:
run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --owner ${{ env.OWNER }}
shell: bash
- name: Write tags file 🏷
run: |
python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/tags/ --owner ${{ env.OWNER }}
shell: bash
- name: Upload tags file 💾
uses: actions/upload-artifact@v3
with:
name: ${{ inputs.image }}-${{ inputs.platform }}-tags
path: /tmp/tags/${{ inputs.platform }}-${{ inputs.image }}.txt
retention-days: 3
- name: Write manifest and build history file 🏷
run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/ --owner ${{ env.OWNER }}
shell: bash
- name: Upload manifest file 💾
uses: actions/upload-artifact@v3
with:
name: ${{ inputs.image }}-${{ inputs.platform }}-manifest
path: /tmp/manifests/${{ inputs.platform }}-${{ inputs.image }}-*.md
retention-days: 3
- name: Upload build history line 💾
uses: actions/upload-artifact@v3
with:
name: ${{ inputs.image }}-${{ inputs.platform }}-history_line
path: /tmp/hist_lines/${{ inputs.platform }}-${{ inputs.image }}-*.txt
retention-days: 3
- name: Save image as a tar for later use 💾
run: docker save ${{ env.OWNER }}/${{ inputs.image }} -o /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
shell: bash

63
.github/workflows/docker-merge-tags.yml vendored Normal file
View File

@@ -0,0 +1,63 @@
name: Download Docker images from GitHub artifacts and merge tags
env:
OWNER: ${{ github.repository_owner }}
on:
workflow_call:
inputs:
images:
description: Stringified JSON object listing image names
required: true
type: string
secrets:
DOCKERHUB_USERNAME:
required: true
DOCKERHUB_TOKEN:
required: true
jobs:
tag-push:
runs-on: ubuntu-latest
strategy:
matrix:
image: ${{ fromJson(inputs.images) }}
steps:
- name: Checkout Repo ⚡️
uses: actions/checkout@v3
- name: Create dev environment 📦
uses: ./.github/actions/create-dev-env
with:
platform: x86_64
- name: Download x86_64 tags file 📥
uses: actions/download-artifact@v3
with:
name: ${{ matrix.image }}-x86_64-tags
path: /tmp/tags/
- name: Download aarch64 tags file 📥
if: matrix.image != 'tensorflow-notebook'
uses: actions/download-artifact@v3
with:
name: ${{ matrix.image }}-aarch64-tags
path: /tmp/tags/
- name: Create an empty aarch64 tags file for tensorflow-notebook 💩
if: matrix.image == 'tensorflow-notebook'
run: touch /tmp/tags/aarch64-tensorflow-notebook.txt
shell: bash
- name: Login to Docker Hub 🔐
if: github.ref == 'refs/heads/main' || github.event_name == 'schedule'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # dependabot updates to latest release
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Merge tags for the images 🔀
if: github.ref == 'refs/heads/main' || github.event_name == 'schedule'
run: python3 -m tagging.merge_tags --short-image-name ${{ matrix.image }} --tags-dir /tmp/tags/
shell: bash

53
.github/workflows/docker-tag-manifest-push.yml vendored
View File

@@ -14,10 +14,6 @@ on:
description: Image platform
required: true
type: string
runsOn:
description: GitHub Actions Runner image
required: true
type: string
secrets:
DOCKERHUB_USERNAME:
required: true
@@ -26,7 +22,7 @@ on:
jobs:
tag-push:
runs-on: ${{ inputs.runsOn }}
runs-on: ubuntu-latest
strategy:
matrix:
@@ -38,50 +34,13 @@ jobs:
- name: Create dev environment 📦
uses: ./.github/actions/create-dev-env
with:
platform: ${{ inputs.platform }}
# Self-hosted runners share a state (whole VM) between runs
- name: Reset docker state and cleanup artifacts 🗑️
if: ${{ inputs.platform != 'amd64' }}
run: |
docker system prune --all --force
rm -rf /tmp/hist_lines/
rm -rf /tmp/manifests/
shell: bash
platform: x86_64
- name: Load image to Docker 📥
uses: ./.github/actions/load-image
with:
image: ${{ matrix.image }}
platform: ${{ inputs.platform }}
- name: Create tags 🏷
run: |
python3 -m tagging.tag_image --short-image-name ${{ matrix.image }} --owner ${{ env.OWNER }}
docker image ls -a
shell: bash
- name: Write manifest and build history file 🏷
run: python3 -m tagging.write_manifest --short-image-name ${{ matrix.image }} --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/ --owner ${{ env.OWNER }}
shell: bash
- name: Upload manifest file 💾
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.image }}-${{ inputs.platform }}-manifest
path: /tmp/manifests/${{ inputs.platform }}-${{ matrix.image }}-*.md
retention-days: 3
- name: Upload build history line 💾
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.image }}-${{ inputs.platform }}-history_line
path: /tmp/hist_lines/${{ inputs.platform }}-${{ matrix.image }}-*.txt
retention-days: 3
- name: Remove aarch64 latest tag 🗑️
if: ${{ inputs.platform != 'amd64' }}
run: docker rmi ${{ env.OWNER }}/${{ matrix.image }}
shell: bash
- name: Login to Docker Hub 🔐
if: github.ref == 'refs/heads/main' || github.event_name == 'schedule'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # dependabot updates to latest release
@@ -89,6 +48,14 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Download tags file 📥
uses: actions/download-artifact@v3
with:
name: ${{ matrix.image }}-${{ inputs.platform }}-tags
path: /tmp/tags/
- name: Apply tags to the loaded image 🏷
run: python3 -m tagging.apply_tags --short-image-name ${{ matrix.image }} --tags-dir /tmp/tags/ --platform ${{ inputs.platform }} --owner ${{ env.OWNER }}
- name: Push Images to Docker Hub 📤
if: github.ref == 'refs/heads/main' || github.event_name == 'schedule'
run: docker push --all-tags ${{ env.OWNER }}/${{ matrix.image }}

2
.github/workflows/docker-wiki-update.yml vendored
View File

@@ -15,7 +15,7 @@ jobs:
- name: Create dev environment 📦
uses: ./.github/actions/create-dev-env
with:
platform: amd64
platform: x86_64
- name: Download all manifests and history lines 📥
uses: ./.github/actions/download-manifests

91
.github/workflows/docker.yml vendored
View File

@@ -10,6 +10,7 @@ on:
# We use local reusable workflows to make architecture clean an simple
# https://docs.github.com/en/actions/using-workflows/reusing-workflows
- ".github/workflows/docker-build-test-upload.yml"
- ".github/workflows/docker-merge-tags.yml"
- ".github/workflows/docker-tag-manifest-push.yml"
- ".github/workflows/docker-wiki-update.yml"
@@ -37,6 +38,7 @@ on:
paths:
- ".github/workflows/docker.yml"
- ".github/workflows/docker-build-test-upload.yml"
- ".github/workflows/docker-merge-tags.yml"
- ".github/workflows/docker-tag-manifest-push.yml"
- ".github/workflows/docker-wiki-update.yml"
@@ -73,12 +75,12 @@ jobs:
platform: aarch64
runsOn: ARM64
amd64-base:
x86_64-base:
uses: ./.github/workflows/docker-build-test-upload.yml
with:
parentImage: ""
image: base-notebook
platform: amd64
platform: x86_64
runsOn: ubuntu-latest
aarch64-minimal:
@@ -90,13 +92,13 @@ jobs:
platform: aarch64
runsOn: ARM64
amd64-minimal:
needs: [amd64-base]
x86_64-minimal:
needs: [x86_64-base]
uses: ./.github/workflows/docker-build-test-upload.yml
with:
parentImage: base-notebook
image: minimal-notebook
platform: amd64
platform: x86_64
runsOn: ubuntu-latest
aarch64-scipy:
@@ -108,13 +110,13 @@ jobs:
platform: aarch64
runsOn: ARM64
amd64-scipy:
needs: [amd64-minimal]
x86_64-scipy:
needs: [x86_64-minimal]
uses: ./.github/workflows/docker-build-test-upload.yml
with:
parentImage: minimal-notebook
image: scipy-notebook
platform: amd64
platform: x86_64
runsOn: ubuntu-latest
aarch64-r:
@@ -126,22 +128,22 @@ jobs:
platform: aarch64
runsOn: ARM64
amd64-r:
needs: [amd64-minimal]
x86_64-r:
needs: [x86_64-minimal]
uses: ./.github/workflows/docker-build-test-upload.yml
with:
parentImage: minimal-notebook
image: r-notebook
platform: amd64
platform: x86_64
runsOn: ubuntu-latest
amd64-tensorflow:
needs: [amd64-scipy]
x86_64-tensorflow:
needs: [x86_64-scipy]
uses: ./.github/workflows/docker-build-test-upload.yml
with:
parentImage: scipy-notebook
image: tensorflow-notebook
platform: amd64
platform: x86_64
runsOn: ubuntu-latest
aarch64-datascience:
@@ -153,13 +155,13 @@ jobs:
platform: aarch64
runsOn: ARM64
amd64-datascience:
needs: [amd64-scipy]
x86_64-datascience:
needs: [x86_64-scipy]
uses: ./.github/workflows/docker-build-test-upload.yml
with:
parentImage: scipy-notebook
image: datascience-notebook
platform: amd64
platform: x86_64
runsOn: ubuntu-latest
aarch64-pyspark:
@@ -171,13 +173,13 @@ jobs:
platform: aarch64
runsOn: ARM64
amd64-pyspark:
needs: [amd64-scipy]
x86_64-pyspark:
needs: [x86_64-scipy]
uses: ./.github/workflows/docker-build-test-upload.yml
with:
parentImage: scipy-notebook
image: pyspark-notebook
platform: amd64
platform: x86_64
runsOn: ubuntu-latest
aarch64-all-spark:
@@ -189,13 +191,13 @@ jobs:
platform: aarch64
runsOn: ARM64
amd64-all-spark:
needs: [amd64-pyspark]
x86_64-all-spark:
needs: [x86_64-pyspark]
uses: ./.github/workflows/docker-build-test-upload.yml
with:
parentImage: pyspark-notebook
image: all-spark-notebook
platform: amd64
platform: x86_64
runsOn: ubuntu-latest
aarch64-images-tag-push:
@@ -215,7 +217,6 @@ jobs:
uses: ./.github/workflows/docker-tag-manifest-push.yml
with:
platform: aarch64
runsOn: ARM64
# https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations
# The strategy property is not supported in any job that calls a reusable workflow
# Using the workaround: https://github.community/t/reusable-workflow-with-strategy-matrix/205676/2
@@ -230,25 +231,43 @@ jobs:
"all-spark-notebook"
]
amd64-images-tag-push:
x86_64-images-tag-push:
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
needs:
[
amd64-base,
amd64-minimal,
amd64-scipy,
amd64-r,
amd64-tensorflow,
amd64-datascience,
amd64-pyspark,
amd64-all-spark,
x86_64-base,
x86_64-minimal,
x86_64-scipy,
x86_64-r,
x86_64-tensorflow,
x86_64-datascience,
x86_64-pyspark,
x86_64-all-spark,
]
uses: ./.github/workflows/docker-tag-manifest-push.yml
with:
platform: amd64
runsOn: ubuntu-latest
platform: x86_64
images: >-
[
"base-notebook",
"minimal-notebook",
"scipy-notebook",
"r-notebook",
"tensorflow-notebook",
"datascience-notebook",
"pyspark-notebook",
"all-spark-notebook"
]
merge-tags:
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
needs: [aarch64-images-tag-push, x86_64-images-tag-push]
uses: ./.github/workflows/docker-merge-tags.yml
with:
images: >-
[
"base-notebook",
@@ -264,5 +283,5 @@ jobs:
wiki-update:
permissions:
contents: write
needs: [aarch64-images-tag-push, amd64-images-tag-push]
needs: [aarch64-images-tag-push, x86_64-images-tag-push]
uses: ./.github/workflows/docker-wiki-update.yml

6
README.md
View File

@@ -115,6 +115,6 @@ This change is tracked in the issue [#1217](https://github.com/jupyter/docker-st
## CPU Architectures
- We publish containers for both `amd64` (`x86_64`) and `aarch64` platforms, except for `tensorflow-notebook`, which only supports `amd64` for now
- We do not create multi-platform images
- Instead, all `arm64` images have _aarch64-_ tag prefix, for example `jupyter/base-notebook:aarch64-python-3.10.5`
- We publish containers for both `x86_64` and `aarch64` platforms, except for `tensorflow-notebook`, which only supports `x86_64` for now
- Single-platform images have either `aarch64` or `x86_64` tag prefixes, for example `jupyter/base-notebook:aarch64-python-3.10.5`
- Starting from `2022-09-21`, we create multi-platform images

2
base-notebook/Dockerfile
View File

@@ -32,7 +32,7 @@ RUN apt-get update --yes && \
fonts-liberation \
locales \
# - pandoc is used to convert notebooks to html files
# it's not present in arm64 ubuntu image, so we install it here
# it's not present in aarch64 ubuntu image, so we install it here
pandoc \
# - run-one - a wrapper script that runs no more
# than one unique instance of some command with a unique set of arguments,

2
docs/using/selecting.md
View File

@@ -208,7 +208,7 @@ Whenever a docker image is pushed to the container registry, it is tagged with:
```{warning}
- Tags before `2022-07-05` were sometimes incorrect. Please, do not rely on them.
- All `arm64` images have _aarch64-_ tag prefix, for example `aarch64-python-3.10.5`.
- Single-platform images have either `aarch64` or `x86_64` tag prefixes, for example `jupyter/base-notebook:aarch64-python-3.10.5`
```
For stability and reproducibility, you should either reference a date formatted

68
tagging/apply_tags.py Executable file
View File

@@ -0,0 +1,68 @@
#!/usr/bin/env python3
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import argparse
import logging
from pathlib import Path
import plumbum
docker = plumbum.local["docker"]
LOGGER = logging.getLogger(__name__)
def apply_tags(
short_image_name: str,
owner: str,
tags_dir: Path,
platform: str,
) -> None:
"""
Tags <owner>/<short_image_name>:latest with the tags
reported by all taggers for the given image.
"""
LOGGER.info(f"Tagging image: {short_image_name}")
image = f"{owner}/{short_image_name}:latest"
filename = f"{platform}-{short_image_name}.txt"
tags = (tags_dir / filename).read_text().splitlines()
for tag in tags:
LOGGER.info(f"Applying tag: {tag}")
docker["tag", image, tag] & plumbum.FG
LOGGER.info("Removing latest tag from the image")
docker["image", "rmi", image] & plumbum.FG
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
"--short-image-name",
required=True,
help="Short image name to apply tags for",
)
arg_parser.add_argument(
"--tags-dir",
required=True,
type=Path,
help="Directory with saved tags file",
)
arg_parser.add_argument(
"--platform",
required=True,
type=str,
choices=["x86_64", "aarch64"],
help="Image platform",
)
arg_parser.add_argument(
"--owner",
required=True,
help="Owner of the image",
)
args = arg_parser.parse_args()
apply_tags(args.short_image_name, args.owner, args.tags_dir, args.platform)

14
tagging/get_platform.py Normal file
View File

@@ -0,0 +1,14 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import platform
ALL_PLATFORMS = {"x86_64", "aarch64"}
def get_platform() -> str:
machine = platform.machine()
return {
"aarch64": "aarch64",
"arm64": "aarch64", # To support local building on aarch64 Macs
"x86_64": "x86_64",
}[machine]

8
tagging/get_tags_prefix.py
View File

@@ -1,8 +0,0 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import platform
def get_tags_prefix() -> str:
machine = platform.machine()
return "" if machine == "x86_64" else f"{machine}-"

79
tagging/merge_tags.py Executable file
View File

@@ -0,0 +1,79 @@
#!/usr/bin/env python3
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import argparse
import logging
from pathlib import Path
import plumbum
from tagging.get_platform import ALL_PLATFORMS
docker = plumbum.local["docker"]
LOGGER = logging.getLogger(__name__)
def merge_tags(
short_image_name: str,
tags_dir: Path,
) -> None:
"""
Merge tags for x86_64 and aarch64 images when possible.
"""
LOGGER.info(f"Merging tags for image: {short_image_name}")
all_tags: set[str] = set()
for platform in ALL_PLATFORMS:
filename = f"{platform}-{short_image_name}.txt"
tags = (tags_dir / filename).read_text().splitlines()
all_tags.update(tag.replace(platform + "-", "") for tag in tags)
LOGGER.info(f"Got tags: {all_tags}")
for tag in all_tags:
LOGGER.info(f"Trying to merge tag: {tag}")
existing_images = []
for platform in ALL_PLATFORMS:
image_with_platform = tag.replace(":", f":{platform}-")
LOGGER.info(f"Trying to pull: {image_with_platform}")
try:
docker["pull", image_with_platform] & plumbum.FG
existing_images.append(image_with_platform)
LOGGER.info("Pull success")
except plumbum.ProcessExecutionError:
LOGGER.info(
"Pull failed, image with this tag and platform doesn't exist"
)
LOGGER.info(f"Found images: {existing_images}")
(
docker[
"manifest",
"create",
tag,
][existing_images]
& plumbum.FG
)
docker["manifest", "push", tag] & plumbum.FG
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
"--short-image-name",
required=True,
help="Short image name to apply tags for",
)
arg_parser.add_argument(
"--tags-dir",
required=True,
type=Path,
help="Directory with saved tags file",
)
args = arg_parser.parse_args()
merge_tags(args.short_image_name, args.tags_dir)

16
tagging/write_manifest.py
View File

@@ -4,14 +4,13 @@
import argparse
import datetime
import logging
import platform
from pathlib import Path
from docker.models.containers import Container
from tagging.docker_runner import DockerRunner
from tagging.get_platform import get_platform
from tagging.get_taggers_and_manifests import get_taggers_and_manifests
from tagging.get_tags_prefix import get_tags_prefix
from tagging.git_helper import GitHelper
from tagging.manifests import ManifestHeader, ManifestInterface
@@ -68,11 +67,6 @@ def write_manifest_file(
(manifest_dir / f"{filename}.md").write_text(markdown_content)
def get_file_prefix() -> str:
machine = platform.machine()
return "amd64" if machine == "x86_64" else "aarch64"
def write_manifest(
short_image_name: str,
owner: str,
@@ -84,13 +78,15 @@ def write_manifest(
image = f"{owner}/{short_image_name}:latest"
file_prefix = get_file_prefix()
file_prefix = get_platform()
commit_hash_tag = GitHelper.commit_hash_tag()
filename = f"{file_prefix}-{short_image_name}-{commit_hash_tag}"
with DockerRunner(image) as container:
tags_prefix = get_tags_prefix()
all_tags = [tags_prefix + tagger.tag_value(container) for tagger in taggers]
tags_prefix = get_platform()
all_tags = [
tags_prefix + "-" + tagger.tag_value(container) for tagger in taggers
]
write_build_history_line(
short_image_name, owner, hist_line_dir, filename, all_tags
)

41
tagging/tag_image.py → tagging/write_tags_file.py
View File

@@ -3,42 +3,41 @@
# Distributed under the terms of the Modified BSD License.
import argparse
import logging
import plumbum
from pathlib import Path
from tagging.docker_runner import DockerRunner
from tagging.get_platform import get_platform
from tagging.get_taggers_and_manifests import get_taggers_and_manifests
from tagging.get_tags_prefix import get_tags_prefix
docker = plumbum.local["docker"]
LOGGER = logging.getLogger(__name__)
def tag_image(short_image_name: str, owner: str) -> None:
def write_tags_file(
short_image_name: str,
owner: str,
tags_dir: Path,
) -> None:
"""
Tags <owner>/<short_image_name>:latest with the tags reported by all taggers
for the given image.
Writes tags file for the image <owner>/<short_image_name>:latest
"""
LOGGER.info(f"Tagging image: {short_image_name}")
taggers, _ = get_taggers_and_manifests(short_image_name)
image = f"{owner}/{short_image_name}:latest"
tags_prefix = get_tags_prefix()
tags_prefix = get_platform()
filename = f"{tags_prefix}-{short_image_name}.txt"
tags = [f"{owner}/{short_image_name}:{tags_prefix}-latest"]
with DockerRunner(image) as container:
for tagger in taggers:
tagger_name = tagger.__class__.__name__
tag_value = tagger.tag_value(container)
LOGGER.info(
f"Applying tag, tagger_name: {tagger_name} tag_value: {tag_value}"
f"Calculated tag, tagger_name: {tagger_name} tag_value: {tag_value}"
)
docker[
"tag", image, f"{owner}/{short_image_name}:{tags_prefix}{tag_value}"
]()
if tags_prefix != "":
LOGGER.info(f"Adding {tags_prefix}latest tag")
docker["tag", image, f"{owner}/{short_image_name}:{tags_prefix}latest"]()
tags.append(f"{owner}/{short_image_name}:{tags_prefix}-{tag_value}")
tags_dir.mkdir(parents=True, exist_ok=True)
(tags_dir / filename).write_text("\n".join(tags))
if __name__ == "__main__":
@@ -48,7 +47,13 @@ if __name__ == "__main__":
arg_parser.add_argument(
"--short-image-name",
required=True,
help="Short image name to apply tags for",
help="Short image name to write tags for",
)
arg_parser.add_argument(
"--tags-dir",
required=True,
type=Path,
help="Directory to save tags file",
)
arg_parser.add_argument(
"--owner",
@@ -57,4 +62,4 @@ if __name__ == "__main__":
)
args = arg_parser.parse_args()
tag_image(args.short_image_name, args.owner)
write_tags_file(args.short_image_name, args.owner, args.tags_dir)