diff --git a/.github/workflows/docker-tag-push.yml b/.github/workflows/docker-tag-manifest-push.yml similarity index 74% rename from .github/workflows/docker-tag-push.yml rename to .github/workflows/docker-tag-manifest-push.yml index 6a5375d5..fa4f18a5 100644 --- a/.github/workflows/docker-tag-push.yml +++ b/.github/workflows/docker-tag-manifest-push.yml @@ -40,6 +40,20 @@ jobs: run: python3 -m tagging.tag_image --short-image-name ${{ matrix.image }} shell: bash + - name: Write manifest files 🏷 + run: python3 -m tagging.write_manifests --short-image-name ${{ matrix.image }} + shell: bash + - name: Upload manifest file 💾 + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.image }}-${{ inputs.platform }}-manifest + path: /tmp/manifests/*.md + - name: Upload build history line 💾 + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.image }}-${{ inputs.platform }}-history_line + path: /tmp/build_history_lines/*.txt + - name: Login to Docker Hub 🔐 if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.event_name == 'schedule' uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # dependabot updates to latest release diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 340bbc60..70733958 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,7 +9,7 @@ on: - ".github/workflows/docker.yml" - ".github/workflows/docker-build-upload.yml" - ".github/workflows/docker-test.yml" - - ".github/workflows/docker-tag-push.yml" + - ".github/workflows/docker-tag-manifest-push.yml" - ".github/actions/create-dev-env/action.yml" - ".github/actions/load-image/action.yml" @@ -34,7 +34,7 @@ on: - ".github/workflows/docker.yml" - ".github/workflows/docker-build-upload.yml" - ".github/workflows/docker-test.yml" - - ".github/workflows/docker-tag-push.yml" + - ".github/workflows/docker-tag-manifest-push.yml" - ".github/actions/create-dev-env/action.yml" - ".github/actions/load-image/action.yml" @@ -94,7 +94,7 @@ jobs: amd64-images-tag-push: needs: [test-amd64-base, test-amd64-minimal] - uses: ./.github/workflows/docker-tag-push.yml + uses: ./.github/workflows/docker-tag-manifest-push.yml with: platform: amd64 runsOn: ubuntu-latest diff --git a/tagging/tag_image.py b/tagging/tag_image.py index 74b61215..629930bb 100755 --- a/tagging/tag_image.py +++ b/tagging/tag_image.py @@ -3,23 +3,18 @@ # Distributed under the terms of the Modified BSD License. import argparse import logging -import platform import plumbum from tagging.docker_runner import DockerRunner from tagging.get_taggers_and_manifests import get_taggers_and_manifests +from tagging.tags_prefix import get_tags_prefix docker = plumbum.local["docker"] LOGGER = logging.getLogger(__name__) -def get_tags_prefix() -> str: - machine = platform.machine() - return "" if machine == "x86_64" else f"{machine}-" - - def tag_image(short_image_name: str, owner: str, tags_prefix: str) -> None: """ Tags /:latest with the tags reported by all taggers @@ -62,6 +57,5 @@ if __name__ == "__main__": args = arg_parser.parse_args() tags_prefix = get_tags_prefix() - LOGGER.info(f"Using: {tags_prefix=}") tag_image(args.short_image_name, args.owner, tags_prefix) diff --git a/tagging/tags_prefix.py b/tagging/tags_prefix.py new file mode 100644 index 00000000..beb8507d --- /dev/null +++ b/tagging/tags_prefix.py @@ -0,0 +1,8 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import platform + + +def get_tags_prefix() -> str: + machine = platform.machine() + return "" if machine == "x86_64" else f"{machine}-" diff --git a/tagging/create_manifests.py b/tagging/write_manifests.py similarity index 64% rename from tagging/create_manifests.py rename to tagging/write_manifests.py index 3771db1e..1753fa6a 100755 --- a/tagging/create_manifests.py +++ b/tagging/write_manifests.py @@ -4,7 +4,7 @@ import argparse import datetime import logging -import os +from pathlib import Path from docker.models.containers import Container @@ -12,6 +12,7 @@ from tagging.docker_runner import DockerRunner from tagging.get_taggers_and_manifests import get_taggers_and_manifests from tagging.git_helper import GitHelper from tagging.manifests import ManifestHeader, ManifestInterface +from tagging.tags_prefix import get_tags_prefix LOGGER = logging.getLogger(__name__) @@ -20,10 +21,10 @@ BUILD_TIMESTAMP = datetime.datetime.utcnow().isoformat()[:-7] + "Z" MARKDOWN_LINE_BREAK = "
" -def append_build_history_line( +def write_build_history_line( short_image_name: str, owner: str, - wiki_path: str, + manifest_filename: str, all_tags: list[str], ) -> None: LOGGER.info("Appending build history line") @@ -33,60 +34,55 @@ def append_build_history_line( f"`{owner}/{short_image_name}:{tag_value}`" for tag_value in all_tags ) commit_hash = GitHelper.commit_hash() - commit_hash_tag = GitHelper.commit_hash_tag() links_column = MARKDOWN_LINE_BREAK.join( [ f"[Git diff](https://github.com/jupyter/docker-stacks/commit/{commit_hash})", f"[Dockerfile](https://github.com/jupyter/docker-stacks/blob/{commit_hash}/{short_image_name}/Dockerfile)", - f"[Build manifest](./{short_image_name}-{commit_hash_tag})", + f"[Build manifest](./{manifest_filename.removesuffix('.md')})", ] ) build_history_line = "|".join([date_column, image_column, links_column]) + "|" - - home_wiki_file = os.path.join(wiki_path, "Home.md") - with open(home_wiki_file) as f: - file = f.read() - TABLE_BEGINNING = "|-|-|-|\n" - file = file.replace(TABLE_BEGINNING, TABLE_BEGINNING + build_history_line + "\n") - with open(home_wiki_file, "w") as f: - f.write(file) + build_history_filename = manifest_filename.replace(".md", ".txt") + Path(f"/tmp/build_history_lines/{build_history_filename}").write_text( + build_history_line + ) -def create_manifest_file( +def write_manifest_file( short_image_name: str, owner: str, - wiki_path: str, + manifest_filename: str, manifests: list[ManifestInterface], container: Container, ) -> None: manifest_names = [manifest.__class__.__name__ for manifest in manifests] LOGGER.info(f"Using manifests: {manifest_names}") - commit_hash_tag = GitHelper.commit_hash_tag() - manifest_file = os.path.join( - wiki_path, - f"manifests/{short_image_name}-{commit_hash_tag}.md", - ) - markdown_pieces = [ ManifestHeader.create_header(short_image_name, owner, BUILD_TIMESTAMP) ] + [manifest.markdown_piece(container) for manifest in manifests] markdown_content = "\n\n".join(markdown_pieces) + "\n" - with open(manifest_file, "w") as f: - f.write(markdown_content) + Path(f"/tmp/manifests/{manifest_filename}").write_text(markdown_content) -def create_manifests(short_image_name: str, owner: str, wiki_path: str) -> None: +def write_manifests(short_image_name: str, owner: str) -> None: LOGGER.info(f"Creating manifests for image: {short_image_name}") taggers, manifests = get_taggers_and_manifests(short_image_name) image = f"{owner}/{short_image_name}:latest" + tags_prefix = get_tags_prefix() + commit_hash_tag = GitHelper.commit_hash_tag() + filename = f"{tags_prefix}{short_image_name}-{commit_hash_tag}.md" + manifest_filename = f"{filename}.md" + with DockerRunner(image) as container: - all_tags = [tagger.tag_value(container) for tagger in taggers] - append_build_history_line(short_image_name, owner, wiki_path, all_tags) - create_manifest_file(short_image_name, owner, wiki_path, manifests, container) + all_tags = [tags_prefix + tagger.tag_value(container) for tagger in taggers] + write_build_history_line(short_image_name, owner, all_tags) + write_manifest_file( + short_image_name, owner, manifest_filename, manifests, container + ) if __name__ == "__main__": @@ -96,12 +92,11 @@ if __name__ == "__main__": arg_parser.add_argument( "--short-image-name", required=True, - help="Short image name to apply tags for", + help="Short image name to create manifests for", ) - arg_parser.add_argument("--owner", required=True, help="Owner of the image") - arg_parser.add_argument("--wiki-path", required=True, help="Path to the wiki pages") + arg_parser.add_argument("--owner", default="jupyter", help="Owner of the image") args = arg_parser.parse_args() LOGGER.info(f"Current build timestamp: {BUILD_TIMESTAMP}") - create_manifests(args.short_image_name, args.owner, args.wiki_path) + write_manifests(args.short_image_name, args.owner)