Better tagging directory structure (#2228)

tagging/apps/apply_tags.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import logging
+from pathlib import Path
+
+import plumbum
+
+from tagging.apps.common_cli_arguments import common_arguments_parser
+from tagging.utils.get_platform import unify_aarch64
+from tagging.utils.get_prefix import get_file_prefix_for_platform
+
+docker = plumbum.local["docker"]
+
+LOGGER = logging.getLogger(__name__)
+
+
+def apply_tags(
+    *,
+    registry: str,
+    owner: str,
+    short_image_name: str,
+    variant: str,
+    platform: str,
+    tags_dir: Path,
+) -> None:
+    """
+    Tags <registry>/<owner>/<short_image_name>:latest with the tags reported by all taggers for this image
+    """
+    LOGGER.info(f"Tagging image: {short_image_name}")
+
+    file_prefix = get_file_prefix_for_platform(platform, variant)
+    image = f"{registry}/{owner}/{short_image_name}:latest"
+    filename = f"{file_prefix}-{short_image_name}.txt"
+    tags = (tags_dir / filename).read_text().splitlines()
+
+    for tag in tags:
+        LOGGER.info(f"Applying tag: {tag}")
+        docker["tag", image, tag] & plumbum.FG
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    arg_parser = common_arguments_parser()
+    arg_parser.add_argument(
+        "--platform",
+        required=True,
+        type=str,
+        choices=["x86_64", "aarch64", "arm64"],
+        help="Image platform",
+    )
+    arg_parser.add_argument(
+        "--tags-dir",
+        required=True,
+        type=Path,
+        help="Directory with saved tags file",
+    )
+    args = arg_parser.parse_args()
+    args.platform = unify_aarch64(args.platform)
+
+    apply_tags(**vars(args))

tagging/apps/common_cli_arguments.py

@@ -0,0 +1,39 @@
+import argparse
+
+
+def common_arguments_parser(
+    registry: bool = True,
+    owner: bool = True,
+    short_image_name: bool = True,
+    variant: bool = True,
+) -> argparse.ArgumentParser:
+    """Add common CLI arguments to parser"""
+
+    parser = argparse.ArgumentParser()
+    if registry:
+        parser.add_argument(
+            "--registry",
+            required=True,
+            choices=["docker.io", "quay.io"],
+            help="Image registry",
+        )
+    if owner:
+        parser.add_argument(
+            "--owner",
+            required=True,
+            help="Owner of the image",
+        )
+    if short_image_name:
+        parser.add_argument(
+            "--short-image-name",
+            required=True,
+            help="Short image name",
+        )
+    if variant:
+        parser.add_argument(
+            "--variant",
+            required=True,
+            help="Variant tag prefix",
+        )
+
+    return parser

tagging/apps/merge_tags.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import logging
+from pathlib import Path
+
+import plumbum
+
+from tagging.apps.common_cli_arguments import common_arguments_parser
+from tagging.utils.get_platform import ALL_PLATFORMS
+from tagging.utils.get_prefix import get_file_prefix_for_platform
+
+docker = plumbum.local["docker"]
+
+LOGGER = logging.getLogger(__name__)
+
+
+def merge_tags(
+    *,
+    short_image_name: str,
+    variant: str,
+    tags_dir: Path,
+) -> None:
+    """
+    Merge tags for x86_64 and aarch64 images when possible.
+    """
+    LOGGER.info(f"Merging tags for image: {short_image_name}")
+
+    all_tags: set[str] = set()
+
+    for platform in ALL_PLATFORMS:
+        file_prefix = get_file_prefix_for_platform(platform, variant)
+        filename = f"{file_prefix}-{short_image_name}.txt"
+        file_path = tags_dir / filename
+        if file_path.exists():
+            tags = file_path.read_text().splitlines()
+            all_tags.update(tag.replace(platform + "-", "") for tag in tags)
+
+    LOGGER.info(f"Got tags: {all_tags}")
+
+    for tag in all_tags:
+        LOGGER.info(f"Trying to merge tag: {tag}")
+        existing_images = []
+        for platform in ALL_PLATFORMS:
+            image_with_platform = tag.replace(":", f":{platform}-")
+            LOGGER.info(f"Trying to pull: {image_with_platform}")
+            try:
+                docker["pull", image_with_platform] & plumbum.FG
+                existing_images.append(image_with_platform)
+                LOGGER.info("Pull success")
+            except plumbum.ProcessExecutionError:
+                LOGGER.info(
+                    "Pull failed, image with this tag and platform doesn't exist"
+                )
+
+        LOGGER.info(f"Found images: {existing_images}")
+        docker["manifest", "create", tag][existing_images] & plumbum.FG
+        docker["manifest", "push", tag] & plumbum.FG
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    arg_parser = common_arguments_parser(registry=False, owner=False)
+    arg_parser.add_argument(
+        "--tags-dir",
+        required=True,
+        type=Path,
+        help="Directory with saved tags file",
+    )
+    args = arg_parser.parse_args()
+
+    merge_tags(**vars(args))

tagging/apps/write_manifest.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import datetime
+import logging
+from pathlib import Path
+
+from docker.models.containers import Container
+
+from tagging.apps.common_cli_arguments import common_arguments_parser
+from tagging.hierarchy.get_taggers_and_manifests import (
+    get_taggers_and_manifests,
+)
+from tagging.manifests.header import ManifestHeader
+from tagging.manifests.manifest_interface import ManifestInterface
+from tagging.utils.docker_runner import DockerRunner
+from tagging.utils.get_prefix import get_file_prefix, get_tag_prefix
+from tagging.utils.git_helper import GitHelper
+
+LOGGER = logging.getLogger(__name__)
+
+# We use a manifest creation timestamp, which happens right after a build
+BUILD_TIMESTAMP = datetime.datetime.now(datetime.UTC).isoformat()[:-13] + "Z"
+MARKDOWN_LINE_BREAK = "<br />"
+
+
+def write_build_history_line(
+    *,
+    registry: str,
+    owner: str,
+    short_image_name: str,
+    hist_lines_dir: Path,
+    filename: str,
+    all_tags: list[str],
+) -> None:
+    LOGGER.info("Appending build history line")
+
+    date_column = f"`{BUILD_TIMESTAMP}`"
+    image_column = MARKDOWN_LINE_BREAK.join(
+        f"`{registry}/{owner}/{short_image_name}:{tag_value}`" for tag_value in all_tags
+    )
+    commit_hash = GitHelper.commit_hash()
+    links_column = MARKDOWN_LINE_BREAK.join(
+        [
+            f"[Git diff](https://github.com/jupyter/docker-stacks/commit/{commit_hash})",
+            f"[Dockerfile](https://github.com/jupyter/docker-stacks/blob/{commit_hash}/images/{short_image_name}/Dockerfile)",
+            f"[Build manifest](./{filename})",
+        ]
+    )
+    build_history_line = f"| {date_column} | {image_column} | {links_column} |"
+    hist_lines_dir.mkdir(parents=True, exist_ok=True)
+    (hist_lines_dir / f"{filename}.txt").write_text(build_history_line)
+
+
+def write_manifest_file(
+    *,
+    registry: str,
+    owner: str,
+    short_image_name: str,
+    manifests_dir: Path,
+    filename: str,
+    manifests: list[ManifestInterface],
+    container: Container,
+) -> None:
+    manifest_names = [manifest.__class__.__name__ for manifest in manifests]
+    LOGGER.info(f"Using manifests: {manifest_names}")
+
+    markdown_pieces = [
+        ManifestHeader.create_header(short_image_name, registry, owner, BUILD_TIMESTAMP)
+    ] + [manifest.markdown_piece(container) for manifest in manifests]
+    markdown_content = "\n\n".join(markdown_pieces) + "\n"
+
+    manifests_dir.mkdir(parents=True, exist_ok=True)
+    (manifests_dir / f"{filename}.md").write_text(markdown_content)
+
+
+def write_manifest(
+    *,
+    registry: str,
+    owner: str,
+    short_image_name: str,
+    variant: str,
+    hist_lines_dir: Path,
+    manifests_dir: Path,
+) -> None:
+    LOGGER.info(f"Creating manifests for image: {short_image_name}")
+    taggers, manifests = get_taggers_and_manifests(short_image_name)
+
+    image = f"{registry}/{owner}/{short_image_name}:latest"
+
+    file_prefix = get_file_prefix(variant)
+    commit_hash_tag = GitHelper.commit_hash_tag()
+    filename = f"{file_prefix}-{short_image_name}-{commit_hash_tag}"
+
+    with DockerRunner(image) as container:
+        tags_prefix = get_tag_prefix(variant)
+        all_tags = [
+            tags_prefix + "-" + tagger.tag_value(container) for tagger in taggers
+        ]
+        write_build_history_line(
+            registry=registry,
+            owner=owner,
+            short_image_name=short_image_name,
+            hist_lines_dir=hist_lines_dir,
+            filename=filename,
+            all_tags=all_tags,
+        )
+        write_manifest_file(
+            registry=registry,
+            owner=owner,
+            short_image_name=short_image_name,
+            manifests_dir=manifests_dir,
+            filename=filename,
+            manifests=manifests,
+            container=container,
+        )
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    arg_parser = common_arguments_parser()
+    arg_parser.add_argument(
+        "--hist-lines-dir",
+        required=True,
+        type=Path,
+        help="Directory to save history line",
+    )
+    arg_parser.add_argument(
+        "--manifests-dir",
+        required=True,
+        type=Path,
+        help="Directory to save manifest file",
+    )
+    args = arg_parser.parse_args()
+
+    LOGGER.info(f"Current build timestamp: {BUILD_TIMESTAMP}")
+
+    write_manifest(**vars(args))

tagging/apps/write_tags_file.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import logging
+from pathlib import Path
+
+from tagging.apps.common_cli_arguments import common_arguments_parser
+from tagging.hierarchy.get_taggers_and_manifests import (
+    get_taggers_and_manifests,
+)
+from tagging.utils.docker_runner import DockerRunner
+from tagging.utils.get_prefix import get_file_prefix, get_tag_prefix
+
+LOGGER = logging.getLogger(__name__)
+
+
+def write_tags_file(
+    *,
+    registry: str,
+    owner: str,
+    short_image_name: str,
+    variant: str,
+    tags_dir: Path,
+) -> None:
+    """
+    Writes tags file for the image <registry>/<owner>/<short_image_name>:latest
+    """
+    LOGGER.info(f"Tagging image: {short_image_name}")
+    taggers, _ = get_taggers_and_manifests(short_image_name)
+
+    image = f"{registry}/{owner}/{short_image_name}:latest"
+    file_prefix = get_file_prefix(variant)
+    filename = f"{file_prefix}-{short_image_name}.txt"
+
+    tags_prefix = get_tag_prefix(variant)
+    tags = [f"{registry}/{owner}/{short_image_name}:{tags_prefix}-latest"]
+    with DockerRunner(image) as container:
+        for tagger in taggers:
+            tagger_name = tagger.__class__.__name__
+            tag_value = tagger.tag_value(container)
+            LOGGER.info(
+                f"Calculated tag, tagger_name: {tagger_name} tag_value: {tag_value}"
+            )
+            tags.append(
+                f"{registry}/{owner}/{short_image_name}:{tags_prefix}-{tag_value}"
+            )
+    tags_dir.mkdir(parents=True, exist_ok=True)
+    (tags_dir / filename).write_text("\n".join(tags))
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    arg_parser = common_arguments_parser()
+    arg_parser.add_argument(
+        "--tags-dir",
+        required=True,
+        type=Path,
+        help="Directory to save tags file",
+    )
+    args = arg_parser.parse_args()
+
+    write_tags_file(**vars(args))