mirror of
https://github.com/jupyter/docker-stacks.git
synced 2025-10-07 18:14:05 +00:00
Add tagging config to pass params easier (#2234)
* Add tagging config to pass params easier * Shorter function signatures
This commit is contained in:
@@ -81,7 +81,7 @@ jobs:
|
|||||||
python3 -m tagging.apps.write_tags_file
|
python3 -m tagging.apps.write_tags_file
|
||||||
--registry ${{ env.REGISTRY }}
|
--registry ${{ env.REGISTRY }}
|
||||||
--owner ${{ env.OWNER }}
|
--owner ${{ env.OWNER }}
|
||||||
--short-image-name ${{ inputs.image }}
|
--image ${{ inputs.image }}
|
||||||
--variant ${{ inputs.variant }}
|
--variant ${{ inputs.variant }}
|
||||||
--tags-dir /tmp/jupyter/tags/
|
--tags-dir /tmp/jupyter/tags/
|
||||||
shell: bash
|
shell: bash
|
||||||
@@ -97,7 +97,7 @@ jobs:
|
|||||||
python3 -m tagging.apps.write_manifest
|
python3 -m tagging.apps.write_manifest
|
||||||
--registry ${{ env.REGISTRY }}
|
--registry ${{ env.REGISTRY }}
|
||||||
--owner ${{ env.OWNER }}
|
--owner ${{ env.OWNER }}
|
||||||
--short-image-name ${{ inputs.image }}
|
--image ${{ inputs.image }}
|
||||||
--variant ${{ inputs.variant }}
|
--variant ${{ inputs.variant }}
|
||||||
--hist-lines-dir /tmp/jupyter/hist_lines/
|
--hist-lines-dir /tmp/jupyter/hist_lines/
|
||||||
--manifests-dir /tmp/jupyter/manifests/
|
--manifests-dir /tmp/jupyter/manifests/
|
||||||
@@ -133,5 +133,5 @@ jobs:
|
|||||||
python3 -m tests.run_tests
|
python3 -m tests.run_tests
|
||||||
--registry ${{ env.REGISTRY }}
|
--registry ${{ env.REGISTRY }}
|
||||||
--owner ${{ env.OWNER }}
|
--owner ${{ env.OWNER }}
|
||||||
--short-image-name ${{ inputs.image }}
|
--image ${{ inputs.image }}
|
||||||
shell: bash
|
shell: bash
|
||||||
|
2
.github/workflows/docker-merge-tags.yml
vendored
2
.github/workflows/docker-merge-tags.yml
vendored
@@ -69,7 +69,7 @@ jobs:
|
|||||||
if: env.PUSH_TO_REGISTRY == 'true'
|
if: env.PUSH_TO_REGISTRY == 'true'
|
||||||
run: >
|
run: >
|
||||||
python3 -m tagging.apps.merge_tags
|
python3 -m tagging.apps.merge_tags
|
||||||
--short-image-name ${{ inputs.image }}
|
--image ${{ inputs.image }}
|
||||||
--variant ${{ inputs.variant }}
|
--variant ${{ inputs.variant }}
|
||||||
--tags-dir /tmp/jupyter/tags/
|
--tags-dir /tmp/jupyter/tags/
|
||||||
shell: bash
|
shell: bash
|
||||||
|
2
.github/workflows/docker-tag-push.yml
vendored
2
.github/workflows/docker-tag-push.yml
vendored
@@ -65,7 +65,7 @@ jobs:
|
|||||||
python3 -m tagging.apps.apply_tags
|
python3 -m tagging.apps.apply_tags
|
||||||
--registry ${{ env.REGISTRY }}
|
--registry ${{ env.REGISTRY }}
|
||||||
--owner ${{ env.OWNER }}
|
--owner ${{ env.OWNER }}
|
||||||
--short-image-name ${{ inputs.image }}
|
--image ${{ inputs.image }}
|
||||||
--variant ${{ inputs.variant }}
|
--variant ${{ inputs.variant }}
|
||||||
--platform ${{ inputs.platform }}
|
--platform ${{ inputs.platform }}
|
||||||
--tags-dir /tmp/jupyter/tags/
|
--tags-dir /tmp/jupyter/tags/
|
||||||
|
10
Makefile
10
Makefile
@@ -41,7 +41,7 @@ build/%: ROOT_IMAGE?=ubuntu:24.04
|
|||||||
build/%: PYTHON_VERSION?=3.12
|
build/%: PYTHON_VERSION?=3.12
|
||||||
build/%: ## build the latest image for a stack using the system's architecture
|
build/%: ## build the latest image for a stack using the system's architecture
|
||||||
docker build $(DOCKER_BUILD_ARGS) --rm --force-rm \
|
docker build $(DOCKER_BUILD_ARGS) --rm --force-rm \
|
||||||
--tag "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" \
|
--tag "$(REGISTRY)/$(OWNER)/$(notdir $@)" \
|
||||||
"./images/$(notdir $@)" \
|
"./images/$(notdir $@)" \
|
||||||
--build-arg REGISTRY="$(REGISTRY)" \
|
--build-arg REGISTRY="$(REGISTRY)" \
|
||||||
--build-arg OWNER="$(OWNER)" \
|
--build-arg OWNER="$(OWNER)" \
|
||||||
@@ -82,13 +82,13 @@ hook/%: ## run post-build hooks for an image
|
|||||||
python3 -m tagging.apps.write_tags_file \
|
python3 -m tagging.apps.write_tags_file \
|
||||||
--registry "$(REGISTRY)" \
|
--registry "$(REGISTRY)" \
|
||||||
--owner "$(OWNER)" \
|
--owner "$(OWNER)" \
|
||||||
--short-image-name "$(notdir $@)" \
|
--image "$(notdir $@)" \
|
||||||
--variant "$(VARIANT)" \
|
--variant "$(VARIANT)" \
|
||||||
--tags-dir /tmp/jupyter/tags/
|
--tags-dir /tmp/jupyter/tags/
|
||||||
python3 -m tagging.apps.write_manifest \
|
python3 -m tagging.apps.write_manifest \
|
||||||
--registry "$(REGISTRY)" \
|
--registry "$(REGISTRY)" \
|
||||||
--owner "$(OWNER)" \
|
--owner "$(OWNER)" \
|
||||||
--short-image-name "$(notdir $@)" \
|
--image "$(notdir $@)" \
|
||||||
--variant "$(VARIANT)" \
|
--variant "$(VARIANT)" \
|
||||||
--hist-lines-dir /tmp/jupyter/hist_lines/ \
|
--hist-lines-dir /tmp/jupyter/hist_lines/ \
|
||||||
--manifests-dir /tmp/jupyter/manifests/ \
|
--manifests-dir /tmp/jupyter/manifests/ \
|
||||||
@@ -96,7 +96,7 @@ hook/%: ## run post-build hooks for an image
|
|||||||
python3 -m tagging.apps.apply_tags \
|
python3 -m tagging.apps.apply_tags \
|
||||||
--registry "$(REGISTRY)" \
|
--registry "$(REGISTRY)" \
|
||||||
--owner "$(OWNER)" \
|
--owner "$(OWNER)" \
|
||||||
--short-image-name "$(notdir $@)" \
|
--image "$(notdir $@)" \
|
||||||
--variant "$(VARIANT)" \
|
--variant "$(VARIANT)" \
|
||||||
--platform "$(shell uname -m)" \
|
--platform "$(shell uname -m)" \
|
||||||
--tags-dir /tmp/jupyter/tags/
|
--tags-dir /tmp/jupyter/tags/
|
||||||
@@ -139,5 +139,5 @@ test/%: ## run tests against a stack
|
|||||||
python3 -m tests.run_tests \
|
python3 -m tests.run_tests \
|
||||||
--registry "$(REGISTRY)" \
|
--registry "$(REGISTRY)" \
|
||||||
--owner "$(OWNER)" \
|
--owner "$(OWNER)" \
|
||||||
--short-image-name "$(notdir $@)"
|
--image "$(notdir $@)"
|
||||||
test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks
|
test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks
|
||||||
|
@@ -2,12 +2,11 @@
|
|||||||
# Copyright (c) Jupyter Development Team.
|
# Copyright (c) Jupyter Development Team.
|
||||||
# Distributed under the terms of the Modified BSD License.
|
# Distributed under the terms of the Modified BSD License.
|
||||||
import logging
|
import logging
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import plumbum
|
import plumbum
|
||||||
|
|
||||||
from tagging.apps.common_cli_arguments import common_arguments_parser
|
from tagging.apps.common_cli_arguments import common_arguments_parser
|
||||||
from tagging.utils.get_platform import unify_aarch64
|
from tagging.utils.config import Config
|
||||||
from tagging.utils.get_prefix import get_file_prefix_for_platform
|
from tagging.utils.get_prefix import get_file_prefix_for_platform
|
||||||
|
|
||||||
docker = plumbum.local["docker"]
|
docker = plumbum.local["docker"]
|
||||||
@@ -15,44 +14,30 @@ docker = plumbum.local["docker"]
|
|||||||
LOGGER = logging.getLogger(__name__)
|
LOGGER = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def apply_tags(
|
def apply_tags(config: Config) -> None:
|
||||||
*,
|
|
||||||
registry: str,
|
|
||||||
owner: str,
|
|
||||||
short_image_name: str,
|
|
||||||
variant: str,
|
|
||||||
platform: str,
|
|
||||||
tags_dir: Path,
|
|
||||||
) -> None:
|
|
||||||
"""
|
"""
|
||||||
Tags <registry>/<owner>/<short_image_name>:latest with the tags reported by all taggers for this image
|
Tags <config.full_image()> with the tags reported by all taggers for this image
|
||||||
"""
|
"""
|
||||||
LOGGER.info(f"Tagging image: {short_image_name}")
|
LOGGER.info(f"Tagging image: {config.image}")
|
||||||
|
|
||||||
file_prefix = get_file_prefix_for_platform(platform, variant)
|
file_prefix = get_file_prefix_for_platform(config.platform, config.variant)
|
||||||
image = f"{registry}/{owner}/{short_image_name}:latest"
|
filename = f"{file_prefix}-{config.image}.txt"
|
||||||
filename = f"{file_prefix}-{short_image_name}.txt"
|
tags = (config.tags_dir / filename).read_text().splitlines()
|
||||||
tags = (tags_dir / filename).read_text().splitlines()
|
|
||||||
|
|
||||||
for tag in tags:
|
for tag in tags:
|
||||||
LOGGER.info(f"Applying tag: {tag}")
|
LOGGER.info(f"Applying tag: {tag}")
|
||||||
docker["tag", image, tag] & plumbum.FG
|
docker["tag", config.full_image(), tag] & plumbum.FG
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
arg_parser = common_arguments_parser(
|
config = common_arguments_parser(
|
||||||
registry=True, owner=True, short_image_name=True, variant=True, tags_dir=True
|
registry=True,
|
||||||
|
owner=True,
|
||||||
|
image=True,
|
||||||
|
variant=True,
|
||||||
|
platform=True,
|
||||||
|
tags_dir=True,
|
||||||
)
|
)
|
||||||
arg_parser.add_argument(
|
apply_tags(config)
|
||||||
"--platform",
|
|
||||||
required=True,
|
|
||||||
type=str,
|
|
||||||
choices=["x86_64", "aarch64", "arm64"],
|
|
||||||
help="Image platform",
|
|
||||||
)
|
|
||||||
args = arg_parser.parse_args()
|
|
||||||
args.platform = unify_aarch64(args.platform)
|
|
||||||
|
|
||||||
apply_tags(**vars(args))
|
|
||||||
|
@@ -3,17 +3,22 @@
|
|||||||
import argparse
|
import argparse
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from tagging.utils.config import Config
|
||||||
|
from tagging.utils.get_platform import unify_aarch64
|
||||||
|
|
||||||
|
|
||||||
def common_arguments_parser(
|
def common_arguments_parser(
|
||||||
*,
|
*,
|
||||||
registry: bool = False,
|
registry: bool = False,
|
||||||
owner: bool = False,
|
owner: bool = False,
|
||||||
short_image_name: bool = False,
|
image: bool = False,
|
||||||
variant: bool = False,
|
variant: bool = False,
|
||||||
|
platform: bool = False,
|
||||||
tags_dir: bool = False,
|
tags_dir: bool = False,
|
||||||
hist_lines_dir: bool = False,
|
hist_lines_dir: bool = False,
|
||||||
manifests_dir: bool = False,
|
manifests_dir: bool = False,
|
||||||
) -> argparse.ArgumentParser:
|
repository: bool = False,
|
||||||
|
) -> Config:
|
||||||
"""Add common CLI arguments to parser"""
|
"""Add common CLI arguments to parser"""
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
@@ -30,9 +35,9 @@ def common_arguments_parser(
|
|||||||
required=True,
|
required=True,
|
||||||
help="Owner of the image",
|
help="Owner of the image",
|
||||||
)
|
)
|
||||||
if short_image_name:
|
if image:
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--short-image-name",
|
"--image",
|
||||||
required=True,
|
required=True,
|
||||||
help="Short image name",
|
help="Short image name",
|
||||||
)
|
)
|
||||||
@@ -42,6 +47,14 @@ def common_arguments_parser(
|
|||||||
required=True,
|
required=True,
|
||||||
help="Variant tag prefix",
|
help="Variant tag prefix",
|
||||||
)
|
)
|
||||||
|
if platform:
|
||||||
|
parser.add_argument(
|
||||||
|
"--platform",
|
||||||
|
required=True,
|
||||||
|
type=str,
|
||||||
|
choices=["x86_64", "aarch64", "arm64"],
|
||||||
|
help="Image platform",
|
||||||
|
)
|
||||||
if tags_dir:
|
if tags_dir:
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--tags-dir",
|
"--tags-dir",
|
||||||
@@ -63,5 +76,14 @@ def common_arguments_parser(
|
|||||||
type=Path,
|
type=Path,
|
||||||
help="Directory for manifests file",
|
help="Directory for manifests file",
|
||||||
)
|
)
|
||||||
|
if repository:
|
||||||
|
parser.add_argument(
|
||||||
|
"--repository",
|
||||||
|
required=True,
|
||||||
|
help="Repository name on GitHub",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
if platform:
|
||||||
|
args.platform = unify_aarch64(args.platform)
|
||||||
|
|
||||||
return parser
|
return Config(**vars(args))
|
||||||
|
@@ -2,11 +2,11 @@
|
|||||||
# Copyright (c) Jupyter Development Team.
|
# Copyright (c) Jupyter Development Team.
|
||||||
# Distributed under the terms of the Modified BSD License.
|
# Distributed under the terms of the Modified BSD License.
|
||||||
import logging
|
import logging
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import plumbum
|
import plumbum
|
||||||
|
|
||||||
from tagging.apps.common_cli_arguments import common_arguments_parser
|
from tagging.apps.common_cli_arguments import common_arguments_parser
|
||||||
|
from tagging.utils.config import Config
|
||||||
from tagging.utils.get_platform import ALL_PLATFORMS
|
from tagging.utils.get_platform import ALL_PLATFORMS
|
||||||
from tagging.utils.get_prefix import get_file_prefix_for_platform
|
from tagging.utils.get_prefix import get_file_prefix_for_platform
|
||||||
|
|
||||||
@@ -15,23 +15,18 @@ docker = plumbum.local["docker"]
|
|||||||
LOGGER = logging.getLogger(__name__)
|
LOGGER = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def merge_tags(
|
def merge_tags(config: Config) -> None:
|
||||||
*,
|
|
||||||
short_image_name: str,
|
|
||||||
variant: str,
|
|
||||||
tags_dir: Path,
|
|
||||||
) -> None:
|
|
||||||
"""
|
"""
|
||||||
Merge tags for x86_64 and aarch64 images when possible.
|
Merge tags for x86_64 and aarch64 images when possible.
|
||||||
"""
|
"""
|
||||||
LOGGER.info(f"Merging tags for image: {short_image_name}")
|
LOGGER.info(f"Merging tags for image: {config.image}")
|
||||||
|
|
||||||
all_tags: set[str] = set()
|
all_tags: set[str] = set()
|
||||||
|
|
||||||
for platform in ALL_PLATFORMS:
|
for platform in ALL_PLATFORMS:
|
||||||
file_prefix = get_file_prefix_for_platform(platform, variant)
|
file_prefix = get_file_prefix_for_platform(platform, config.variant)
|
||||||
filename = f"{file_prefix}-{short_image_name}.txt"
|
filename = f"{file_prefix}-{config.image}.txt"
|
||||||
file_path = tags_dir / filename
|
file_path = config.tags_dir / filename
|
||||||
if file_path.exists():
|
if file_path.exists():
|
||||||
tags = file_path.read_text().splitlines()
|
tags = file_path.read_text().splitlines()
|
||||||
all_tags.update(tag.replace(platform + "-", "") for tag in tags)
|
all_tags.update(tag.replace(platform + "-", "") for tag in tags)
|
||||||
@@ -62,9 +57,5 @@ def merge_tags(
|
|||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
arg_parser = common_arguments_parser(
|
config = common_arguments_parser(image=True, variant=True, tags_dir=True)
|
||||||
short_image_name=True, variant=True, tags_dir=True
|
merge_tags(config)
|
||||||
)
|
|
||||||
args = arg_parser.parse_args()
|
|
||||||
|
|
||||||
merge_tags(**vars(args))
|
|
||||||
|
@@ -3,7 +3,6 @@
|
|||||||
# Distributed under the terms of the Modified BSD License.
|
# Distributed under the terms of the Modified BSD License.
|
||||||
import datetime
|
import datetime
|
||||||
import logging
|
import logging
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from docker.models.containers import Container
|
from docker.models.containers import Container
|
||||||
|
|
||||||
@@ -13,6 +12,7 @@ from tagging.hierarchy.get_taggers_and_manifests import (
|
|||||||
)
|
)
|
||||||
from tagging.manifests.header import ManifestHeader
|
from tagging.manifests.header import ManifestHeader
|
||||||
from tagging.manifests.manifest_interface import ManifestInterface
|
from tagging.manifests.manifest_interface import ManifestInterface
|
||||||
|
from tagging.utils.config import Config
|
||||||
from tagging.utils.docker_runner import DockerRunner
|
from tagging.utils.docker_runner import DockerRunner
|
||||||
from tagging.utils.get_prefix import get_file_prefix, get_tag_prefix
|
from tagging.utils.get_prefix import get_file_prefix, get_tag_prefix
|
||||||
from tagging.utils.git_helper import GitHelper
|
from tagging.utils.git_helper import GitHelper
|
||||||
@@ -25,130 +25,77 @@ MARKDOWN_LINE_BREAK = "<br />"
|
|||||||
|
|
||||||
|
|
||||||
def write_build_history_line(
|
def write_build_history_line(
|
||||||
*,
|
config: Config, filename: str, all_tags: list[str]
|
||||||
registry: str,
|
|
||||||
owner: str,
|
|
||||||
short_image_name: str,
|
|
||||||
hist_lines_dir: Path,
|
|
||||||
filename: str,
|
|
||||||
all_tags: list[str],
|
|
||||||
repository: str,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
LOGGER.info("Appending build history line")
|
LOGGER.info("Appending build history line")
|
||||||
|
|
||||||
date_column = f"`{BUILD_TIMESTAMP}`"
|
date_column = f"`{BUILD_TIMESTAMP}`"
|
||||||
image_column = MARKDOWN_LINE_BREAK.join(
|
image_column = MARKDOWN_LINE_BREAK.join(
|
||||||
f"`{registry}/{owner}/{short_image_name}:{tag_value}`" for tag_value in all_tags
|
f"`{config.full_image()}:{tag_value}`" for tag_value in all_tags
|
||||||
)
|
)
|
||||||
commit_hash = GitHelper.commit_hash()
|
commit_hash = GitHelper.commit_hash()
|
||||||
links_column = MARKDOWN_LINE_BREAK.join(
|
links_column = MARKDOWN_LINE_BREAK.join(
|
||||||
[
|
[
|
||||||
f"[Git diff](https://github.com/{repository}/commit/{commit_hash})",
|
f"[Git diff](https://github.com/{config.repository}/commit/{commit_hash})",
|
||||||
f"[Dockerfile](https://github.com/{repository}/blob/{commit_hash}/images/{short_image_name}/Dockerfile)",
|
f"[Dockerfile](https://github.com/{config.repository}/blob/{commit_hash}/images/{config.image}/Dockerfile)",
|
||||||
f"[Build manifest](./{filename})",
|
f"[Build manifest](./{filename})",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
build_history_line = f"| {date_column} | {image_column} | {links_column} |"
|
build_history_line = f"| {date_column} | {image_column} | {links_column} |"
|
||||||
hist_lines_dir.mkdir(parents=True, exist_ok=True)
|
config.hist_lines_dir.mkdir(parents=True, exist_ok=True)
|
||||||
file = hist_lines_dir / f"{filename}.txt"
|
file = config.hist_lines_dir / f"{filename}.txt"
|
||||||
file.write_text(build_history_line)
|
file.write_text(build_history_line)
|
||||||
LOGGER.info(f"Build history line written to: {file}")
|
LOGGER.info(f"Build history line written to: {file}")
|
||||||
|
|
||||||
|
|
||||||
def write_manifest_file(
|
def write_manifest_file(
|
||||||
*,
|
config: Config,
|
||||||
registry: str,
|
|
||||||
owner: str,
|
|
||||||
short_image_name: str,
|
|
||||||
manifests_dir: Path,
|
|
||||||
filename: str,
|
filename: str,
|
||||||
manifests: list[ManifestInterface],
|
manifests: list[ManifestInterface],
|
||||||
container: Container,
|
container: Container,
|
||||||
repository: str,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
manifest_names = [manifest.__class__.__name__ for manifest in manifests]
|
manifest_names = [manifest.__class__.__name__ for manifest in manifests]
|
||||||
LOGGER.info(f"Using manifests: {manifest_names}")
|
LOGGER.info(f"Using manifests: {manifest_names}")
|
||||||
|
|
||||||
markdown_pieces = [
|
markdown_pieces = [ManifestHeader.create_header(config, BUILD_TIMESTAMP)] + [
|
||||||
ManifestHeader.create_header(
|
manifest.markdown_piece(container) for manifest in manifests
|
||||||
registry=registry,
|
]
|
||||||
owner=owner,
|
|
||||||
short_image_name=short_image_name,
|
|
||||||
build_timestamp=BUILD_TIMESTAMP,
|
|
||||||
repository=repository,
|
|
||||||
)
|
|
||||||
] + [manifest.markdown_piece(container) for manifest in manifests]
|
|
||||||
markdown_content = "\n\n".join(markdown_pieces) + "\n"
|
markdown_content = "\n\n".join(markdown_pieces) + "\n"
|
||||||
|
|
||||||
manifests_dir.mkdir(parents=True, exist_ok=True)
|
config.manifests_dir.mkdir(parents=True, exist_ok=True)
|
||||||
file = manifests_dir / f"{filename}.md"
|
file = config.manifests_dir / f"{filename}.md"
|
||||||
file.write_text(markdown_content)
|
file.write_text(markdown_content)
|
||||||
LOGGER.info(f"Manifest file written to: {file}")
|
LOGGER.info(f"Manifest file written to: {file}")
|
||||||
|
|
||||||
|
|
||||||
def write_manifest(
|
def write_manifest(config: Config) -> None:
|
||||||
*,
|
LOGGER.info(f"Creating manifests for image: {config.image}")
|
||||||
registry: str,
|
taggers, manifests = get_taggers_and_manifests(config.image)
|
||||||
owner: str,
|
|
||||||
short_image_name: str,
|
|
||||||
variant: str,
|
|
||||||
hist_lines_dir: Path,
|
|
||||||
manifests_dir: Path,
|
|
||||||
repository: str,
|
|
||||||
) -> None:
|
|
||||||
LOGGER.info(f"Creating manifests for image: {registry}/{owner}/{short_image_name}")
|
|
||||||
taggers, manifests = get_taggers_and_manifests(short_image_name)
|
|
||||||
|
|
||||||
image = f"{registry}/{owner}/{short_image_name}:latest"
|
file_prefix = get_file_prefix(config.variant)
|
||||||
|
|
||||||
file_prefix = get_file_prefix(variant)
|
|
||||||
commit_hash_tag = GitHelper.commit_hash_tag()
|
commit_hash_tag = GitHelper.commit_hash_tag()
|
||||||
filename = f"{file_prefix}-{short_image_name}-{commit_hash_tag}"
|
filename = f"{file_prefix}-{config.image}-{commit_hash_tag}"
|
||||||
|
|
||||||
with DockerRunner(image) as container:
|
with DockerRunner(config.full_image()) as container:
|
||||||
tags_prefix = get_tag_prefix(variant)
|
tags_prefix = get_tag_prefix(config.variant)
|
||||||
all_tags = [
|
all_tags = [
|
||||||
tags_prefix + "-" + tagger.tag_value(container) for tagger in taggers
|
tags_prefix + "-" + tagger.tag_value(container) for tagger in taggers
|
||||||
]
|
]
|
||||||
write_build_history_line(
|
write_build_history_line(config, filename, all_tags)
|
||||||
registry=registry,
|
write_manifest_file(config, filename, manifests, container)
|
||||||
owner=owner,
|
|
||||||
short_image_name=short_image_name,
|
|
||||||
hist_lines_dir=hist_lines_dir,
|
|
||||||
filename=filename,
|
|
||||||
all_tags=all_tags,
|
|
||||||
repository=repository,
|
|
||||||
)
|
|
||||||
write_manifest_file(
|
|
||||||
registry=registry,
|
|
||||||
owner=owner,
|
|
||||||
short_image_name=short_image_name,
|
|
||||||
manifests_dir=manifests_dir,
|
|
||||||
filename=filename,
|
|
||||||
manifests=manifests,
|
|
||||||
container=container,
|
|
||||||
repository=repository,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
LOGGER.info(f"Current build timestamp: {BUILD_TIMESTAMP}")
|
||||||
|
|
||||||
arg_parser = common_arguments_parser(
|
config = common_arguments_parser(
|
||||||
registry=True,
|
registry=True,
|
||||||
owner=True,
|
owner=True,
|
||||||
short_image_name=True,
|
image=True,
|
||||||
variant=True,
|
variant=True,
|
||||||
hist_lines_dir=True,
|
hist_lines_dir=True,
|
||||||
manifests_dir=True,
|
manifests_dir=True,
|
||||||
|
repository=True,
|
||||||
)
|
)
|
||||||
arg_parser.add_argument(
|
write_manifest(config)
|
||||||
"--repository",
|
|
||||||
required=True,
|
|
||||||
help="Repository name on GitHub",
|
|
||||||
)
|
|
||||||
args = arg_parser.parse_args()
|
|
||||||
|
|
||||||
LOGGER.info(f"Current build timestamp: {BUILD_TIMESTAMP}")
|
|
||||||
|
|
||||||
write_manifest(**vars(args))
|
|
||||||
|
@@ -2,50 +2,40 @@
|
|||||||
# Copyright (c) Jupyter Development Team.
|
# Copyright (c) Jupyter Development Team.
|
||||||
# Distributed under the terms of the Modified BSD License.
|
# Distributed under the terms of the Modified BSD License.
|
||||||
import logging
|
import logging
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from tagging.apps.common_cli_arguments import common_arguments_parser
|
from tagging.apps.common_cli_arguments import common_arguments_parser
|
||||||
from tagging.hierarchy.get_taggers_and_manifests import (
|
from tagging.hierarchy.get_taggers_and_manifests import (
|
||||||
get_taggers_and_manifests,
|
get_taggers_and_manifests,
|
||||||
)
|
)
|
||||||
|
from tagging.utils.config import Config
|
||||||
from tagging.utils.docker_runner import DockerRunner
|
from tagging.utils.docker_runner import DockerRunner
|
||||||
from tagging.utils.get_prefix import get_file_prefix, get_tag_prefix
|
from tagging.utils.get_prefix import get_file_prefix, get_tag_prefix
|
||||||
|
|
||||||
LOGGER = logging.getLogger(__name__)
|
LOGGER = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def write_tags_file(
|
def write_tags_file(config: Config) -> None:
|
||||||
*,
|
|
||||||
registry: str,
|
|
||||||
owner: str,
|
|
||||||
short_image_name: str,
|
|
||||||
variant: str,
|
|
||||||
tags_dir: Path,
|
|
||||||
) -> None:
|
|
||||||
"""
|
"""
|
||||||
Writes tags file for the image <registry>/<owner>/<short_image_name>:latest
|
Writes tags file for the image {config.full_image()}
|
||||||
"""
|
"""
|
||||||
LOGGER.info(f"Tagging image: {registry}/{owner}/{short_image_name}")
|
LOGGER.info(f"Tagging image: {config.image}")
|
||||||
taggers, _ = get_taggers_and_manifests(short_image_name)
|
taggers, _ = get_taggers_and_manifests(config.image)
|
||||||
|
|
||||||
image = f"{registry}/{owner}/{short_image_name}:latest"
|
file_prefix = get_file_prefix(config.variant)
|
||||||
file_prefix = get_file_prefix(variant)
|
filename = f"{file_prefix}-{config.image}.txt"
|
||||||
filename = f"{file_prefix}-{short_image_name}.txt"
|
|
||||||
|
|
||||||
tags_prefix = get_tag_prefix(variant)
|
tags_prefix = get_tag_prefix(config.variant)
|
||||||
tags = [f"{registry}/{owner}/{short_image_name}:{tags_prefix}-latest"]
|
tags = [f"{config.full_image()}:{tags_prefix}-latest"]
|
||||||
with DockerRunner(image) as container:
|
with DockerRunner(config.full_image()) as container:
|
||||||
for tagger in taggers:
|
for tagger in taggers:
|
||||||
tagger_name = tagger.__class__.__name__
|
tagger_name = tagger.__class__.__name__
|
||||||
tag_value = tagger.tag_value(container)
|
tag_value = tagger.tag_value(container)
|
||||||
LOGGER.info(
|
LOGGER.info(
|
||||||
f"Calculated tag, tagger_name: {tagger_name} tag_value: {tag_value}"
|
f"Calculated tag, tagger_name: {tagger_name} tag_value: {tag_value}"
|
||||||
)
|
)
|
||||||
tags.append(
|
tags.append(f"{config.full_image()}:{tags_prefix}-{tag_value}")
|
||||||
f"{registry}/{owner}/{short_image_name}:{tags_prefix}-{tag_value}"
|
config.tags_dir.mkdir(parents=True, exist_ok=True)
|
||||||
)
|
file = config.tags_dir / filename
|
||||||
tags_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
file = tags_dir / filename
|
|
||||||
file.write_text("\n".join(tags))
|
file.write_text("\n".join(tags))
|
||||||
LOGGER.info(f"Tags file written to: {file}")
|
LOGGER.info(f"Tags file written to: {file}")
|
||||||
|
|
||||||
@@ -53,9 +43,7 @@ def write_tags_file(
|
|||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
arg_parser = common_arguments_parser(
|
config = common_arguments_parser(
|
||||||
registry=True, owner=True, short_image_name=True, variant=True, tags_dir=True
|
registry=True, owner=True, image=True, variant=True, tags_dir=True
|
||||||
)
|
)
|
||||||
args = arg_parser.parse_args()
|
write_tags_file(config)
|
||||||
|
|
||||||
write_tags_file(**vars(args))
|
|
||||||
|
@@ -6,12 +6,12 @@ from tagging.taggers.tagger_interface import TaggerInterface
|
|||||||
|
|
||||||
|
|
||||||
def get_taggers_and_manifests(
|
def get_taggers_and_manifests(
|
||||||
short_image_name: str | None,
|
image: str | None,
|
||||||
) -> tuple[list[TaggerInterface], list[ManifestInterface]]:
|
) -> tuple[list[TaggerInterface], list[ManifestInterface]]:
|
||||||
if short_image_name is None:
|
if image is None:
|
||||||
return [], []
|
return [], []
|
||||||
|
|
||||||
image_description = ALL_IMAGES[short_image_name]
|
image_description = ALL_IMAGES[image]
|
||||||
parent_taggers, parent_manifests = get_taggers_and_manifests(
|
parent_taggers, parent_manifests = get_taggers_and_manifests(
|
||||||
image_description.parent_image
|
image_description.parent_image
|
||||||
)
|
)
|
||||||
|
@@ -2,6 +2,7 @@
|
|||||||
# Distributed under the terms of the Modified BSD License.
|
# Distributed under the terms of the Modified BSD License.
|
||||||
import plumbum
|
import plumbum
|
||||||
|
|
||||||
|
from tagging.utils.config import Config
|
||||||
from tagging.utils.git_helper import GitHelper
|
from tagging.utils.git_helper import GitHelper
|
||||||
|
|
||||||
docker = plumbum.local["docker"]
|
docker = plumbum.local["docker"]
|
||||||
@@ -11,36 +12,30 @@ class ManifestHeader:
|
|||||||
"""ManifestHeader doesn't fall under common interface, and we run it separately"""
|
"""ManifestHeader doesn't fall under common interface, and we run it separately"""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def create_header(
|
def create_header(config: Config, build_timestamp: str) -> str:
|
||||||
registry: str,
|
|
||||||
owner: str,
|
|
||||||
short_image_name: str,
|
|
||||||
build_timestamp: str,
|
|
||||||
repository: str,
|
|
||||||
) -> str:
|
|
||||||
commit_hash = GitHelper.commit_hash()
|
commit_hash = GitHelper.commit_hash()
|
||||||
commit_hash_tag = GitHelper.commit_hash_tag()
|
commit_hash_tag = GitHelper.commit_hash_tag()
|
||||||
commit_message = GitHelper.commit_message()
|
commit_message = GitHelper.commit_message()
|
||||||
|
|
||||||
# Unfortunately, `docker images` doesn't work when specifying `docker.io` as registry
|
# Unfortunately, `docker images` doesn't work when specifying `docker.io` as registry
|
||||||
fixed_registry = registry + "/" if registry != "docker.io" else ""
|
fixed_registry = config.registry + "/" if config.registry != "docker.io" else ""
|
||||||
|
|
||||||
image_size = docker[
|
image_size = docker[
|
||||||
"images",
|
"images",
|
||||||
f"{fixed_registry}{owner}/{short_image_name}:latest",
|
f"{fixed_registry}{config.owner}/{config.image}:latest",
|
||||||
"--format",
|
"--format",
|
||||||
"{{.Size}}",
|
"{{.Size}}",
|
||||||
]().rstrip()
|
]().rstrip()
|
||||||
|
|
||||||
return f"""\
|
return f"""\
|
||||||
# Build manifest for image: {short_image_name}:{commit_hash_tag}
|
# Build manifest for image: {config.image}:{commit_hash_tag}
|
||||||
|
|
||||||
## Build Info
|
## Build Info
|
||||||
|
|
||||||
- Build timestamp: {build_timestamp}
|
- Build timestamp: {build_timestamp}
|
||||||
- Docker image: `{registry}/{owner}/{short_image_name}:{commit_hash_tag}`
|
- Docker image: `{config.full_image()}:{commit_hash_tag}`
|
||||||
- Docker image size: {image_size}
|
- Docker image size: {image_size}
|
||||||
- Git commit SHA: [{commit_hash}](https://github.com/{repository}/commit/{commit_hash})
|
- Git commit SHA: [{commit_hash}](https://github.com/{config.repository}/commit/{commit_hash})
|
||||||
- Git commit message:
|
- Git commit message:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
|
22
tagging/utils/config.py
Normal file
22
tagging/utils/config.py
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
# Copyright (c) Jupyter Development Team.
|
||||||
|
# Distributed under the terms of the Modified BSD License.
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Config:
|
||||||
|
registry: str = ""
|
||||||
|
owner: str = ""
|
||||||
|
image: str = ""
|
||||||
|
variant: str = ""
|
||||||
|
platform: str = ""
|
||||||
|
|
||||||
|
tags_dir: Path = Path()
|
||||||
|
hist_lines_dir: Path = Path()
|
||||||
|
manifests_dir: Path = Path()
|
||||||
|
|
||||||
|
repository: str = ""
|
||||||
|
|
||||||
|
def full_image(self) -> str:
|
||||||
|
return f"{self.registry}/{self.owner}/{self.image}"
|
@@ -24,14 +24,12 @@ _IMAGE_PARENT = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def get_test_dirs(
|
def get_test_dirs(image: str | None) -> list[Path]:
|
||||||
short_image_name: str | None,
|
if image is None:
|
||||||
) -> list[Path]:
|
|
||||||
if short_image_name is None:
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
test_dirs = get_test_dirs(_IMAGE_PARENT[short_image_name])
|
test_dirs = get_test_dirs(_IMAGE_PARENT[image])
|
||||||
current_test_dir = IMAGE_SPECIFIC_TESTS_DIR / short_image_name
|
current_test_dir = IMAGE_SPECIFIC_TESTS_DIR / image
|
||||||
assert current_test_dir.exists(), f"{current_test_dir} does not exist."
|
assert current_test_dir.exists(), f"{current_test_dir} does not exist."
|
||||||
test_dirs.append(current_test_dir)
|
test_dirs.append(current_test_dir)
|
||||||
return test_dirs
|
return test_dirs
|
||||||
|
@@ -12,10 +12,10 @@ def test_units(container: TrackedContainer) -> None:
|
|||||||
"""Various units tests
|
"""Various units tests
|
||||||
Add a py file in the `tests/<somestack>/units` dir, and it will be automatically tested
|
Add a py file in the `tests/<somestack>/units` dir, and it will be automatically tested
|
||||||
"""
|
"""
|
||||||
short_image_name = container.image_name[container.image_name.rfind("/") + 1 :]
|
image = container.image_name[container.image_name.rfind("/") + 1 :]
|
||||||
LOGGER.info(f"Running unit tests for: {short_image_name}")
|
LOGGER.info(f"Running unit tests for: {image}")
|
||||||
|
|
||||||
test_dirs = get_test_dirs(short_image_name)
|
test_dirs = get_test_dirs(image)
|
||||||
|
|
||||||
for test_dir in test_dirs:
|
for test_dir in test_dirs:
|
||||||
host_data_dir = test_dir / "units"
|
host_data_dir = test_dir / "units"
|
||||||
|
@@ -13,16 +13,11 @@ python3 = plumbum.local["python3"]
|
|||||||
LOGGER = logging.getLogger(__name__)
|
LOGGER = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def test_image(
|
def test_image(*, registry: str, owner: str, image: str) -> None:
|
||||||
*,
|
LOGGER.info(f"Testing image: {image}")
|
||||||
registry: str,
|
test_dirs = get_test_dirs(image)
|
||||||
owner: str,
|
|
||||||
short_image_name: str,
|
|
||||||
) -> None:
|
|
||||||
LOGGER.info(f"Testing image: {short_image_name}")
|
|
||||||
test_dirs = get_test_dirs(short_image_name)
|
|
||||||
LOGGER.info(f"Test dirs to be run: {test_dirs}")
|
LOGGER.info(f"Test dirs to be run: {test_dirs}")
|
||||||
with plumbum.local.env(TEST_IMAGE=f"{registry}/{owner}/{short_image_name}"):
|
with plumbum.local.env(TEST_IMAGE=f"{registry}/{owner}/{image}"):
|
||||||
(
|
(
|
||||||
python3[
|
python3[
|
||||||
"-m",
|
"-m",
|
||||||
@@ -53,7 +48,7 @@ if __name__ == "__main__":
|
|||||||
help="Owner of the image",
|
help="Owner of the image",
|
||||||
)
|
)
|
||||||
arg_parser.add_argument(
|
arg_parser.add_argument(
|
||||||
"--short-image-name",
|
"--image",
|
||||||
required=True,
|
required=True,
|
||||||
help="Short image name",
|
help="Short image name",
|
||||||
)
|
)
|
||||||
|
Reference in New Issue
Block a user