hazza/docker-stacks
1
0
Fork 0
mirror of https://github.com/jupyter/docker-stacks.git synced 2025-10-07 18:14:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve handling common parser arguments (#2221)

Browse Source
This commit is contained in:
Ayaz Salikhov
2025-02-18 11:06:10 +00:00
committed by GitHub
parent 35b440186e
commit e57047801c
12 changed files with 149 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
.github/workflows/docker-build-test-upload.yml vendored
View File

@@ -66,8 +66,13 @@ jobs:
shell: bash
- name: Write tags file 🏷
run: |
python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} --variant ${{ inputs.variant }}
run: >
python3 -m tagging.write_tags_file
--registry ${{ env.REGISTRY }}
--owner ${{ env.OWNER }}
--short-image-name ${{ inputs.image }}
--variant ${{ inputs.variant }}
--tags-dir /tmp/jupyter/tags/
shell: bash
- name: Upload tags file 💾
uses: actions/upload-artifact@v4
@@ -77,7 +82,14 @@ jobs:
retention-days: 3
- name: Write manifest and build history file 🏷
run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-lines-dir /tmp/jupyter/hist_lines/ --manifests-dir /tmp/jupyter/manifests/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} --variant ${{ inputs.variant }}
run: >
python3 -m tagging.write_manifest
--registry ${{ env.REGISTRY }}
--owner ${{ env.OWNER }}
--short-image-name ${{ inputs.image }}
--variant ${{ inputs.variant }}
--hist-lines-dir /tmp/jupyter/hist_lines/
--manifests-dir /tmp/jupyter/manifests/
shell: bash
- name: Upload manifest file 💾
uses: actions/upload-artifact@v4
@@ -105,5 +117,9 @@ jobs:
retention-days: 3
- name: Run tests ✅
run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
run: >
python3 -m tests.run_tests
--registry ${{ env.REGISTRY }}
--owner ${{ env.OWNER }}
--short-image-name ${{ inputs.image }}
shell: bash

6
.github/workflows/docker-merge-tags.yml vendored
View File

@@ -62,5 +62,9 @@ jobs:
- name: Merge tags for the images 🔀
if: env.PUSH_TO_REGISTRY == 'true'
run: python3 -m tagging.merge_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --variant ${{ inputs.variant }}
run: >
python3 -m tagging.merge_tags
--short-image-name ${{ inputs.image }}
--variant ${{ inputs.variant }}
--tags-dir /tmp/jupyter/tags/
shell: bash

9
.github/workflows/docker-tag-push.yml vendored
View File

@@ -56,7 +56,14 @@ jobs:
name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-tags
path: /tmp/jupyter/tags/
- name: Apply tags to the loaded image 🏷
run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --platform ${{ inputs.platform }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} --variant ${{ inputs.variant }}
run: >
python3 -m tagging.apply_tags
--registry ${{ env.REGISTRY }}
--owner ${{ env.OWNER }}
--short-image-name ${{ inputs.image }}
--variant ${{ inputs.variant }}
--platform ${{ inputs.platform }}
--tags-dir /tmp/jupyter/tags/
# This step is needed to prevent pushing non-multi-arch "latest" tag
- name: Remove the "latest" tag from the image 🗑️
run: docker image rmi ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }}:latest

6
.github/workflows/docker-wiki-update.yml vendored
View File

@@ -39,7 +39,11 @@ jobs:
path: wiki/
- name: Update wiki 🏷
run: python3 -m tagging.update_wiki --wiki-dir wiki/ --hist-lines-dir /tmp/jupyter/hist_lines/ --manifests-dir /tmp/jupyter/manifests/
run: >
python3 -m tagging.update_wiki
--wiki-dir wiki/
--hist-lines-dir /tmp/jupyter/hist_lines/
--manifests-dir /tmp/jupyter/manifests/
shell: bash
- name: Push Wiki to GitHub 📤

26
Makefile
View File

@@ -79,25 +79,25 @@ linkcheck-docs: ## check broken links
hook/%: VARIANT?=default
hook/%: ## run post-build hooks for an image
python3 -m tagging.write_tags_file \
--short-image-name "$(notdir $@)" \
--tags-dir /tmp/jupyter/tags/ \
--registry "$(REGISTRY)" \
--owner "$(OWNER)" \
--variant "$(VARIANT)"
--short-image-name "$(notdir $@)" \
--variant "$(VARIANT)" \
--tags-dir /tmp/jupyter/tags/
python3 -m tagging.write_manifest \
--registry "$(REGISTRY)" \
--owner "$(OWNER)" \
--short-image-name "$(notdir $@)" \
--variant "$(VARIANT)" \
--hist-lines-dir /tmp/jupyter/hist_lines/ \
--manifests-dir /tmp/jupyter/manifests/ \
--registry "$(REGISTRY)" \
--owner "$(OWNER)" \
--variant "$(VARIANT)"
--manifests-dir /tmp/jupyter/manifests/
python3 -m tagging.apply_tags \
--short-image-name "$(notdir $@)" \
--tags-dir /tmp/jupyter/tags/ \
--platform "$(shell uname -m)" \
--registry "$(REGISTRY)" \
--owner "$(OWNER)" \
--variant "$(VARIANT)"
--short-image-name "$(notdir $@)" \
--variant "$(VARIANT)" \
--platform "$(shell uname -m)" \
--tags-dir /tmp/jupyter/tags/
hook-all: $(foreach I, $(ALL_IMAGES), hook/$(I)) ## run post-build hooks for all images
@@ -135,7 +135,7 @@ run-sudo-shell/%: ## run bash in interactive mode as root in a stack
test/%: ## run tests against a stack
python3 -m tests.run_tests \
--short-image-name "$(notdir $@)" \
--registry "$(REGISTRY)" \
--owner "$(OWNER)"
--owner "$(OWNER)" \
--short-image-name "$(notdir $@)"
test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks

48
tagging/apply_tags.py
View File

@@ -1,12 +1,12 @@
#!/usr/bin/env python3
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import argparse
import logging
from pathlib import Path
import plumbum
from tagging.common_arguments import common_arguments_parser
from tagging.get_platform import unify_aarch64
from tagging.get_prefix import get_file_prefix_for_platform
@@ -16,12 +16,13 @@ LOGGER = logging.getLogger(__name__)
def apply_tags(
short_image_name: str,
*,
registry: str,
owner: str,
tags_dir: Path,
platform: str,
short_image_name: str,
variant: str,
platform: str,
tags_dir: Path,
) -> None:
"""
Tags <registry>/<owner>/<short_image_name>:latest with the tags reported by all taggers for this image
@@ -41,18 +42,7 @@ def apply_tags(
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
"--short-image-name",
required=True,
help="Short image name",
)
arg_parser.add_argument(
"--tags-dir",
required=True,
type=Path,
help="Directory with saved tags file",
)
arg_parser = common_arguments_parser()
arg_parser.add_argument(
"--platform",
required=True,
@@ -61,30 +51,12 @@ if __name__ == "__main__":
help="Image platform",
)
arg_parser.add_argument(
"--registry",
"--tags-dir",
required=True,
type=str,
choices=["docker.io", "quay.io"],
help="Image registry",
)
arg_parser.add_argument(
"--owner",
required=True,
help="Owner of the image",
)
arg_parser.add_argument(
"--variant",
required=True,
help="Variant tag prefix",
type=Path,
help="Directory with saved tags file",
)
args = arg_parser.parse_args()
args.platform = unify_aarch64(args.platform)
apply_tags(
args.short_image_name,
args.registry,
args.owner,
args.tags_dir,
args.platform,
args.variant,
)
apply_tags(**vars(args))

39
tagging/common_arguments.py Normal file
View File

@@ -0,0 +1,39 @@
import argparse
def common_arguments_parser(
registry: bool = True,
owner: bool = True,
short_image_name: bool = True,
variant: bool = True,
) -> argparse.ArgumentParser:
"""Add common CLI arguments to parser"""
parser = argparse.ArgumentParser()
if registry:
parser.add_argument(
"--registry",
required=True,
choices=["docker.io", "quay.io"],
help="Image registry",
)
if owner:
parser.add_argument(
"--owner",
required=True,
help="Owner of the image",
)
if short_image_name:
parser.add_argument(
"--short-image-name",
required=True,
help="Short image name",
)
if variant:
parser.add_argument(
"--variant",
required=True,
help="Variant tag prefix",
)
return parser

17
tagging/merge_tags.py
View File

@@ -1,12 +1,12 @@
#!/usr/bin/env python3
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import argparse
import logging
from pathlib import Path
import plumbum
from tagging.common_arguments import common_arguments_parser
from tagging.get_platform import ALL_PLATFORMS
from tagging.get_prefix import get_file_prefix_for_platform
@@ -16,6 +16,7 @@ LOGGER = logging.getLogger(__name__)
def merge_tags(
*,
short_image_name: str,
variant: str,
tags_dir: Path,
@@ -60,23 +61,13 @@ def merge_tags(
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
"--short-image-name",
required=True,
help="Short image name",
)
arg_parser = common_arguments_parser(registry=False, owner=False)
arg_parser.add_argument(
"--tags-dir",
required=True,
type=Path,
help="Directory with saved tags file",
)
arg_parser.add_argument(
"--variant",
required=True,
help="Variant tag prefix",
)
args = arg_parser.parse_args()
merge_tags(args.short_image_name, args.variant, args.tags_dir)
merge_tags(**vars(args))

10
tagging/update_wiki.py
View File

@@ -155,7 +155,11 @@ def remove_old_manifests(wiki_dir: Path) -> None:
def update_wiki(
wiki_dir: Path, hist_lines_dir: Path, manifests_dir: Path, allow_no_files: bool
*,
wiki_dir: Path,
hist_lines_dir: Path,
manifests_dir: Path,
allow_no_files: bool,
) -> None:
LOGGER.info("Updating wiki")
@@ -214,6 +218,4 @@ if __name__ == "__main__":
)
args = arg_parser.parse_args()
update_wiki(
args.wiki_dir, args.hist_lines_dir, args.manifests_dir, args.allow_no_files
)
update_wiki(**vars(args))

65
tagging/write_manifest.py
View File

@@ -1,13 +1,13 @@
#!/usr/bin/env python3
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import argparse
import datetime
import logging
from pathlib import Path
from docker.models.containers import Container
from tagging.common_arguments import common_arguments_parser
from tagging.docker_runner import DockerRunner
from tagging.get_prefix import get_file_prefix, get_tag_prefix
from tagging.get_taggers_and_manifests import get_taggers_and_manifests
@@ -22,9 +22,10 @@ MARKDOWN_LINE_BREAK = "<br />"
def write_build_history_line(
short_image_name: str,
*,
registry: str,
owner: str,
short_image_name: str,
hist_lines_dir: Path,
filename: str,
all_tags: list[str],
@@ -49,9 +50,10 @@ def write_build_history_line(
def write_manifest_file(
short_image_name: str,
*,
registry: str,
owner: str,
short_image_name: str,
manifests_dir: Path,
filename: str,
manifests: list[ManifestInterface],
@@ -70,9 +72,10 @@ def write_manifest_file(
def write_manifest(
short_image_name: str,
*,
registry: str,
owner: str,
short_image_name: str,
variant: str,
hist_lines_dir: Path,
manifests_dir: Path,
@@ -92,28 +95,28 @@ def write_manifest(
tags_prefix + "-" + tagger.tag_value(container) for tagger in taggers
]
write_build_history_line(
short_image_name, registry, owner, hist_lines_dir, filename, all_tags
registry=registry,
owner=owner,
short_image_name=short_image_name,
hist_lines_dir=hist_lines_dir,
filename=filename,
all_tags=all_tags,
)
write_manifest_file(
short_image_name,
registry,
owner,
manifests_dir,
filename,
manifests,
container,
registry=registry,
owner=owner,
short_image_name=short_image_name,
manifests_dir=manifests_dir,
filename=filename,
manifests=manifests,
container=container,
)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
"--short-image-name",
required=True,
help="Short image name",
)
arg_parser = common_arguments_parser()
arg_parser.add_argument(
"--hist-lines-dir",
required=True,
@@ -126,32 +129,8 @@ if __name__ == "__main__":
type=Path,
help="Directory to save manifest file",
)
arg_parser.add_argument(
"--registry",
required=True,
type=str,
choices=["docker.io", "quay.io"],
help="Image registry",
)
arg_parser.add_argument(
"--owner",
required=True,
help="Owner of the image",
)
arg_parser.add_argument(
"--variant",
required=True,
help="Variant tag prefix",
)
args = arg_parser.parse_args()
LOGGER.info(f"Current build timestamp: {BUILD_TIMESTAMP}")
write_manifest(
args.short_image_name,
args.registry,
args.owner,
args.variant,
args.hist_lines_dir,
args.manifests_dir,
)
write_manifest(**vars(args))

37
tagging/write_tags_file.py
View File

@@ -1,10 +1,10 @@
#!/usr/bin/env python3
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import argparse
import logging
from pathlib import Path
from tagging.common_arguments import common_arguments_parser
from tagging.docker_runner import DockerRunner
from tagging.get_prefix import get_file_prefix, get_tag_prefix
from tagging.get_taggers_and_manifests import get_taggers_and_manifests
@@ -13,9 +13,10 @@ LOGGER = logging.getLogger(__name__)
def write_tags_file(
short_image_name: str,
*,
registry: str,
owner: str,
short_image_name: str,
variant: str,
tags_dir: Path,
) -> None:
@@ -48,41 +49,13 @@ def write_tags_file(
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
"--short-image-name",
required=True,
help="Short image name",
)
arg_parser = common_arguments_parser()
arg_parser.add_argument(
"--tags-dir",
required=True,
type=Path,
help="Directory to save tags file",
)
arg_parser.add_argument(
"--registry",
required=True,
type=str,
choices=["docker.io", "quay.io"],
help="Image registry",
)
arg_parser.add_argument(
"--owner",
required=True,
help="Owner of the image",
)
arg_parser.add_argument(
"--variant",
required=True,
help="Variant tag prefix",
)
args = arg_parser.parse_args()
write_tags_file(
args.short_image_name,
args.registry,
args.owner,
args.variant,
args.tags_dir,
)
write_tags_file(**vars(args))

21
tests/run_tests.py
View File

@@ -13,7 +13,12 @@ python3 = plumbum.local["python3"]
LOGGER = logging.getLogger(__name__)
def test_image(short_image_name: str, registry: str, owner: str) -> None:
def test_image(
*,
registry: str,
owner: str,
short_image_name: str,
) -> None:
LOGGER.info(f"Testing image: {short_image_name}")
test_dirs = get_test_dirs(short_image_name)
LOGGER.info(f"Test dirs to be run: {test_dirs}")
@@ -36,15 +41,9 @@ if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
"--short-image-name",
required=True,
help="Short image name",
)
arg_parser.add_argument(
"--registry",
required=True,
type=str,
choices=["docker.io", "quay.io"],
help="Image registry",
)
@@ -53,7 +52,11 @@ if __name__ == "__main__":
required=True,
help="Owner of the image",
)
arg_parser.add_argument(
"--short-image-name",
required=True,
help="Short image name",
)
args = arg_parser.parse_args()
test_image(args.short_image_name, args.registry, args.owner)
test_image(**vars(args))