Cleanup docs about multi platform builds

This commit is contained in:
Ayaz Salikhov
2022-05-13 02:35:47 +03:00
parent 944664eb28
commit 0487c99660
5 changed files with 7 additions and 96 deletions

View File

@@ -7,21 +7,6 @@ SHELL:=bash
OWNER?=jupyter OWNER?=jupyter
# Need to list the images in build dependency order # Need to list the images in build dependency order
# Images supporting the following architectures:
# - linux/amd64
# - linux/arm64
MULTI_IMAGES:= \
base-notebook \
minimal-notebook \
r-notebook \
scipy-notebook \
pyspark-notebook \
all-spark-notebook
# Images that can only be built on the amd64 architecture (aka. x86_64)
AMD64_ONLY_IMAGES:= \
datascience-notebook \
tensorflow-notebook
# All of the images # All of the images
ALL_IMAGES:= \ ALL_IMAGES:= \
base-notebook \ base-notebook \
@@ -42,7 +27,7 @@ export DOCKER_BUILDKIT:=1
help: help:
@echo "jupyter/docker-stacks" @echo "jupyter/docker-stacks"
@echo "=====================" @echo "====================="
@echo "Replace % with a stack directory name (e.g., make build-multi/minimal-notebook)" @echo "Replace % with a stack directory name (e.g., make build/minimal-notebook)"
@echo @echo
@grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -57,60 +42,6 @@ build/%: ## build the latest image for a stack using the system's architecture
@echo "::endgroup::" @echo "::endgroup::"
build-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks build-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks
# Limitations on docker buildx build (using docker/buildx 0.5.1):
#
# 1. Can't --load and --push at the same time
#
# 2. Can't --load multiple platforms
#
# What does it mean to --load?
#
# - It means that the built image can be referenced by `docker` CLI, for example
# when using the `docker tag` or `docker push` commands.
#
# Workarounds due to limitations:
#
# 1. We always build a dedicated image using the current system architecture
# named as OWNER/<stack>-notebook so we always can reference that image no
# matter what during tests etc.
#
# 2. We always also build a multi-platform image during build-multi that will be
# inaccessible with `docker tag` and `docker push` etc, but this will help us
# test the build on the different platform and provide cached layers for
# later.
#
# 3. We let push-multi refer to rebuilding a multi image with `--push`.
#
# We can rely on the cached layer from build-multi now even though we never
# tagged the multi image.
#
# Outcomes of the workaround:
#
# 1. We can keep using the previously defined Makefile commands that doesn't
# include `-multi` suffix as before.
#
# 2. Assuming we have setup docker/dockerx properly to build in arm64
# architectures as well, then we can build and publish such images via the
# `-multi` suffix without needing a local registry.
#
# 3. If we get dedicated arm64 runners, we can test everything separately
# without needing to update this Makefile, and if all tests succeeds we can
# do a publish job that creates a multi-platform image for us.
#
build-multi/%: DOCKER_BUILD_ARGS?=
build-multi/%: ## build the latest image for a stack on both amd64 and arm64
@echo "::group::Build $(OWNER)/$(notdir $@) (system's architecture)"
docker buildx build $(DOCKER_BUILD_ARGS) -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --load
@echo -n "Built image size: "
@docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}"
@echo "::endgroup::"
@echo "::group::Build $(OWNER)/$(notdir $@) (amd64,arm64)"
docker buildx build $(DOCKER_BUILD_ARGS) -t build-multi-tmp-cache/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64,linux/arm64"
@echo "::endgroup::"
build-all-multi: $(foreach I, $(MULTI_IMAGES), build-multi/$(I)) $(foreach I, $(AMD64_ONLY_IMAGES), build/$(I)) ## build all stacks
check-outdated/%: ## check the outdated mamba/conda packages in a stack and produce a report (experimental) check-outdated/%: ## check the outdated mamba/conda packages in a stack and produce a report (experimental)
@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest tests/base-notebook/test_outdated.py @TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest tests/base-notebook/test_outdated.py
@@ -176,13 +107,6 @@ push/%: ## push all tags for a jupyter image
@echo "::endgroup::" @echo "::endgroup::"
push-all: $(foreach I, $(ALL_IMAGES), push/$(I)) ## push all tagged images push-all: $(foreach I, $(ALL_IMAGES), push/$(I)) ## push all tagged images
push-multi/%: DOCKER_BUILD_ARGS?=
push-multi/%: ## push all tags for a jupyter image that support multiple architectures
@echo "::group::Push $(OWNER)/$(notdir $@) (amd64,arm64)"
docker buildx build $(DOCKER_BUILD_ARGS) $($(subst -,_,$(notdir $@))_EXTRA_TAG_ARGS) -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64,linux/arm64" --push
@echo "::endgroup::"
push-all-multi: $(foreach I, $(MULTI_IMAGES), push-multi/$(I)) $(foreach I, $(AMD64_ONLY_IMAGES), push/$(I)) ## push all tagged images
run-shell/%: ## run a bash in interactive mode in a stack run-shell/%: ## run a bash in interactive mode in a stack

View File

@@ -115,14 +115,9 @@ This change is tracked in the issue [#1217](https://github.com/jupyter/docker-st
## CPU Architectures ## CPU Architectures
All published containers support amd64 (x86_64) and aarch64, except for `datascience-notebook` and `tensorflow-notebook`, which only support amd64 for now. We published containers both for `amd64` (`x86_64`) and `aarch64`, except for `datascience-notebook` and `tensorflow-notebook`, which only support amd64 for now.
### Caveats for arm64 images ### Caveats for arm64 images
- The manifests we publish in this project's wiki as well as the image tags for - We do not create multi-platform images
the multi-platform images that also support arm, are all based on the amd64 - All arm64 images have `aarch64-` tag prefix, for example `jupyter/base-notebook:aarch64-python-3.9.6`
version even though details about the installed packages versions could differ
between architectures. For the status about this, see
[#1401](https://github.com/jupyter/docker-stacks/issues/1401).
- Only the amd64 images are actively tested currently. For the status about
this, see [#1402](https://github.com/jupyter/docker-stacks/issues/1402).

View File

@@ -5,11 +5,7 @@ We greatly appreciate pull requests that extend the automated tests that vet the
## How the Tests Work ## How the Tests Work
A [GitHub Action workflow](https://github.com/jupyter/docker-stacks/blob/master/.github/workflows/docker.yml) A [GitHub Action workflow](https://github.com/jupyter/docker-stacks/blob/master/.github/workflows/docker.yml)
runs the following commands against pull requests submitted to the `jupyter/docker-stacks` repository: runs tests against pull requests submitted to the `jupyter/docker-stacks` repository:
1. `make build-all-multi` - which builds all the Docker images
2. `make test-all` - which tests the newly created Docker images
This `make` command builds and then tests every docker image.
We use `pytest` module to run tests on the image. We use `pytest` module to run tests on the image.
`conftest.py` and `pytest.ini` in the `tests` folder define the environment in which tests are run. `conftest.py` and `pytest.ini` in the `tests` folder define the environment in which tests are run.
@@ -40,8 +36,8 @@ Please follow the process below to add new tests:
If you use `make`, call: If you use `make`, call:
```bash ```bash
make build/somestack-notebook make build/<somestack>-notebook
make test/somestack-notebook make test/<somestack>-notebook
``` ```
3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request) 3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)

View File

@@ -19,9 +19,6 @@ def tag_image(short_image_name: str, owner: str, tags_prefix: str) -> None:
""" """
Tags <owner>/<short_image_name>:latest with the tags reported by all taggers Tags <owner>/<short_image_name>:latest with the tags reported by all taggers
for the given image. for the given image.
Tags are in a GitHub Actions environment also saved to environment variables
in a format making it easy to append them.
""" """
LOGGER.info(f"Tagging image: {short_image_name}") LOGGER.info(f"Tagging image: {short_image_name}")
taggers, _ = get_taggers_and_manifests(short_image_name) taggers, _ = get_taggers_and_manifests(short_image_name)

View File

@@ -27,7 +27,6 @@ def test_package_manager(
LOGGER.info( LOGGER.info(
f"Test that the package manager {package_manager} is working properly ..." f"Test that the package manager {package_manager} is working properly ..."
) )
# Increased timout to make aarch64 images tests work in QEMU
container.run_and_wait( container.run_and_wait(
timeout=30, timeout=30,
tty=True, tty=True,