hazza/docker-stacks
1
0
Fork 0
mirror of https://github.com/jupyter/docker-stacks.git synced 2025-10-11 12:02:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup docs about multi platform builds

Browse Source
This commit is contained in:
Ayaz Salikhov
2022-05-13 02:35:47 +03:00
parent 944664eb28
commit 0487c99660
5 changed files with 7 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
Makefile
View File

@@ -7,21 +7,6 @@ SHELL:=bash
OWNER?=jupyter
# Need to list the images in build dependency order
# Images supporting the following architectures:
# - linux/amd64
# - linux/arm64
MULTI_IMAGES:= \
base-notebook \
minimal-notebook \
r-notebook \
scipy-notebook \
pyspark-notebook \
all-spark-notebook
# Images that can only be built on the amd64 architecture (aka. x86_64)
AMD64_ONLY_IMAGES:= \
datascience-notebook \
tensorflow-notebook
# All of the images
ALL_IMAGES:= \
base-notebook \
@@ -42,7 +27,7 @@ export DOCKER_BUILDKIT:=1
help:
@echo "jupyter/docker-stacks"
@echo "====================="
@echo "Replace % with a stack directory name (e.g., make build-multi/minimal-notebook)"
@echo "Replace % with a stack directory name (e.g., make build/minimal-notebook)"
@echo
@grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -57,60 +42,6 @@ build/%: ## build the latest image for a stack using the system's architecture
@echo "::endgroup::"
build-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks
# Limitations on docker buildx build (using docker/buildx 0.5.1):
#
# 1. Can't --load and --push at the same time
#
# 2. Can't --load multiple platforms
#
# What does it mean to --load?
#
# - It means that the built image can be referenced by `docker` CLI, for example
# when using the `docker tag` or `docker push` commands.
#
# Workarounds due to limitations:
#
# 1. We always build a dedicated image using the current system architecture
# named as OWNER/<stack>-notebook so we always can reference that image no
# matter what during tests etc.
#
# 2. We always also build a multi-platform image during build-multi that will be
# inaccessible with `docker tag` and `docker push` etc, but this will help us
# test the build on the different platform and provide cached layers for
# later.
#
# 3. We let push-multi refer to rebuilding a multi image with `--push`.
#
# We can rely on the cached layer from build-multi now even though we never
# tagged the multi image.
#
# Outcomes of the workaround:
#
# 1. We can keep using the previously defined Makefile commands that doesn't
# include `-multi` suffix as before.
#
# 2. Assuming we have setup docker/dockerx properly to build in arm64
# architectures as well, then we can build and publish such images via the
# `-multi` suffix without needing a local registry.
#
# 3. If we get dedicated arm64 runners, we can test everything separately
# without needing to update this Makefile, and if all tests succeeds we can
# do a publish job that creates a multi-platform image for us.
#
build-multi/%: DOCKER_BUILD_ARGS?=
build-multi/%: ## build the latest image for a stack on both amd64 and arm64
@echo "::group::Build $(OWNER)/$(notdir $@) (system's architecture)"
docker buildx build $(DOCKER_BUILD_ARGS) -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --load
@echo -n "Built image size: "
@docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}"
@echo "::endgroup::"
@echo "::group::Build $(OWNER)/$(notdir $@) (amd64,arm64)"
docker buildx build $(DOCKER_BUILD_ARGS) -t build-multi-tmp-cache/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64,linux/arm64"
@echo "::endgroup::"
build-all-multi: $(foreach I, $(MULTI_IMAGES), build-multi/$(I)) $(foreach I, $(AMD64_ONLY_IMAGES), build/$(I)) ## build all stacks
check-outdated/%: ## check the outdated mamba/conda packages in a stack and produce a report (experimental)
@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest tests/base-notebook/test_outdated.py
@@ -176,13 +107,6 @@ push/%: ## push all tags for a jupyter image
@echo "::endgroup::"
push-all: $(foreach I, $(ALL_IMAGES), push/$(I)) ## push all tagged images
push-multi/%: DOCKER_BUILD_ARGS?=
push-multi/%: ## push all tags for a jupyter image that support multiple architectures
@echo "::group::Push $(OWNER)/$(notdir $@) (amd64,arm64)"
docker buildx build $(DOCKER_BUILD_ARGS) $($(subst -,_,$(notdir $@))_EXTRA_TAG_ARGS) -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64,linux/arm64" --push
@echo "::endgroup::"
push-all-multi: $(foreach I, $(MULTI_IMAGES), push-multi/$(I)) $(foreach I, $(AMD64_ONLY_IMAGES), push/$(I)) ## push all tagged images
run-shell/%: ## run a bash in interactive mode in a stack

11
README.md
View File

@@ -115,14 +115,9 @@ This change is tracked in the issue [#1217](https://github.com/jupyter/docker-st
## CPU Architectures
All published containers support amd64 (x86_64) and aarch64, except for `datascience-notebook` and `tensorflow-notebook`, which only support amd64 for now.
We published containers both for `amd64` (`x86_64`) and `aarch64`, except for `datascience-notebook` and `tensorflow-notebook`, which only support amd64 for now.
### Caveats for arm64 images
- The manifests we publish in this project's wiki as well as the image tags for
the multi-platform images that also support arm, are all based on the amd64
version even though details about the installed packages versions could differ
between architectures. For the status about this, see
[#1401](https://github.com/jupyter/docker-stacks/issues/1401).
- Only the amd64 images are actively tested currently. For the status about
this, see [#1402](https://github.com/jupyter/docker-stacks/issues/1402).
- We do not create multi-platform images
- All arm64 images have `aarch64-` tag prefix, for example `jupyter/base-notebook:aarch64-python-3.9.6`

10
docs/contributing/tests.md
View File

@@ -5,11 +5,7 @@ We greatly appreciate pull requests that extend the automated tests that vet the
## How the Tests Work
A [GitHub Action workflow](https://github.com/jupyter/docker-stacks/blob/master/.github/workflows/docker.yml)
runs the following commands against pull requests submitted to the `jupyter/docker-stacks` repository:
1. `make build-all-multi` - which builds all the Docker images
2. `make test-all` - which tests the newly created Docker images
This `make` command builds and then tests every docker image.
runs tests against pull requests submitted to the `jupyter/docker-stacks` repository:
We use `pytest` module to run tests on the image.
`conftest.py` and `pytest.ini` in the `tests` folder define the environment in which tests are run.
@@ -40,8 +36,8 @@ Please follow the process below to add new tests:
If you use `make`, call:
```bash
make build/somestack-notebook
make test/somestack-notebook
make build/<somestack>-notebook
make test/<somestack>-notebook
```
3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)

3
tagging/tag_image.py
View File

@@ -19,9 +19,6 @@ def tag_image(short_image_name: str, owner: str, tags_prefix: str) -> None:
"""
Tags <owner>/<short_image_name>:latest with the tags reported by all taggers
for the given image.
Tags are in a GitHub Actions environment also saved to environment variables
in a format making it easy to append them.
"""
LOGGER.info(f"Tagging image: {short_image_name}")
taggers, _ = get_taggers_and_manifests(short_image_name)

1
tests/base-notebook/test_package_managers.py
View File

@@ -27,7 +27,6 @@ def test_package_manager(
LOGGER.info(
f"Test that the package manager {package_manager} is working properly ..."
)
# Increased timout to make aarch64 images tests work in QEMU
container.run_and_wait(
timeout=30,
tty=True,