mirror of
https://github.com/jupyter/docker-stacks.git
synced 2025-10-11 20:12:58 +00:00
Cleanup docs about multi platform builds
This commit is contained in:
78
Makefile
78
Makefile
@@ -7,21 +7,6 @@ SHELL:=bash
|
|||||||
OWNER?=jupyter
|
OWNER?=jupyter
|
||||||
|
|
||||||
# Need to list the images in build dependency order
|
# Need to list the images in build dependency order
|
||||||
|
|
||||||
# Images supporting the following architectures:
|
|
||||||
# - linux/amd64
|
|
||||||
# - linux/arm64
|
|
||||||
MULTI_IMAGES:= \
|
|
||||||
base-notebook \
|
|
||||||
minimal-notebook \
|
|
||||||
r-notebook \
|
|
||||||
scipy-notebook \
|
|
||||||
pyspark-notebook \
|
|
||||||
all-spark-notebook
|
|
||||||
# Images that can only be built on the amd64 architecture (aka. x86_64)
|
|
||||||
AMD64_ONLY_IMAGES:= \
|
|
||||||
datascience-notebook \
|
|
||||||
tensorflow-notebook
|
|
||||||
# All of the images
|
# All of the images
|
||||||
ALL_IMAGES:= \
|
ALL_IMAGES:= \
|
||||||
base-notebook \
|
base-notebook \
|
||||||
@@ -42,7 +27,7 @@ export DOCKER_BUILDKIT:=1
|
|||||||
help:
|
help:
|
||||||
@echo "jupyter/docker-stacks"
|
@echo "jupyter/docker-stacks"
|
||||||
@echo "====================="
|
@echo "====================="
|
||||||
@echo "Replace % with a stack directory name (e.g., make build-multi/minimal-notebook)"
|
@echo "Replace % with a stack directory name (e.g., make build/minimal-notebook)"
|
||||||
@echo
|
@echo
|
||||||
@grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
|
@grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
|
||||||
|
|
||||||
@@ -57,60 +42,6 @@ build/%: ## build the latest image for a stack using the system's architecture
|
|||||||
@echo "::endgroup::"
|
@echo "::endgroup::"
|
||||||
build-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks
|
build-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks
|
||||||
|
|
||||||
# Limitations on docker buildx build (using docker/buildx 0.5.1):
|
|
||||||
#
|
|
||||||
# 1. Can't --load and --push at the same time
|
|
||||||
#
|
|
||||||
# 2. Can't --load multiple platforms
|
|
||||||
#
|
|
||||||
# What does it mean to --load?
|
|
||||||
#
|
|
||||||
# - It means that the built image can be referenced by `docker` CLI, for example
|
|
||||||
# when using the `docker tag` or `docker push` commands.
|
|
||||||
#
|
|
||||||
# Workarounds due to limitations:
|
|
||||||
#
|
|
||||||
# 1. We always build a dedicated image using the current system architecture
|
|
||||||
# named as OWNER/<stack>-notebook so we always can reference that image no
|
|
||||||
# matter what during tests etc.
|
|
||||||
#
|
|
||||||
# 2. We always also build a multi-platform image during build-multi that will be
|
|
||||||
# inaccessible with `docker tag` and `docker push` etc, but this will help us
|
|
||||||
# test the build on the different platform and provide cached layers for
|
|
||||||
# later.
|
|
||||||
#
|
|
||||||
# 3. We let push-multi refer to rebuilding a multi image with `--push`.
|
|
||||||
#
|
|
||||||
# We can rely on the cached layer from build-multi now even though we never
|
|
||||||
# tagged the multi image.
|
|
||||||
#
|
|
||||||
# Outcomes of the workaround:
|
|
||||||
#
|
|
||||||
# 1. We can keep using the previously defined Makefile commands that doesn't
|
|
||||||
# include `-multi` suffix as before.
|
|
||||||
#
|
|
||||||
# 2. Assuming we have setup docker/dockerx properly to build in arm64
|
|
||||||
# architectures as well, then we can build and publish such images via the
|
|
||||||
# `-multi` suffix without needing a local registry.
|
|
||||||
#
|
|
||||||
# 3. If we get dedicated arm64 runners, we can test everything separately
|
|
||||||
# without needing to update this Makefile, and if all tests succeeds we can
|
|
||||||
# do a publish job that creates a multi-platform image for us.
|
|
||||||
#
|
|
||||||
build-multi/%: DOCKER_BUILD_ARGS?=
|
|
||||||
build-multi/%: ## build the latest image for a stack on both amd64 and arm64
|
|
||||||
@echo "::group::Build $(OWNER)/$(notdir $@) (system's architecture)"
|
|
||||||
docker buildx build $(DOCKER_BUILD_ARGS) -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --load
|
|
||||||
@echo -n "Built image size: "
|
|
||||||
@docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}"
|
|
||||||
@echo "::endgroup::"
|
|
||||||
|
|
||||||
@echo "::group::Build $(OWNER)/$(notdir $@) (amd64,arm64)"
|
|
||||||
docker buildx build $(DOCKER_BUILD_ARGS) -t build-multi-tmp-cache/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64,linux/arm64"
|
|
||||||
@echo "::endgroup::"
|
|
||||||
build-all-multi: $(foreach I, $(MULTI_IMAGES), build-multi/$(I)) $(foreach I, $(AMD64_ONLY_IMAGES), build/$(I)) ## build all stacks
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
check-outdated/%: ## check the outdated mamba/conda packages in a stack and produce a report (experimental)
|
check-outdated/%: ## check the outdated mamba/conda packages in a stack and produce a report (experimental)
|
||||||
@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest tests/base-notebook/test_outdated.py
|
@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest tests/base-notebook/test_outdated.py
|
||||||
@@ -176,13 +107,6 @@ push/%: ## push all tags for a jupyter image
|
|||||||
@echo "::endgroup::"
|
@echo "::endgroup::"
|
||||||
push-all: $(foreach I, $(ALL_IMAGES), push/$(I)) ## push all tagged images
|
push-all: $(foreach I, $(ALL_IMAGES), push/$(I)) ## push all tagged images
|
||||||
|
|
||||||
push-multi/%: DOCKER_BUILD_ARGS?=
|
|
||||||
push-multi/%: ## push all tags for a jupyter image that support multiple architectures
|
|
||||||
@echo "::group::Push $(OWNER)/$(notdir $@) (amd64,arm64)"
|
|
||||||
docker buildx build $(DOCKER_BUILD_ARGS) $($(subst -,_,$(notdir $@))_EXTRA_TAG_ARGS) -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64,linux/arm64" --push
|
|
||||||
@echo "::endgroup::"
|
|
||||||
push-all-multi: $(foreach I, $(MULTI_IMAGES), push-multi/$(I)) $(foreach I, $(AMD64_ONLY_IMAGES), push/$(I)) ## push all tagged images
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
run-shell/%: ## run a bash in interactive mode in a stack
|
run-shell/%: ## run a bash in interactive mode in a stack
|
||||||
|
11
README.md
11
README.md
@@ -115,14 +115,9 @@ This change is tracked in the issue [#1217](https://github.com/jupyter/docker-st
|
|||||||
|
|
||||||
## CPU Architectures
|
## CPU Architectures
|
||||||
|
|
||||||
All published containers support amd64 (x86_64) and aarch64, except for `datascience-notebook` and `tensorflow-notebook`, which only support amd64 for now.
|
We published containers both for `amd64` (`x86_64`) and `aarch64`, except for `datascience-notebook` and `tensorflow-notebook`, which only support amd64 for now.
|
||||||
|
|
||||||
### Caveats for arm64 images
|
### Caveats for arm64 images
|
||||||
|
|
||||||
- The manifests we publish in this project's wiki as well as the image tags for
|
- We do not create multi-platform images
|
||||||
the multi-platform images that also support arm, are all based on the amd64
|
- All arm64 images have `aarch64-` tag prefix, for example `jupyter/base-notebook:aarch64-python-3.9.6`
|
||||||
version even though details about the installed packages versions could differ
|
|
||||||
between architectures. For the status about this, see
|
|
||||||
[#1401](https://github.com/jupyter/docker-stacks/issues/1401).
|
|
||||||
- Only the amd64 images are actively tested currently. For the status about
|
|
||||||
this, see [#1402](https://github.com/jupyter/docker-stacks/issues/1402).
|
|
||||||
|
@@ -5,11 +5,7 @@ We greatly appreciate pull requests that extend the automated tests that vet the
|
|||||||
## How the Tests Work
|
## How the Tests Work
|
||||||
|
|
||||||
A [GitHub Action workflow](https://github.com/jupyter/docker-stacks/blob/master/.github/workflows/docker.yml)
|
A [GitHub Action workflow](https://github.com/jupyter/docker-stacks/blob/master/.github/workflows/docker.yml)
|
||||||
runs the following commands against pull requests submitted to the `jupyter/docker-stacks` repository:
|
runs tests against pull requests submitted to the `jupyter/docker-stacks` repository:
|
||||||
|
|
||||||
1. `make build-all-multi` - which builds all the Docker images
|
|
||||||
2. `make test-all` - which tests the newly created Docker images
|
|
||||||
This `make` command builds and then tests every docker image.
|
|
||||||
|
|
||||||
We use `pytest` module to run tests on the image.
|
We use `pytest` module to run tests on the image.
|
||||||
`conftest.py` and `pytest.ini` in the `tests` folder define the environment in which tests are run.
|
`conftest.py` and `pytest.ini` in the `tests` folder define the environment in which tests are run.
|
||||||
@@ -40,8 +36,8 @@ Please follow the process below to add new tests:
|
|||||||
If you use `make`, call:
|
If you use `make`, call:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make build/somestack-notebook
|
make build/<somestack>-notebook
|
||||||
make test/somestack-notebook
|
make test/<somestack>-notebook
|
||||||
```
|
```
|
||||||
|
|
||||||
3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)
|
3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)
|
||||||
|
@@ -19,9 +19,6 @@ def tag_image(short_image_name: str, owner: str, tags_prefix: str) -> None:
|
|||||||
"""
|
"""
|
||||||
Tags <owner>/<short_image_name>:latest with the tags reported by all taggers
|
Tags <owner>/<short_image_name>:latest with the tags reported by all taggers
|
||||||
for the given image.
|
for the given image.
|
||||||
|
|
||||||
Tags are in a GitHub Actions environment also saved to environment variables
|
|
||||||
in a format making it easy to append them.
|
|
||||||
"""
|
"""
|
||||||
LOGGER.info(f"Tagging image: {short_image_name}")
|
LOGGER.info(f"Tagging image: {short_image_name}")
|
||||||
taggers, _ = get_taggers_and_manifests(short_image_name)
|
taggers, _ = get_taggers_and_manifests(short_image_name)
|
||||||
|
@@ -27,7 +27,6 @@ def test_package_manager(
|
|||||||
LOGGER.info(
|
LOGGER.info(
|
||||||
f"Test that the package manager {package_manager} is working properly ..."
|
f"Test that the package manager {package_manager} is working properly ..."
|
||||||
)
|
)
|
||||||
# Increased timout to make aarch64 images tests work in QEMU
|
|
||||||
container.run_and_wait(
|
container.run_and_wait(
|
||||||
timeout=30,
|
timeout=30,
|
||||||
tty=True,
|
tty=True,
|
||||||
|
Reference in New Issue
Block a user