diff --git a/Makefile b/Makefile index 54b5f89d..46ebc8b4 100644 --- a/Makefile +++ b/Makefile @@ -7,21 +7,6 @@ SHELL:=bash OWNER?=jupyter # Need to list the images in build dependency order - -# Images supporting the following architectures: -# - linux/amd64 -# - linux/arm64 -MULTI_IMAGES:= \ - base-notebook \ - minimal-notebook \ - r-notebook \ - scipy-notebook \ - pyspark-notebook \ - all-spark-notebook -# Images that can only be built on the amd64 architecture (aka. x86_64) -AMD64_ONLY_IMAGES:= \ - datascience-notebook \ - tensorflow-notebook # All of the images ALL_IMAGES:= \ base-notebook \ @@ -42,7 +27,7 @@ export DOCKER_BUILDKIT:=1 help: @echo "jupyter/docker-stacks" @echo "=====================" - @echo "Replace % with a stack directory name (e.g., make build-multi/minimal-notebook)" + @echo "Replace % with a stack directory name (e.g., make build/minimal-notebook)" @echo @grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -57,60 +42,6 @@ build/%: ## build the latest image for a stack using the system's architecture @echo "::endgroup::" build-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks -# Limitations on docker buildx build (using docker/buildx 0.5.1): -# -# 1. Can't --load and --push at the same time -# -# 2. Can't --load multiple platforms -# -# What does it mean to --load? -# -# - It means that the built image can be referenced by `docker` CLI, for example -# when using the `docker tag` or `docker push` commands. -# -# Workarounds due to limitations: -# -# 1. We always build a dedicated image using the current system architecture -# named as OWNER/-notebook so we always can reference that image no -# matter what during tests etc. -# -# 2. We always also build a multi-platform image during build-multi that will be -# inaccessible with `docker tag` and `docker push` etc, but this will help us -# test the build on the different platform and provide cached layers for -# later. -# -# 3. We let push-multi refer to rebuilding a multi image with `--push`. -# -# We can rely on the cached layer from build-multi now even though we never -# tagged the multi image. -# -# Outcomes of the workaround: -# -# 1. We can keep using the previously defined Makefile commands that doesn't -# include `-multi` suffix as before. -# -# 2. Assuming we have setup docker/dockerx properly to build in arm64 -# architectures as well, then we can build and publish such images via the -# `-multi` suffix without needing a local registry. -# -# 3. If we get dedicated arm64 runners, we can test everything separately -# without needing to update this Makefile, and if all tests succeeds we can -# do a publish job that creates a multi-platform image for us. -# -build-multi/%: DOCKER_BUILD_ARGS?= -build-multi/%: ## build the latest image for a stack on both amd64 and arm64 - @echo "::group::Build $(OWNER)/$(notdir $@) (system's architecture)" - docker buildx build $(DOCKER_BUILD_ARGS) -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --load - @echo -n "Built image size: " - @docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}" - @echo "::endgroup::" - - @echo "::group::Build $(OWNER)/$(notdir $@) (amd64,arm64)" - docker buildx build $(DOCKER_BUILD_ARGS) -t build-multi-tmp-cache/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64,linux/arm64" - @echo "::endgroup::" -build-all-multi: $(foreach I, $(MULTI_IMAGES), build-multi/$(I)) $(foreach I, $(AMD64_ONLY_IMAGES), build/$(I)) ## build all stacks - - check-outdated/%: ## check the outdated mamba/conda packages in a stack and produce a report (experimental) @TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest tests/base-notebook/test_outdated.py @@ -176,13 +107,6 @@ push/%: ## push all tags for a jupyter image @echo "::endgroup::" push-all: $(foreach I, $(ALL_IMAGES), push/$(I)) ## push all tagged images -push-multi/%: DOCKER_BUILD_ARGS?= -push-multi/%: ## push all tags for a jupyter image that support multiple architectures - @echo "::group::Push $(OWNER)/$(notdir $@) (amd64,arm64)" - docker buildx build $(DOCKER_BUILD_ARGS) $($(subst -,_,$(notdir $@))_EXTRA_TAG_ARGS) -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64,linux/arm64" --push - @echo "::endgroup::" -push-all-multi: $(foreach I, $(MULTI_IMAGES), push-multi/$(I)) $(foreach I, $(AMD64_ONLY_IMAGES), push/$(I)) ## push all tagged images - run-shell/%: ## run a bash in interactive mode in a stack diff --git a/README.md b/README.md index 5e933051..c04123d5 100644 --- a/README.md +++ b/README.md @@ -115,14 +115,9 @@ This change is tracked in the issue [#1217](https://github.com/jupyter/docker-st ## CPU Architectures -All published containers support amd64 (x86_64) and aarch64, except for `datascience-notebook` and `tensorflow-notebook`, which only support amd64 for now. +We published containers both for `amd64` (`x86_64`) and `aarch64`, except for `datascience-notebook` and `tensorflow-notebook`, which only support amd64 for now. ### Caveats for arm64 images -- The manifests we publish in this project's wiki as well as the image tags for - the multi-platform images that also support arm, are all based on the amd64 - version even though details about the installed packages versions could differ - between architectures. For the status about this, see - [#1401](https://github.com/jupyter/docker-stacks/issues/1401). -- Only the amd64 images are actively tested currently. For the status about - this, see [#1402](https://github.com/jupyter/docker-stacks/issues/1402). +- We do not create multi-platform images +- All arm64 images have `aarch64-` tag prefix, for example `jupyter/base-notebook:aarch64-python-3.9.6` diff --git a/docs/contributing/tests.md b/docs/contributing/tests.md index fff6c578..87198a98 100644 --- a/docs/contributing/tests.md +++ b/docs/contributing/tests.md @@ -5,11 +5,7 @@ We greatly appreciate pull requests that extend the automated tests that vet the ## How the Tests Work A [GitHub Action workflow](https://github.com/jupyter/docker-stacks/blob/master/.github/workflows/docker.yml) -runs the following commands against pull requests submitted to the `jupyter/docker-stacks` repository: - -1. `make build-all-multi` - which builds all the Docker images -2. `make test-all` - which tests the newly created Docker images - This `make` command builds and then tests every docker image. +runs tests against pull requests submitted to the `jupyter/docker-stacks` repository: We use `pytest` module to run tests on the image. `conftest.py` and `pytest.ini` in the `tests` folder define the environment in which tests are run. @@ -40,8 +36,8 @@ Please follow the process below to add new tests: If you use `make`, call: ```bash - make build/somestack-notebook - make test/somestack-notebook + make build/-notebook + make test/-notebook ``` 3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request) diff --git a/tagging/tag_image.py b/tagging/tag_image.py index 629930bb..775baad4 100755 --- a/tagging/tag_image.py +++ b/tagging/tag_image.py @@ -19,9 +19,6 @@ def tag_image(short_image_name: str, owner: str, tags_prefix: str) -> None: """ Tags /:latest with the tags reported by all taggers for the given image. - - Tags are in a GitHub Actions environment also saved to environment variables - in a format making it easy to append them. """ LOGGER.info(f"Tagging image: {short_image_name}") taggers, _ = get_taggers_and_manifests(short_image_name) diff --git a/tests/base-notebook/test_package_managers.py b/tests/base-notebook/test_package_managers.py index dd543839..2ca16ec7 100644 --- a/tests/base-notebook/test_package_managers.py +++ b/tests/base-notebook/test_package_managers.py @@ -27,7 +27,6 @@ def test_package_manager( LOGGER.info( f"Test that the package manager {package_manager} is working properly ..." ) - # Increased timout to make aarch64 images tests work in QEMU container.run_and_wait( timeout=30, tty=True,