Merge remote-tracking branch 'upstream/master'

.github/workflows/docker-amd64.yml

@@ -18,10 +18,8 @@ on:
       - "tensorflow-notebook/**"
 
       - "tagging/**"
-      - "test/**"
+      - "tests/**"
-      - "conftest.py"
       - "Makefile"
-      - "pytest.ini"
       - "requirements-dev.txt"
   push:
     branches:
@@ -40,10 +38,8 @@ on:
       - "tensorflow-notebook/**"
 
       - "tagging/**"
-      - "test/**"
+      - "tests/**"
-      - "conftest.py"
       - "Makefile"
-      - "pytest.ini"
       - "requirements-dev.txt"
   workflow_dispatch:
 

.github/workflows/docker.yml

@@ -18,10 +18,8 @@ on:
       - "tensorflow-notebook/**"
 
       - "tagging/**"
-      - "test/**"
+      - "tests/**"
-      - "conftest.py"
       - "Makefile"
-      - "pytest.ini"
       - "requirements-dev.txt"
   push:
     branches:
@@ -40,10 +38,8 @@ on:
       - "tensorflow-notebook/**"
 
       - "tagging/**"
-      - "test/**"
+      - "tests/**"
-      - "conftest.py"
       - "Makefile"
-      - "pytest.ini"
       - "requirements-dev.txt"
   workflow_dispatch:
 

Makefile

@@ -113,7 +113,7 @@ build-all-multi: $(foreach I, $(MULTI_IMAGES), build-multi/$(I)) $(foreach I, $(
 
 
 check-outdated/%: ## check the outdated mamba/conda packages in a stack and produce a report (experimental)
-	@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test/test_outdated.py
+	@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest tests/base-notebook/test_outdated.py
 check-outdated-all: $(foreach I, $(ALL_IMAGES), check-outdated/$(I)) ## check all the stacks for outdated packages
 
 
@@ -205,9 +205,8 @@ run-sudo-shell/%: ## run a bash in interactive mode as root in a stack
 
 
 
-test/%: ## run tests against a stack (only common tests or common tests + specific tests)
+test/%: ## run tests against a stack
 	@echo "::group::test/$(OWNER)/$(notdir $@)"
-	@if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest --numprocesses=auto -m "not info" test; \
+	tests/run_tests.py --short-image-name "$(notdir $@)" --owner "$(OWNER)"
-	else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest --numprocesses=auto -m "not info" test $(notdir $@)/test; fi
 	@echo "::endgroup::"
 test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks

all-spark-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

base-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

datascience-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

docs/contributing/tests.md

@@ -1,34 +1,42 @@
 # Image Tests
 
-We greatly appreciate pull requests that extend the automated tests that vet the basic functionality
+We greatly appreciate pull requests that extend the automated tests that vet the basic functionality of the Docker images.
-of the Docker images.
 
 ## How the Tests Work
 
-GitHub Action executes `make build-test-all` against pull requests submitted to the `jupyter/docker-stacks` repository.
+A [GitHub Action workflow](https://github.com/jupyter/docker-stacks/blob/master/.github/workflows/docker.yml)
-This `make` command builds every docker image.
+runs the following commands against pull requests submitted to the `jupyter/docker-stacks` repository:
-After building each image, the `make` command executes `pytest` to run both image-specific tests like those in
+
-[base-notebook/test/](https://github.com/jupyter/docker-stacks/tree/master/base-notebook/test) and
+1. `make -C main build-all-multi` - which builds all the Docker images
-common tests defined in [test/](https://github.com/jupyter/docker-stacks/tree/master/test).
+2. `make -C main test-all` - which tests the newly created Docker images
-Both kinds of tests make use of global [pytest fixtures](https://docs.pytest.org/en/latest/reference/fixtures.html)
+   This `make` command builds and then tests every docker image.
-defined in the [conftest.py](https://github.com/jupyter/docker-stacks/blob/master/conftest.py) file at the root of the projects.
+
+We use `pytest` module to run tests on the image.
+`conftest.py` and `pytest.ini` in the `tests` folder define the environment in which tests are run.
+More info on `pytest` can be found [here](https://docs.pytest.org/en/latest/contents.html).
+
+The actual image-specific test files are located in folders like `tests/<somestack>-notebook/`.
+
+```{note}
+If your test is located in `tests/<somestack>-notebook/`, it will be run against `jupyter/<somestack>-notebook` image and against all the [images inherited from this image](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#image-relationships.
+```
+
+Many tests make use of global [pytest fixtures](https://docs.pytest.org/en/latest/reference/fixtures.html)
+defined in the [conftest.py](https://github.com/jupyter/docker-stacks/blob/master/tests/conftest.py) file.
 
 ## Unit tests
 
 If you want to run a python script in one of our images, you could add a unit test.
-You can do this by creating a `<somestack>-notebook/test/units/` directory, if it doesn't already exist and put your file there.
+You can do this by creating a `tests/<somestack>-notebook/units/` directory, if it doesn't already exist and put your file there.
-File in this folder will run automatically when tests are run.
+Files in this folder will be executed in the container when tests are run.
-You could see an example for tensorflow package [here](https://github.com/jupyter/docker-stacks/blob/HEAD/tensorflow-notebook/test/units/unit_tensorflow.py).
+You could see an [example for the TensorFlow package here](https://github.com/jupyter/docker-stacks/blob/HEAD/tests/tensorflow-notebook/units/unit_tensorflow.py).
 
 ## Contributing New Tests
 
 Please follow the process below to add new tests:
 
-1. If the test should run against every image built, add your test code to one of the modules in
+1. Add your test code to one of the modules in `tests/<somestack>-notebook/` directory or create a new module.
-   [test/](https://github.com/jupyter/docker-stacks/tree/master/test) or create a new module.
+2. Build one or more images you intend to test and run the tests locally.
-2. If your test should run against a single image, add your test code to one of the modules in
-   `some-notebook/test/` or create a new module.
-3. Build one or more images you intend to test and run the tests locally.
    If you use `make`, call:
 
    ```bash
@@ -36,7 +44,7 @@ Please follow the process below to add new tests:
    make test/somestack-notebook
    ```
 
-4. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)
+3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)
    (PR) with your changes.
-5. Watch for GitHub to report a build success or failure for your PR on GitHub.
+4. Watch for GitHub to report a build success or failure for your PR on GitHub.
-6. Discuss changes with the maintainers and address any issues running the tests on GitHub.
+5. Discuss changes with the maintainers and address any issues running the tests on GitHub.

minimal-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

pyspark-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

r-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

scipy-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

tensorflow-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

test/README.md

@@ -1,15 +0,0 @@
-# Docker stacks testing
-
-We test our images using `pytest` module.
-
-`conftest.py` and `pytest.ini` in the root of our repository define the environment in which tests are run.
-More info on pytest can be found [here](https://docs.pytest.org/en/latest/contents.html).
-
-There are two kinds of tests we use:
-
-- General tests - these are located in [this](https://github.com/jupyter/docker-stacks/blob/master/test) folder
-- Image specific tests - for example, [base-notebook/test](https://github.com/jupyter/docker-stacks/blob/master/base-notebook/test) folder
-
-We also have a way to easily run arbitrary python files in a container.
-This is useful for running unit tests of packages we use, so we put these files in `{image}/test/units` folder.
-An example of such a test is [unit_pandas.py](https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/test/units/unit_pandas.py).

test/test_units.py

@@ -1,35 +0,0 @@
-# Copyright (c) Jupyter Development Team.
-# Distributed under the terms of the Modified BSD License.
-
-import logging
-from pathlib import Path
-
-from conftest import TrackedContainer
-
-LOGGER = logging.getLogger(__name__)
-THIS_DIR = Path(__file__).parent.resolve()
-
-
-def test_units(container: TrackedContainer) -> None:
-    """Various units tests
-    Add a py file in the {image}/test/units dir and it will be automatically tested
-    """
-    short_image_name = container.image_name[container.image_name.rfind("/") + 1 :]
-    host_data_dir = THIS_DIR / f"../{short_image_name}/test/units"
-    LOGGER.info(f"Searching for units tests in {host_data_dir}")
-    cont_data_dir = "/home/jovyan/data"
-
-    if not host_data_dir.exists():
-        LOGGER.info(f"Not found unit tests for image: {container.image_name}")
-        return
-
-    for test_file in host_data_dir.iterdir():
-        test_file_name = test_file.name
-        LOGGER.info(f"Running unit test: {test_file_name}")
-
-        container.run_and_wait(
-            timeout=30,
-            volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}},
-            tty=True,
-            command=["start.sh", "python", f"{cont_data_dir}/{test_file_name}"],
-        )

tests/README.md

@@ -0,0 +1,3 @@
+# Docker stacks testing
+
+Please, refer to the [testing section of documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/tests.html) to see how the tests are run.

tests/base-notebook/test_packages.py

@@ -18,19 +18,18 @@ However it could be easily changed (or completed) to cover also dependencies `pa
 
 Example:
 
-    $ make test/datascience-notebook
+    $ make test/base-notebook
 
     # [...]
     # test/test_packages.py::test_python_packages
-    # --------------------------------------------------------------------------------------------- live log setup ----------------------------------------------------------------------------------------------
+    # tests/base-notebook/test_packages.py::test_python_packages
-    # 2020-03-08 09:56:04 [    INFO] Starting container jupyter/datascience-notebook ... (package_helper.py:51)
+    # ---------------------------------------------------------------------------------------------- live log setup ----------------------------------------------------------------------------------------------
-    # 2020-03-08 09:56:04 [    INFO] Running jupyter/datascience-notebook with args {'detach': True, 'ports': {'8888/tcp': 8888}, 'tty': True, 'command': ['start.sh', 'bash', '-c', 'sleep infinity']} ... (conftest.py:78)
+    # 2022-02-17 16:44:36 [    INFO] Starting container jupyter/base-notebook ... (package_helper.py:55)
-    # 2020-03-08 09:56:04 [    INFO] Grabing the list of manually requested packages ... (package_helper.py:76)
+    # 2022-02-17 16:44:36 [    INFO] Running jupyter/base-notebook with args {'detach': True, 'tty': True, 'command': ['start.sh', 'bash', '-c', 'sleep infinity']} ... (conftest.py:95)
-    # ---------------------------------------------------------------------------------------------- live log call ----------------------------------------------------------------------------------------------
+    # 2022-02-17 16:44:37 [    INFO] Grabing the list of manually requested packages ... (package_helper.py:83)
-    # 2020-03-08 09:56:07 [    INFO] Testing the import of packages ... (test_packages.py:125)
+    # ---------------------------------------------------------------------------------------------- live log call -----------------------------------------------------------------------------------------------
-    # 2020-03-08 09:56:07 [    INFO] Trying to import conda (test_packages.py:127)
+    # 2022-02-17 16:44:38 [    INFO] Testing the import of packages ... (test_packages.py:144)
-    # 2020-03-08 09:56:07 [    INFO] Trying to import notebook (test_packages.py:127)
+    # 2022-02-17 16:44:38 [    INFO] Trying to import mamba (test_packages.py:146)
-    # 2020-03-08 09:56:08 [    INFO] Trying to import jupyterhub (test_packages.py:127)
     # [...]
 
 """

tests/base-notebook/test_units.py

@@ -0,0 +1,39 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import logging
+
+from conftest import TrackedContainer
+from images_hierarchy import get_test_dirs
+
+LOGGER = logging.getLogger(__name__)
+
+
+def test_units(container: TrackedContainer) -> None:
+    """Various units tests
+    Add a py file in the `tests/{somestack}-notebook/units` dir and it will be automatically tested
+    """
+    short_image_name = container.image_name[container.image_name.rfind("/") + 1 :]
+    LOGGER.info(f"Running unit tests for: {short_image_name}")
+
+    test_dirs = get_test_dirs(short_image_name)
+
+    for test_dir in test_dirs:
+        host_data_dir = test_dir / "units"
+        LOGGER.info(f"Searching for units tests in {host_data_dir}")
+        cont_data_dir = "/home/jovyan/data"
+
+        if not host_data_dir.exists():
+            LOGGER.info(f"Not found unit tests for image: {container.image_name}")
+            continue
+
+        for test_file in host_data_dir.iterdir():
+            test_file_name = test_file.name
+            LOGGER.info(f"Running unit test: {test_file_name}")
+
+            container.run_and_wait(
+                timeout=30,
+                volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}},
+                tty=True,
+                command=["start.sh", "python", f"{cont_data_dir}/{test_file_name}"],
+            )

tests/images_hierarchy.py

@@ -0,0 +1,31 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+from pathlib import Path
+from typing import Optional
+
+THIS_DIR = Path(__file__).parent.resolve()
+
+# Please, take a look at the hierarchy of the images here:
+# https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#image-relationships
+ALL_IMAGES = {
+    "base-notebook": None,
+    "minimal-notebook": "base-notebook",
+    "scipy-notebook": "minimal-notebook",
+    "r-notebook": "minimal-notebook",
+    "tensorflow-notebook": "scipy-notebook",
+    "datascience-notebook": "scipy-notebook",
+    "pyspark-notebook": "scipy-notebook",
+    "all-spark-notebook": "pyspark-notebook",
+}
+
+
+def get_test_dirs(
+    short_image_name: Optional[str],
+) -> list[Path]:
+    if short_image_name is None:
+        return []  # type: ignore
+
+    test_dirs = get_test_dirs(ALL_IMAGES[short_image_name])
+    if (current_image_tests_dir := THIS_DIR / short_image_name).exists():
+        test_dirs.append(current_image_tests_dir)
+    return test_dirs

tests/run_tests.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import argparse
+import logging
+
+import plumbum
+from plumbum.cmd import pytest
+
+from images_hierarchy import get_test_dirs
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+def test_image(short_image_name: str, owner: str) -> None:
+    LOGGER.info(f"Testing image: {short_image_name}")
+    test_dirs = get_test_dirs(short_image_name)
+    LOGGER.info(f"Test dirs to be run: {test_dirs}")
+    with plumbum.local.env(TEST_IMAGE=f"{owner}/{short_image_name}"):
+        (
+            pytest[
+                "--numprocesses",
+                "auto",
+                "-m",
+                "not info",
+                test_dirs,
+            ]
+            & plumbum.FG
+        )
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument(
+        "--short-image-name",
+        required=True,
+        help="Short image name to run test on",
+    )
+    arg_parser.add_argument("--owner", required=True, help="Owner of the image")
+
+    args = arg_parser.parse_args()
+
+    test_image(args.short_image_name, args.owner)

tests/scipy-notebook/test_matplotlib.py

@@ -35,7 +35,7 @@ def test_matplotlib(
     - Test that matplotlib is able to plot a graph and write it as an image
     - Test matplotlib latex fonts, which depend on the cm-super package
     """
-    host_data_dir = THIS_DIR / "data"
+    host_data_dir = THIS_DIR / "data/matplotlib"
     cont_data_dir = "/home/jovyan/data"
     output_dir = "/tmp"
     LOGGER.info(description)