Run tests on all children images

.github/workflows/docker-amd64.yml

@@ -19,9 +19,7 @@ on:
 
       - "tagging/**"
       - "test/**"
-      - "conftest.py"
       - "Makefile"
-      - "pytest.ini"
       - "requirements-dev.txt"
   push:
     branches:
@@ -41,9 +39,7 @@ on:
 
       - "tagging/**"
       - "test/**"
-      - "conftest.py"
       - "Makefile"
-      - "pytest.ini"
       - "requirements-dev.txt"
   workflow_dispatch:
 

.github/workflows/docker.yml

@@ -19,9 +19,7 @@ on:
 
       - "tagging/**"
       - "test/**"
-      - "conftest.py"
       - "Makefile"
-      - "pytest.ini"
       - "requirements-dev.txt"
   push:
     branches:
@@ -41,9 +39,7 @@ on:
 
       - "tagging/**"
       - "test/**"
-      - "conftest.py"
       - "Makefile"
-      - "pytest.ini"
       - "requirements-dev.txt"
   workflow_dispatch:
 

Makefile

@@ -205,9 +205,8 @@ run-sudo-shell/%: ## run a bash in interactive mode as root in a stack
 
 
 
-test/%: ## run tests against a stack (only common tests or common tests + specific tests)
+test/%: ## run tests against a stack
 	@echo "::group::test/$(OWNER)/$(notdir $@)"
-	@if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest --numprocesses=auto -m "not info" test; \
-	else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest --numprocesses=auto -m "not info" test $(notdir $@)/test; fi
+	tests/run_tests.py --short-image-name "$(notdir $@)" --owner "$(OWNER)"
 	@echo "::endgroup::"
 test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks

all-spark-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

base-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

datascience-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

docs/contributing/tests.md

@@ -1,34 +1,38 @@
 # Image Tests
 
-We greatly appreciate pull requests that extend the automated tests that vet the basic functionality
-of the Docker images.
+We greatly appreciate pull requests that extend the automated tests that vet the basic functionality of the Docker images.
 
 ## How the Tests Work
 
 GitHub Action executes `make build-test-all` against pull requests submitted to the `jupyter/docker-stacks` repository.
-This `make` command builds every docker image.
-After building each image, the `make` command executes `pytest` to run both image-specific tests like those in
-[base-notebook/test/](https://github.com/jupyter/docker-stacks/tree/master/base-notebook/test) and
-common tests defined in [test/](https://github.com/jupyter/docker-stacks/tree/master/test).
-Both kinds of tests make use of global [pytest fixtures](https://docs.pytest.org/en/latest/reference/fixtures.html)
-defined in the [conftest.py](https://github.com/jupyter/docker-stacks/blob/master/conftest.py) file at the root of the projects.
+This `make` command builds and then tests every docker image.
+
+We use `pytest` module to run tests on the image.
+`conftest.py` and `pytest.ini` in the `test` folder define the environment in which tests are run.
+More info on `pytest` can be found [here](https://docs.pytest.org/en/latest/contents.html).
+
+All the actual test files are located in folders like `test/<somestack>-notebook`.
+
+```{note}
+If your test is located in `test/<somestack>-notebook`, it will be run against `jupyter/<somestack>-notebook` image and against all the images inherited from this image.
+```
+
+Many tests make use of global [pytest fixtures](https://docs.pytest.org/en/latest/reference/fixtures.html)
+defined in the [conftest.py](https://github.com/jupyter/docker-stacks/blob/master/tests/conftest.py) file.
 
 ## Unit tests
 
 If you want to run a python script in one of our images, you could add a unit test.
-You can do this by creating a `<somestack>-notebook/test/units/` directory, if it doesn't already exist and put your file there.
-File in this folder will run automatically when tests are run.
-You could see an example for tensorflow package [here](https://github.com/jupyter/docker-stacks/blob/HEAD/tensorflow-notebook/test/units/unit_tensorflow.py).
+You can do this by creating a `test/<somestack>-notebook/units/` directory, if it doesn't already exist and put your file there.
+Files in this folder will run automatically when tests are run.
+You could see an example for tensorflow package [here](https://github.com/jupyter/docker-stacks/blob/HEAD/tests/tensorflow-notebook/units/unit_tensorflow.py).
 
 ## Contributing New Tests
 
 Please follow the process below to add new tests:
 
-1. If the test should run against every image built, add your test code to one of the modules in
-   [test/](https://github.com/jupyter/docker-stacks/tree/master/test) or create a new module.
-2. If your test should run against a single image, add your test code to one of the modules in
-   `some-notebook/test/` or create a new module.
-3. Build one or more images you intend to test and run the tests locally.
+1. Add your test code to one of the modules in `<somestack>-notebook/tests/` directory or create a new module.
+2. Build one or more images you intend to test and run the tests locally.
    If you use `make`, call:
 
    ```bash
@@ -36,7 +40,7 @@ Please follow the process below to add new tests:
    make test/somestack-notebook
    ```
 
-4. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)
+3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)
    (PR) with your changes.
-5. Watch for GitHub to report a build success or failure for your PR on GitHub.
-6. Discuss changes with the maintainers and address any issues running the tests on GitHub.
+4. Watch for GitHub to report a build success or failure for your PR on GitHub.
+5. Discuss changes with the maintainers and address any issues running the tests on GitHub.

minimal-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

pyspark-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

r-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

scipy-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

tensorflow-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

test/README.md

@@ -1,15 +0,0 @@
-# Docker stacks testing
-
-We test our images using `pytest` module.
-
-`conftest.py` and `pytest.ini` in the root of our repository define the environment in which tests are run.
-More info on pytest can be found [here](https://docs.pytest.org/en/latest/contents.html).
-
-There are two kinds of tests we use:
-
-- General tests - these are located in [this](https://github.com/jupyter/docker-stacks/blob/master/test) folder
-- Image specific tests - for example, [base-notebook/test](https://github.com/jupyter/docker-stacks/blob/master/base-notebook/test) folder
-
-We also have a way to easily run arbitrary python files in a container.
-This is useful for running unit tests of packages we use, so we put these files in `{image}/test/units` folder.
-An example of such a test is [unit_pandas.py](https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/test/units/unit_pandas.py).

test/test_units.py

@@ -1,35 +0,0 @@
-# Copyright (c) Jupyter Development Team.
-# Distributed under the terms of the Modified BSD License.
-
-import logging
-from pathlib import Path
-
-from conftest import TrackedContainer
-
-LOGGER = logging.getLogger(__name__)
-THIS_DIR = Path(__file__).parent.resolve()
-
-
-def test_units(container: TrackedContainer) -> None:
-    """Various units tests
-    Add a py file in the {image}/test/units dir and it will be automatically tested
-    """
-    short_image_name = container.image_name[container.image_name.rfind("/") + 1 :]
-    host_data_dir = THIS_DIR / f"../{short_image_name}/test/units"
-    LOGGER.info(f"Searching for units tests in {host_data_dir}")
-    cont_data_dir = "/home/jovyan/data"
-
-    if not host_data_dir.exists():
-        LOGGER.info(f"Not found unit tests for image: {container.image_name}")
-        return
-
-    for test_file in host_data_dir.iterdir():
-        test_file_name = test_file.name
-        LOGGER.info(f"Running unit test: {test_file_name}")
-
-        container.run_and_wait(
-            timeout=30,
-            volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}},
-            tty=True,
-            command=["start.sh", "python", f"{cont_data_dir}/{test_file_name}"],
-        )

tests/README.md

@@ -0,0 +1,3 @@
+# Docker stacks testing
+
+Please, refer to the [corresponding section of documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/tests.html) to see how the tests are run.

tests/images_hierarchy.py

@@ -0,0 +1,30 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+from pathlib import Path
+from typing import Optional
+
+THIS_DIR = Path(__file__).parent.resolve()
+
+
+ALL_IMAGES = {
+    "base-notebook": None,
+    "minimal-notebook": "base-notebook",
+    "scipy-notebook": "minimal-notebook",
+    "r-notebook": "minimal-notebook",
+    "tensorflow-notebook": "scipy-notebook",
+    "datascience-notebook": "scipy-notebook",
+    "pyspark-notebook": "scipy-notebook",
+    "all-spark-notebook": "pyspark-notebook",
+}
+
+
+def get_test_dirs(
+    short_image_name: Optional[str],
+) -> list[Path]:
+    if short_image_name is None:
+        return []  # type: ignore
+
+    test_dirs = get_test_dirs(ALL_IMAGES[short_image_name])
+    if (current_image_tests_dir := THIS_DIR / short_image_name).exists():
+        test_dirs.append(current_image_tests_dir)
+    return test_dirs

tests/run_tests.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import argparse
+import logging
+from pathlib import Path
+
+import plumbum
+from plumbum.cmd import pytest
+
+from images_hierarchy import get_test_dirs
+
+THIS_DIR = Path(__file__).parent.resolve()
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+def test_image(short_image_name: str, owner: str) -> None:
+    LOGGER.info(f"Testing image: {short_image_name}")
+    test_dirs = get_test_dirs(short_image_name)
+    LOGGER.info(f"Test dirs to be run: {test_dirs}")
+    with plumbum.local.env(TEST_IMAGE=f"{owner}/{short_image_name}"):
+        (
+            pytest[
+                "--numprocesses",
+                "auto",
+                "-m",
+                "not info",
+                THIS_DIR / "test_units.py",
+                test_dirs,
+            ]
+            & plumbum.FG
+        )
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument(
+        "--short-image-name",
+        required=True,
+        help="Short image name to run test on",
+    )
+    arg_parser.add_argument("--owner", required=True, help="Owner of the image")
+
+    args = arg_parser.parse_args()
+
+    test_image(args.short_image_name, args.owner)

tests/test_units.py

@@ -0,0 +1,39 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import logging
+
+from conftest import TrackedContainer
+from images_hierarchy import get_test_dirs
+
+LOGGER = logging.getLogger(__name__)
+
+
+def test_units(container: TrackedContainer) -> None:
+    """Various units tests
+    Add a py file in the `test/{somestack}-notebook/units` dir and it will be automatically tested
+    """
+    short_image_name = container.image_name[container.image_name.rfind("/") + 1 :]
+    LOGGER.info(f"Running unit tests for: {short_image_name}")
+
+    test_dirs = get_test_dirs(short_image_name)
+
+    for test_dir in test_dirs:
+        host_data_dir = test_dir / "units"
+        LOGGER.info(f"Searching for units tests in {host_data_dir}")
+        cont_data_dir = "/home/jovyan/data"
+
+        if not host_data_dir.exists():
+            LOGGER.info(f"Not found unit tests for image: {container.image_name}")
+            return
+
+        for test_file in host_data_dir.iterdir():
+            test_file_name = test_file.name
+            LOGGER.info(f"Running unit test: {test_file_name}")
+
+            container.run_and_wait(
+                timeout=30,
+                volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}},
+                tty=True,
+                command=["start.sh", "python", f"{cont_data_dir}/{test_file_name}"],
+            )