Better tests directory structure (#2231)

CHANGELOG.md

@@ -8,6 +8,7 @@ All image manifests can be found in [the wiki](https://github.com/jupyter/docker
 Affected: all images.
 
 - **Non-breaking:** Better tagging directory structure ([#2228](https://github.com/jupyter/docker-stacks/pull/2228)).
+- **Non-breaking:** Better testing directory structure ([#2231](https://github.com/jupyter/docker-stacks/pull/2231)).
 
 ## 2025-02-18
 

tests/conftest.py

@@ -1,33 +1,15 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
-import logging
 import os
-import socket
-from contextlib import closing
-from typing import Any
+from collections.abc import Generator
 
 import docker
 import pytest  # type: ignore
 import requests
-from docker.models.containers import Container
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
-LOGGER = logging.getLogger(__name__)
-
-
-def find_free_port() -> str:
-    """Returns the available host port. Can be called in multiple threads/processes."""
-    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
-        s.bind(("", 0))
-        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        return s.getsockname()[1]  # type: ignore
-
-
-def get_health(container: Container) -> str:
-    api_client = docker.APIClient()
-    inspect_results = api_client.inspect_container(container.name)
-    return inspect_results["State"]["Health"]["Status"]  # type: ignore
+from tests.utils.tracked_container import TrackedContainer
 
 
 @pytest.fixture(scope="session")
@@ -52,95 +34,10 @@ def image_name() -> str:
     return os.environ["TEST_IMAGE"]
 
 
-class TrackedContainer:
-    """Wrapper that collects docker container configuration and delays
-    container creation/execution.
-
-    Parameters
-    ----------
-    docker_client: docker.DockerClient
-        Docker client instance
-    image_name: str
-        Name of the docker image to launch
-    **kwargs: dict, optional
-        Default keyword arguments to pass to docker.DockerClient.containers.run
-    """
-
-    def __init__(
-        self,
-        docker_client: docker.DockerClient,
-        image_name: str,
-        **kwargs: Any,
-    ):
-        self.container: Container | None = None
-        self.docker_client: docker.DockerClient = docker_client
-        self.image_name: str = image_name
-        self.kwargs: Any = kwargs
-
-    def run_detached(self, **kwargs: Any) -> Container:
-        """Runs a docker container using the pre-configured image name
-        and a mix of the pre-configured container options and those passed
-        to this method.
-
-        Keeps track of the docker.Container instance spawned to kill it
-        later.
-
-        Parameters
-        ----------
-        **kwargs: dict, optional
-            Keyword arguments to pass to docker.DockerClient.containers.run
-            extending and/or overriding key/value pairs passed to the constructor
-
-        Returns
-        -------
-        docker.Container
-        """
-        all_kwargs = self.kwargs | kwargs
-        LOGGER.info(f"Running {self.image_name} with args {all_kwargs} ...")
-        self.container = self.docker_client.containers.run(
-            self.image_name,
-            **all_kwargs,
-        )
-        return self.container
-
-    def run_and_wait(
-        self,
-        timeout: int,
-        no_warnings: bool = True,
-        no_errors: bool = True,
-        no_failure: bool = True,
-        **kwargs: Any,
-    ) -> str:
-        running_container = self.run_detached(**kwargs)
-        rv = running_container.wait(timeout=timeout)
-        logs = running_container.logs().decode("utf-8")
-        assert isinstance(logs, str)
-        LOGGER.debug(logs)
-        assert no_warnings == (not self.get_warnings(logs))
-        assert no_errors == (not self.get_errors(logs))
-        assert no_failure == (rv["StatusCode"] == 0)
-        return logs
-
-    @staticmethod
-    def get_errors(logs: str) -> list[str]:
-        return TrackedContainer._lines_starting_with(logs, "ERROR")
-
-    @staticmethod
-    def get_warnings(logs: str) -> list[str]:
-        return TrackedContainer._lines_starting_with(logs, "WARNING")
-
-    @staticmethod
-    def _lines_starting_with(logs: str, pattern: str) -> list[str]:
-        return [line for line in logs.splitlines() if line.startswith(pattern)]
-
-    def remove(self) -> None:
-        """Kills and removes the tracked docker container."""
-        if self.container:
-            self.container.remove(force=True)
-
-
 @pytest.fixture(scope="function")
-def container(docker_client: docker.DockerClient, image_name: str) -> Container:
+def container(
+    docker_client: docker.DockerClient, image_name: str
+) -> Generator[TrackedContainer]:
     """Notebook container with initial configuration appropriate for testing
     (e.g., HTTP port exposed to the host for HTTP calls).
 

tests/hierarchy/images_hierarchy.py

@@ -3,6 +3,9 @@
 from pathlib import Path
 
 THIS_DIR = Path(__file__).parent.resolve()
+IMAGE_SPECIFIC_TESTS_DIR = THIS_DIR.parent / "image_specific_tests"
+
+assert IMAGE_SPECIFIC_TESTS_DIR.exists(), f"{IMAGE_SPECIFIC_TESTS_DIR} does not exist."
 
 # Please, take a look at the hierarchy of the images here:
 # https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#image-relationships
@@ -28,6 +31,8 @@ def get_test_dirs(
         return []
 
     test_dirs = get_test_dirs(ALL_IMAGES[short_image_name])
-    if (current_image_tests_dir := THIS_DIR / short_image_name).exists():
+    if (
+        current_image_tests_dir := IMAGE_SPECIFIC_TESTS_DIR / short_image_name
+    ).exists():
         test_dirs.append(current_image_tests_dir)
     return test_dirs

tests/image_specific_tests/all-spark-notebook/test_spark_notebooks.py

@@ -5,7 +5,7 @@ from pathlib import Path
 
 import pytest  # type: ignore
 
-from tests.conftest import TrackedContainer
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 THIS_DIR = Path(__file__).parent.resolve()

tests/image_specific_tests/base-notebook/test_container_options.py

@@ -6,7 +6,8 @@ import time
 import pytest  # type: ignore
 import requests
 
-from tests.conftest import TrackedContainer, find_free_port
+from tests.utils.find_free_port import find_free_port
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 

tests/image_specific_tests/base-notebook/test_healthcheck.py

@@ -5,7 +5,8 @@ import time
 
 import pytest  # type: ignore
 
-from tests.conftest import TrackedContainer, get_health
+from tests.utils.get_container_health import get_health
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 

tests/image_specific_tests/base-notebook/test_notebook.py

@@ -2,7 +2,8 @@
 # Distributed under the terms of the Modified BSD License.
 import requests
 
-from tests.conftest import TrackedContainer, find_free_port
+from tests.utils.find_free_port import find_free_port
+from tests.utils.tracked_container import TrackedContainer
 
 
 def test_secured_server(

tests/image_specific_tests/base-notebook/test_pandoc.py

@@ -2,7 +2,7 @@
 # Distributed under the terms of the Modified BSD License.
 import logging
 
-from tests.conftest import TrackedContainer
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 

tests/image_specific_tests/base-notebook/test_start_container.py

@@ -6,7 +6,8 @@ import time
 import pytest  # type: ignore
 import requests
 
-from tests.conftest import TrackedContainer, find_free_port
+from tests.utils.find_free_port import find_free_port
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 

tests/image_specific_tests/datascience-notebook/test_julia_datascience.py

@@ -1,7 +1,7 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
-from tests.conftest import TrackedContainer
-from tests.run_command import run_command
+from tests.utils.run_command import run_command
+from tests.utils.tracked_container import TrackedContainer
 
 
 def test_julia(container: TrackedContainer) -> None:

tests/image_specific_tests/datascience-notebook/test_mimetypes.py

@@ -1,7 +1,7 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
-from tests.conftest import TrackedContainer
-from tests.R_mimetype_check import check_r_mimetypes
+from tests.shared_checks.R_mimetype_check import check_r_mimetypes
+from tests.utils.tracked_container import TrackedContainer
 
 
 def test_mimetypes(container: TrackedContainer) -> None:

tests/image_specific_tests/datascience-notebook/test_pluto_datascience.py

@@ -2,8 +2,8 @@
 # Distributed under the terms of the Modified BSD License.
 import requests
 
-from tests.conftest import TrackedContainer
-from tests.pluto_check import check_pluto_proxy
+from tests.shared_checks.pluto_check import check_pluto_proxy
+from tests.utils.tracked_container import TrackedContainer
 
 
 def test_pluto_proxy(

tests/image_specific_tests/docker-stacks-foundation/test_outdated.py

@@ -4,8 +4,8 @@ import logging
 
 import pytest  # type: ignore
 
-from tests.conftest import TrackedContainer
-from tests.package_helper import CondaPackageHelper
+from tests.utils.conda_package_helper import CondaPackageHelper
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 

tests/image_specific_tests/docker-stacks-foundation/test_package_managers.py

@@ -2,8 +2,8 @@
 # Distributed under the terms of the Modified BSD License.
 import pytest  # type: ignore
 
-from tests.conftest import TrackedContainer
-from tests.run_command import run_command
+from tests.utils.run_command import run_command
+from tests.utils.tracked_container import TrackedContainer
 
 
 @pytest.mark.parametrize(

tests/image_specific_tests/docker-stacks-foundation/test_packages.py

@@ -44,8 +44,8 @@ from collections.abc import Callable, Iterable
 
 import pytest  # type: ignore
 
-from tests.conftest import TrackedContainer
-from tests.package_helper import CondaPackageHelper
+from tests.utils.conda_package_helper import CondaPackageHelper
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 

tests/image_specific_tests/docker-stacks-foundation/test_python_version.py

@@ -2,7 +2,7 @@
 # Distributed under the terms of the Modified BSD License.
 import logging
 
-from tests.conftest import TrackedContainer
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 EXPECTED_PYTHON_VERSION = "3.12"

tests/image_specific_tests/docker-stacks-foundation/test_run_hooks.py

@@ -3,7 +3,7 @@
 import logging
 from pathlib import Path
 
-from tests.conftest import TrackedContainer
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 THIS_DIR = Path(__file__).parent.resolve()

tests/image_specific_tests/docker-stacks-foundation/test_units.py

@@ -2,8 +2,8 @@
 # Distributed under the terms of the Modified BSD License.
 import logging
 
-from tests.conftest import TrackedContainer
-from tests.images_hierarchy import get_test_dirs
+from tests.hierarchy.images_hierarchy import get_test_dirs
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 

tests/image_specific_tests/docker-stacks-foundation/test_user_options.py

@@ -6,7 +6,7 @@ import time
 
 import pytest  # type: ignore
 
-from tests.conftest import TrackedContainer
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 

tests/image_specific_tests/julia-notebook/test_julia.py

@@ -1,7 +1,7 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
-from tests.conftest import TrackedContainer
-from tests.run_command import run_command
+from tests.utils.run_command import run_command
+from tests.utils.tracked_container import TrackedContainer
 
 
 def test_julia(container: TrackedContainer) -> None:

tests/image_specific_tests/julia-notebook/test_pluto.py

@@ -2,8 +2,8 @@
 # Distributed under the terms of the Modified BSD License.
 import requests
 
-from tests.conftest import TrackedContainer
-from tests.pluto_check import check_pluto_proxy
+from tests.shared_checks.pluto_check import check_pluto_proxy
+from tests.utils.tracked_container import TrackedContainer
 
 
 def test_pluto_proxy(

tests/image_specific_tests/minimal-notebook/test_nbconvert.py

@@ -5,7 +5,7 @@ from pathlib import Path
 
 import pytest  # type: ignore
 
-from tests.conftest import TrackedContainer
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 THIS_DIR = Path(__file__).parent.resolve()

tests/image_specific_tests/pyspark-notebook/test_spark.py

@@ -2,7 +2,7 @@
 # Distributed under the terms of the Modified BSD License.
 import logging
 
-from tests.conftest import TrackedContainer
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 

tests/image_specific_tests/r-notebook/test_R_mimetypes.py

@@ -1,7 +1,7 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
-from tests.conftest import TrackedContainer
-from tests.R_mimetype_check import check_r_mimetypes
+from tests.shared_checks.R_mimetype_check import check_r_mimetypes
+from tests.utils.tracked_container import TrackedContainer
 
 
 def test_mimetypes(container: TrackedContainer) -> None:

tests/image_specific_tests/scipy-notebook/test_cython.py

@@ -2,7 +2,7 @@
 # Distributed under the terms of the Modified BSD License.
 from pathlib import Path
 
-from tests.conftest import TrackedContainer
+from tests.utils.tracked_container import TrackedContainer
 
 THIS_DIR = Path(__file__).parent.resolve()
 

tests/image_specific_tests/scipy-notebook/test_extensions.py

@@ -4,7 +4,7 @@ import logging
 
 import pytest  # type: ignore
 
-from tests.conftest import TrackedContainer
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 

tests/image_specific_tests/scipy-notebook/test_matplotlib.py

@@ -5,7 +5,7 @@ from pathlib import Path
 
 import pytest  # type: ignore
 
-from tests.conftest import TrackedContainer
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 THIS_DIR = Path(__file__).parent.resolve()

tests/run_tests.py

@@ -6,7 +6,7 @@ import logging
 
 import plumbum
 
-from tests.images_hierarchy import get_test_dirs
+from tests.hierarchy.images_hierarchy import get_test_dirs
 
 python3 = plumbum.local["python3"]
 

tests/shared_checks/R_mimetype_check.py

@@ -2,7 +2,7 @@
 # Distributed under the terms of the Modified BSD License.
 import logging
 
-from tests.conftest import TrackedContainer
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 

tests/shared_checks/pluto_check.py

@@ -6,7 +6,8 @@ import time
 
 import requests
 
-from tests.conftest import TrackedContainer, find_free_port
+from tests.utils.find_free_port import find_free_port
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 

tests/utils/conda_package_helper.py

@@ -32,7 +32,7 @@ from typing import Any
 from docker.models.containers import Container
 from tabulate import tabulate
 
-from tests.conftest import TrackedContainer
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 

tests/utils/find_free_port.py

@@ -0,0 +1,12 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import socket
+from contextlib import closing
+
+
+def find_free_port() -> str:
+    """Returns the available host port. Can be called in multiple threads/processes."""
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        s.bind(("", 0))
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        return s.getsockname()[1]  # type: ignore

tests/utils/get_container_health.py

@@ -0,0 +1,10 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import docker
+from docker.models.containers import Container
+
+
+def get_health(container: Container) -> str:
+    api_client = docker.APIClient()
+    inspect_results = api_client.inspect_container(container.name)
+    return inspect_results["State"]["Health"]["Status"]  # type: ignore

tests/utils/run_command.py

@@ -2,7 +2,7 @@
 # Distributed under the terms of the Modified BSD License.
 import logging
 
-from tests.conftest import TrackedContainer
+from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 

tests/utils/tracked_container.py

@@ -0,0 +1,96 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import logging
+from typing import Any
+
+import docker
+from docker.models.containers import Container
+
+LOGGER = logging.getLogger(__name__)
+
+
+class TrackedContainer:
+    """Wrapper that collects docker container configuration and delays
+    container creation/execution.
+
+    Parameters
+    ----------
+    docker_client: docker.DockerClient
+        Docker client instance
+    image_name: str
+        Name of the docker image to launch
+    **kwargs: dict, optional
+        Default keyword arguments to pass to docker.DockerClient.containers.run
+    """
+
+    def __init__(
+        self,
+        docker_client: docker.DockerClient,
+        image_name: str,
+        **kwargs: Any,
+    ):
+        self.container: Container | None = None
+        self.docker_client: docker.DockerClient = docker_client
+        self.image_name: str = image_name
+        self.kwargs: Any = kwargs
+
+    def run_detached(self, **kwargs: Any) -> Container:
+        """Runs a docker container using the pre-configured image name
+        and a mix of the pre-configured container options and those passed
+        to this method.
+
+        Keeps track of the docker.Container instance spawned to kill it
+        later.
+
+        Parameters
+        ----------
+        **kwargs: dict, optional
+            Keyword arguments to pass to docker.DockerClient.containers.run
+            extending and/or overriding key/value pairs passed to the constructor
+
+        Returns
+        -------
+        docker.Container
+        """
+        all_kwargs = self.kwargs | kwargs
+        LOGGER.info(f"Running {self.image_name} with args {all_kwargs} ...")
+        self.container = self.docker_client.containers.run(
+            self.image_name,
+            **all_kwargs,
+        )
+        return self.container
+
+    def run_and_wait(
+        self,
+        timeout: int,
+        no_warnings: bool = True,
+        no_errors: bool = True,
+        no_failure: bool = True,
+        **kwargs: Any,
+    ) -> str:
+        running_container = self.run_detached(**kwargs)
+        rv = running_container.wait(timeout=timeout)
+        logs = running_container.logs().decode("utf-8")
+        assert isinstance(logs, str)
+        LOGGER.debug(logs)
+        assert no_warnings == (not self.get_warnings(logs))
+        assert no_errors == (not self.get_errors(logs))
+        assert no_failure == (rv["StatusCode"] == 0)
+        return logs
+
+    @staticmethod
+    def get_errors(logs: str) -> list[str]:
+        return TrackedContainer._lines_starting_with(logs, "ERROR")
+
+    @staticmethod
+    def get_warnings(logs: str) -> list[str]:
+        return TrackedContainer._lines_starting_with(logs, "WARNING")
+
+    @staticmethod
+    def _lines_starting_with(logs: str, pattern: str) -> list[str]:
+        return [line for line in logs.splitlines() if line.startswith(pattern)]
+
+    def remove(self) -> None:
+        """Kills and removes the tracked docker container."""
+        if self.container:
+            self.container.remove(force=True)