Add a way to easily test units

all-spark-notebook/test/test_spark_notebooks.py

@@ -7,6 +7,7 @@ import pytest
 import os
 
 LOGGER = logging.getLogger(__name__)
+THIS_DIR = os.path.dirname(os.path.realpath(__file__))
 
 
 @pytest.mark.parametrize(
@@ -16,7 +17,7 @@ LOGGER = logging.getLogger(__name__)
 )
 def test_nbconvert(container, test_file):
     """Check if Spark notebooks can be executed"""
-    host_data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data")
+    host_data_dir = os.path.join(THIS_DIR, "data")
     cont_data_dir = "/home/jovyan/data"
     output_dir = "/tmp"
     timeout_ms = 600

base-notebook/test/test_package_managers.py

@@ -8,22 +8,23 @@ LOGGER = logging.getLogger(__name__)
 
 
 @pytest.mark.parametrize(
-    "package_manager, cmd",
+    "package_manager, version_arg",
     [
         ("apt", "--version"),
         ("conda", "--version"),
         ("mamba", "--version"),
         ("npm", "--version"),
-        ("pip", "--version"),
+        ("pip", "--version")
-    ],
+    ]
 )
-def test_package_manager(container, package_manager, cmd):
+def test_package_manager(container, package_manager, version_arg):
     """Test the notebook start-notebook script"""
     LOGGER.info(
         f"Test that the package manager {package_manager} is working properly ..."
     )
     c = container.run(
-        tty=True, command=["start.sh", "bash", "-c", f"{package_manager} {cmd}"]
+        tty=True,
+        command=["start.sh", "bash", "-c", f"{package_manager} {version_arg}"]
     )
     rv = c.wait(timeout=5)
     logs = c.logs(stdout=True).decode("utf-8")

base-notebook/test/test_pandoc.py

@@ -9,7 +9,8 @@ LOGGER = logging.getLogger(__name__)
 def test_pandoc(container):
     """Pandoc shall be able to convert MD to HTML."""
     c = container.run(
-        tty=True, command=["start.sh", "bash", "-c", 'echo "**BOLD**" | pandoc']
+        tty=True,
+        command=["start.sh", "bash", "-c", 'echo "**BOLD**" | pandoc']
     )
     c.wait(timeout=10)
     logs = c.logs(stdout=True).decode("utf-8")

base-notebook/test/test_start_container.py

@@ -8,7 +8,11 @@ LOGGER = logging.getLogger(__name__)
 
 
 @pytest.mark.parametrize(
-    "env,expected_server", [(["JUPYTER_ENABLE_LAB=yes"], "lab"), (None, "notebook"), ],
+    "env,expected_server",
+    [
+        (["JUPYTER_ENABLE_LAB=yes"], "lab"),
+        (None, "notebook")
+    ]
 )
 def test_start_notebook(container, http_client, env, expected_server):
     """Test the notebook start-notebook script"""

datascience-notebook/test/test_julia.py

@@ -9,7 +9,8 @@ def test_julia(container):
     """Basic julia test"""
     LOGGER.info("Test that julia is correctly installed ...")
     running_container = container.run(
-        tty=True, command=["start.sh", "bash", "-c", "sleep infinity"]
+        tty=True,
+        command=["start.sh", "bash", "-c", "sleep infinity"]
     )
     command = "julia --version"
     cmd = running_container.exec_run(command)

minimal-notebook/test/test_nbconvert.py

@@ -7,15 +7,21 @@ import pytest
 import os
 
 LOGGER = logging.getLogger(__name__)
+THIS_DIR = os.path.dirname(os.path.realpath(__file__))
 
 
-@pytest.mark.parametrize("test_file, output_format,", [
+@pytest.mark.parametrize(
-    ("notebook_math", "pdf"), ("notebook_math", "html"),
+    "test_file, output_format",
-    ("notebook_svg", "pdf"), ("notebook_svg", "html"),
+    [
-])
+        ("notebook_math", "pdf"),
+        ("notebook_math", "html"),
+        ("notebook_svg", "pdf"),
+        ("notebook_svg", "html")
+    ]
+)
 def test_nbconvert(container, test_file, output_format):
     """Check if nbconvert is able to convert a notebook file"""
-    host_data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data")
+    host_data_dir = os.path.join(THIS_DIR, "data")
     cont_data_dir = "/home/jovyan/data"
     output_dir = "/tmp"
     LOGGER.info(f"Test that the example notebook {test_file} can be converted to {output_format.upper()} ...")

pyspark-notebook/test/test_spark.py

@@ -16,15 +16,3 @@ def test_spark_shell(container):
     logs = c.logs(stdout=True).decode('utf-8')
     LOGGER.debug(logs)
     assert 'res0: Int = 2' in logs, "spark-shell does not work"
-
-
-def test_pyspark(container):
-    """PySpark should be in the Python path"""
-    c = container.run(
-        tty=True,
-        command=['start.sh', 'python', '-c', 'import pyspark']
-    )
-    rv = c.wait(timeout=30)
-    assert rv == 0 or rv["StatusCode"] == 0, "pyspark not in PYTHONPATH"
-    logs = c.logs(stdout=True).decode('utf-8')
-    LOGGER.debug(logs)

pyspark-notebook/test/units/unit_spark.py

@@ -0,0 +1,4 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import pyspark  # noqa: F401

requirements-dev.txt

@@ -1,5 +1,6 @@
 docker
 myst-parser
+packaging
 plumbum
 pre-commit
 pytest

scipy-notebook/test/test_matplotlib.py

@@ -7,23 +7,23 @@ import pytest
 import os
 
 LOGGER = logging.getLogger(__name__)
+THIS_DIR = os.path.dirname(os.path.realpath(__file__))
 
 
-@pytest.mark.parametrize("test_file,expected_file,description",
+@pytest.mark.parametrize(
-                         [
+    "test_file,expected_file,description",
-                             ("matplotlib_1.py", "test.png",
+    [
-                              "Test that matplotlib is able to plot a graph and write it as an image ..."),
+        ("matplotlib_1.py", "test.png", "Test that matplotlib is able to plot a graph and write it as an image ..."),
-                             ("matplotlib_fonts_1.py", "test_fonts.png",
+        ("matplotlib_fonts_1.py", "test_fonts.png", "Test cm-super latex labels in matplotlib ...")
-                              "Test cm-super latex labels in matplotlib ...")
+    ]
-                         ])
+)
 def test_matplotlib(container, test_file, expected_file, description):
     """Various tests performed on matplotlib
 
     - Test that matplotlib is able to plot a graph and write it as an image
     - Test matplotlib latex fonts, which depend on the cm-super package
     """
-    host_data_dir = os.path.join(os.path.dirname(
+    host_data_dir = os.path.join(THIS_DIR, "data")
-        os.path.realpath(__file__)), "data")
     cont_data_dir = "/home/jovyan/data"
     output_dir = "/tmp"
     LOGGER.info(description)

scipy-notebook/test/test_pandas.py

@@ -1,32 +0,0 @@
-# Copyright (c) Jupyter Development Team.
-# Distributed under the terms of the Modified BSD License.
-import logging
-
-import pytest
-
-LOGGER = logging.getLogger(__name__)
-
-
-@pytest.mark.parametrize(
-    "name,command_list",
-    [
-        (
-            "Sum series",
-            [
-                "import pandas as pd",
-                "import numpy as np",
-                "np.random.seed(0)",
-                "print(pd.Series(np.random.randint(0, 7, size=10)).sum())"
-            ]
-        ),
-    ],
-)
-def test_pandas(container, name, command_list):
-    """Basic pandas tests"""
-    LOGGER.info(f"Testing pandas: {name} ...")
-    command = ';'.join(command_list)
-    c = container.run(tty=True, command=["start.sh", "python", "-c", command])
-    rv = c.wait(timeout=30)
-    assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"
-    logs = c.logs(stdout=True).decode("utf-8")
-    LOGGER.debug(logs)

scipy-notebook/test/units/unit_pandas.py

@@ -0,0 +1,9 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import numpy as np
+import pandas as pd
+
+
+np.random.seed(0)
+print(pd.Series(np.random.randint(0, 7, size=10)).sum())

tensorflow-notebook/test/test_tensorflow.py

@@ -1,30 +0,0 @@
-# Copyright (c) Jupyter Development Team.
-# Distributed under the terms of the Modified BSD License.
-import logging
-
-import pytest
-
-LOGGER = logging.getLogger(__name__)
-
-
-@pytest.mark.parametrize(
-    "name,command",
-    [
-        (
-            "Hello world",
-            "import tensorflow as tf;print(tf.constant('Hello, TensorFlow'))",
-        ),
-        (
-            "Sum",
-            "import tensorflow as tf;print(tf.reduce_sum(tf.random.normal([1000, 1000])))",
-        ),
-    ],
-)
-def test_tensorflow(container, name, command):
-    """Basic tensorflow tests"""
-    LOGGER.info(f"Testing tensorflow: {name} ...")
-    c = container.run(tty=True, command=["start.sh", "python", "-c", command])
-    rv = c.wait(timeout=30)
-    assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"
-    logs = c.logs(stdout=True).decode("utf-8")
-    LOGGER.debug(logs)

tensorflow-notebook/test/units/unit_tensorflow.py

@@ -0,0 +1,7 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import tensorflow as tf
+
+
+print(tf.constant('Hello, TensorFlow'))
+print(tf.reduce_sum(tf.random.normal([1000, 1000])))

test/helpers.py

@@ -50,7 +50,8 @@ class CondaPackageHelper:
         """Start the TrackedContainer and return an instance of a running container"""
         LOGGER.info(f"Starting container {container.image_name} ...")
         return container.run(
-            tty=True, command=["start.sh", "bash", "-c", "sleep infinity"]
+            tty=True,
+            command=["start.sh", "bash", "-c", "sleep infinity"]
         )
 
     @staticmethod

test/test_units.py

@@ -0,0 +1,33 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import logging
+import os
+
+LOGGER = logging.getLogger(__name__)
+THIS_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+def test_units(container):
+    """Various units tests
+    Add a py file in the {image}/test/units dir and it will be automatically tested
+    """
+    host_data_dir = os.path.join(THIS_DIR, "../", container.image_name(), "test/units")
+    cont_data_dir = "/home/jovyan/data"
+
+    if not os.path.exists(host_data_dir):
+        LOGGER.info(f"Not found unit tests for image: {container.image_name()}")
+        return
+
+    command = "sleep infinity"
+    running_container = container.run(
+        volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}},
+        tty=True,
+        command=["start.sh", "bash", "-c", command],
+    )
+    for test_file in os.listdir(host_data_dir):
+        LOGGER.info("Running unit test: {test_file}")
+        command = f"python {cont_data_dir}/{test_file}"
+        cmd = running_container.exec_run(command)
+        assert cmd.exit_code == 0, f"Command {command} failed"
+        LOGGER.debug(cmd.output.decode("utf-8"))