From 1af2bfc0703b0ffce6950cce8fe452421a0c3d86 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 25 Mar 2025 13:00:19 +0000 Subject: [PATCH] Add check_nbconvert function and run tests for appropriate images (#2266) --- .../test_spark_notebooks.py | 48 ----------------- .../test_spark_r_nbconvert.py | 31 +++++++++++ .../minimal-notebook/test_nbconvert.py | 28 ++-------- .../data/issue_1168.ipynb | 0 .../data/local_pyspark.ipynb | 0 .../pyspark-notebook/test_spark_nbconvert.py | 31 +++++++++++ tests/shared_checks/nbconvert_check.py | 51 +++++++++++++++++++ 7 files changed, 117 insertions(+), 72 deletions(-) delete mode 100644 tests/by_image/all-spark-notebook/test_spark_notebooks.py create mode 100644 tests/by_image/all-spark-notebook/test_spark_r_nbconvert.py rename tests/by_image/{all-spark-notebook => pyspark-notebook}/data/issue_1168.ipynb (100%) rename tests/by_image/{all-spark-notebook => pyspark-notebook}/data/local_pyspark.ipynb (100%) create mode 100644 tests/by_image/pyspark-notebook/test_spark_nbconvert.py create mode 100644 tests/shared_checks/nbconvert_check.py diff --git a/tests/by_image/all-spark-notebook/test_spark_notebooks.py b/tests/by_image/all-spark-notebook/test_spark_notebooks.py deleted file mode 100644 index 985e4b3a..00000000 --- a/tests/by_image/all-spark-notebook/test_spark_notebooks.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import logging -from pathlib import Path - -import pytest # type: ignore - -from tests.utils.tracked_container import TrackedContainer - -LOGGER = logging.getLogger(__name__) -THIS_DIR = Path(__file__).parent.resolve() - - -@pytest.mark.flaky(retries=3, delay=1) -@pytest.mark.parametrize( - "test_file", - ["issue_1168", "local_pyspark", "local_sparkR"], -) -def test_nbconvert(container: TrackedContainer, test_file: str) -> None: - """Check if Spark notebooks can be executed""" - host_data_dir = THIS_DIR / "data" - cont_data_dir = "/home/jovyan/data" - output_dir = "/tmp" - conversion_timeout_ms = 5000 - LOGGER.info(f"Test that {test_file} notebook can be executed ...") - command = [ - "jupyter", - "nbconvert", - "--to", - "markdown", - f"--ExecutePreprocessor.timeout={conversion_timeout_ms}", - "--output-dir", - output_dir, - "--execute", - f"{cont_data_dir}/{test_file}.ipynb", - ] - logs = container.run_and_wait( - timeout=60, - no_warnings=False, - volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}}, - command=command, - ) - warnings = TrackedContainer.get_warnings(logs) - assert len(warnings) == 1 - assert "Using incubator modules: jdk.incubator.vector" in warnings[0] - - expected_file = f"{output_dir}/{test_file}.md" - assert expected_file in logs, f"Expected file {expected_file} not generated" diff --git a/tests/by_image/all-spark-notebook/test_spark_r_nbconvert.py b/tests/by_image/all-spark-notebook/test_spark_r_nbconvert.py new file mode 100644 index 00000000..a5fde1b4 --- /dev/null +++ b/tests/by_image/all-spark-notebook/test_spark_r_nbconvert.py @@ -0,0 +1,31 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import logging +from pathlib import Path + +import pytest # type: ignore + +from tests.shared_checks.nbconvert_check import check_nbconvert +from tests.utils.tracked_container import TrackedContainer + +LOGGER = logging.getLogger(__name__) +THIS_DIR = Path(__file__).parent.resolve() + + +@pytest.mark.flaky(retries=3, delay=1) +@pytest.mark.parametrize( + "test_file", + ["local_sparkR"], +) +@pytest.mark.parametrize("output_format", ["pdf", "html", "markdown"]) +def test_spark_r_nbconvert( + container: TrackedContainer, test_file: str, output_format: str +) -> None: + host_data_file = THIS_DIR / "data" / f"{test_file}.ipynb" + logs = check_nbconvert( + container, host_data_file, "markdown", execute=True, no_warnings=False + ) + + warnings = TrackedContainer.get_warnings(logs) + assert len(warnings) == 1 + assert "Using incubator modules: jdk.incubator.vector" in warnings[0] diff --git a/tests/by_image/minimal-notebook/test_nbconvert.py b/tests/by_image/minimal-notebook/test_nbconvert.py index 94600cb9..03b9d8f1 100644 --- a/tests/by_image/minimal-notebook/test_nbconvert.py +++ b/tests/by_image/minimal-notebook/test_nbconvert.py @@ -5,6 +5,7 @@ from pathlib import Path import pytest # type: ignore +from tests.shared_checks.nbconvert_check import check_nbconvert from tests.utils.tracked_container import TrackedContainer LOGGER = logging.getLogger(__name__) @@ -12,30 +13,9 @@ THIS_DIR = Path(__file__).parent.resolve() @pytest.mark.parametrize("test_file", ["notebook_math", "notebook_svg"]) -@pytest.mark.parametrize("output_format", ["pdf", "html"]) +@pytest.mark.parametrize("output_format", ["pdf", "html", "markdown"]) def test_nbconvert( container: TrackedContainer, test_file: str, output_format: str ) -> None: - """Check if nbconvert is able to convert a notebook file""" - host_data_dir = THIS_DIR / "data" - cont_data_dir = "/home/jovyan/data" - output_dir = "/tmp" - LOGGER.info( - f"Test that the example notebook {test_file} can be converted to {output_format} ..." - ) - command = [ - "jupyter", - "nbconvert", - f"{cont_data_dir}/{test_file}.ipynb", - "--output-dir", - output_dir, - "--to", - output_format, - ] - logs = container.run_and_wait( - timeout=30, - volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}}, - command=command, - ) - expected_file = f"{output_dir}/{test_file}.{output_format}" - assert expected_file in logs, f"Expected file {expected_file} not generated" + host_data_file = THIS_DIR / "data" / f"{test_file}.ipynb" + check_nbconvert(container, host_data_file, output_format, execute=False) diff --git a/tests/by_image/all-spark-notebook/data/issue_1168.ipynb b/tests/by_image/pyspark-notebook/data/issue_1168.ipynb similarity index 100% rename from tests/by_image/all-spark-notebook/data/issue_1168.ipynb rename to tests/by_image/pyspark-notebook/data/issue_1168.ipynb diff --git a/tests/by_image/all-spark-notebook/data/local_pyspark.ipynb b/tests/by_image/pyspark-notebook/data/local_pyspark.ipynb similarity index 100% rename from tests/by_image/all-spark-notebook/data/local_pyspark.ipynb rename to tests/by_image/pyspark-notebook/data/local_pyspark.ipynb diff --git a/tests/by_image/pyspark-notebook/test_spark_nbconvert.py b/tests/by_image/pyspark-notebook/test_spark_nbconvert.py new file mode 100644 index 00000000..b3972818 --- /dev/null +++ b/tests/by_image/pyspark-notebook/test_spark_nbconvert.py @@ -0,0 +1,31 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import logging +from pathlib import Path + +import pytest # type: ignore + +from tests.shared_checks.nbconvert_check import check_nbconvert +from tests.utils.tracked_container import TrackedContainer + +LOGGER = logging.getLogger(__name__) +THIS_DIR = Path(__file__).parent.resolve() + + +@pytest.mark.flaky(retries=3, delay=1) +@pytest.mark.parametrize( + "test_file", + ["issue_1168", "local_pyspark"], +) +@pytest.mark.parametrize("output_format", ["pdf", "html", "markdown"]) +def test_spark_nbconvert( + container: TrackedContainer, test_file: str, output_format: str +) -> None: + host_data_file = THIS_DIR / "data" / f"{test_file}.ipynb" + logs = check_nbconvert( + container, host_data_file, "markdown", execute=True, no_warnings=False + ) + + warnings = TrackedContainer.get_warnings(logs) + assert len(warnings) == 1 + assert "Using incubator modules: jdk.incubator.vector" in warnings[0] diff --git a/tests/shared_checks/nbconvert_check.py b/tests/shared_checks/nbconvert_check.py new file mode 100644 index 00000000..cf1dbc6b --- /dev/null +++ b/tests/shared_checks/nbconvert_check.py @@ -0,0 +1,51 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import logging +from pathlib import Path + +from tests.utils.tracked_container import TrackedContainer + +LOGGER = logging.getLogger(__name__) + + +def check_nbconvert( + container: TrackedContainer, + host_data_file: Path, + output_format: str, + *, + execute: bool, + no_warnings: bool = True, +) -> str: + """Check if nbconvert is able to convert a notebook file""" + cont_data_file = "/home/jovyan/data/" + host_data_file.name + + output_dir = "/tmp" + LOGGER.info( + f"Test that the example notebook {host_data_file.name} can be converted to {output_format} ..." + ) + command = [ + "jupyter", + "nbconvert", + cont_data_file, + "--output-dir", + output_dir, + "--to", + output_format, + ] + if execute: + conversion_timeout_ms = 5000 + command += [ + "--execute", + f"--ExecutePreprocessor.timeout={conversion_timeout_ms}", + ] + logs = container.run_and_wait( + timeout=60, + volumes={str(host_data_file): {"bind": cont_data_file, "mode": "ro"}}, + command=command, + no_warnings=no_warnings, + ) + output_ext = "md" if output_format == "markdown" else output_format + expected_file = f"{output_dir}/{host_data_file.stem}.{output_ext}" + assert expected_file in logs, f"Expected file {expected_file} not generated" + + return logs