hazza/docker-stacks
1
0
Fork 0
mirror of https://github.com/jupyter/docker-stacks.git synced 2025-10-12 20:42:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Start using spark4-preview versions (#2159)

Browse Source 
* Start using spark4-preview versions

* Allow to download preview versions

* Expect warnings in spark

* Disable local_sparklyr test for now
This commit is contained in:
Ayaz Salikhov
2024-10-22 11:47:45 +01:00
committed by GitHub
parent 2f1cf2a2ef
commit b744182207
6 changed files with 27 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tests/all-spark-notebook/test_spark_notebooks.py
View File

@@ -14,7 +14,7 @@ THIS_DIR = Path(__file__).parent.resolve()
@pytest.mark.flaky(retries=3, delay=1)
@pytest.mark.parametrize(
"test_file",
["issue_1168", "local_pyspark", "local_sparklyr", "local_sparkR"],
["issue_1168", "local_pyspark", "local_sparkR"],
)
def test_nbconvert(container: TrackedContainer, test_file: str) -> None:
"""Check if Spark notebooks can be executed"""
@@ -31,10 +31,14 @@ def test_nbconvert(container: TrackedContainer, test_file: str) -> None:
)
logs = container.run_and_wait(
timeout=60,
no_warnings=False,
volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}},
tty=True,
command=["bash", "-c", command],
)
warnings = TrackedContainer.get_warnings(logs)
assert len(warnings) == 1
assert "Using incubator modules: jdk.incubator.vector" in warnings[0]
expected_file = f"{output_dir}/{test_file}.md"
assert expected_file in logs, f"Expected file {expected_file} not generated"