hazza/docker-stacks
1
0
Fork 0
mirror of https://github.com/jupyter/docker-stacks.git synced 2025-10-12 04:22:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Start using spark4-preview versions (#2159)

Browse Source 
* Start using spark4-preview versions

* Allow to download preview versions

* Expect warnings in spark

* Disable local_sparklyr test for now
This commit is contained in:
Ayaz Salikhov
2024-10-22 11:47:45 +01:00
committed by GitHub
parent 2f1cf2a2ef
commit b744182207
6 changed files with 27 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tests/pyspark-notebook/test_spark.py
View File

@@ -3,12 +3,20 @@
import logging
from tests.conftest import TrackedContainer
from tests.run_command import run_command
LOGGER = logging.getLogger(__name__)
def test_spark_shell(container: TrackedContainer) -> None:
"""Checking if Spark (spark-shell) is running properly"""
logs = run_command(container, 'spark-shell <<< "1+1"', timeout=60)
logs = container.run_and_wait(
timeout=60,
no_warnings=False,
tty=True,
command=["bash", "-c", 'spark-shell <<< "1+1"'],
)
warnings = TrackedContainer.get_warnings(logs)
assert len(warnings) == 1
assert "Using incubator modules: jdk.incubator.vector" in warnings[0]
assert "res0: Int = 2" in logs, "spark-shell does not work"