hazza/docker-stacks
1
0
Fork 0
mirror of https://github.com/jupyter/docker-stacks.git synced 2025-10-18 07:22:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Test payspark import

Browse Source
This commit is contained in:
Peter Parente
2020-02-15 18:53:58 -05:00
parent 3aa61f94c2
commit 4a8b58a41b
2 changed files with 13 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pyspark-notebook/Dockerfile
View File

@@ -40,7 +40,7 @@ ENV SPARK_HOME=/usr/local/spark
ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip \ ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip \
MESOS_NATIVE_LIBRARY=/usr/local/lib/libmesos.so \ MESOS_NATIVE_LIBRARY=/usr/local/lib/libmesos.so \
SPARK_OPTS="--driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info" \ SPARK_OPTS="--driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info" \
PATH=$PATH:$SPARH_HOME/bin PATH=$PATH:$SPARK_HOME/bin
USER $NB_UID USER $NB_UID

13
pyspark-notebook/test/test_spark.py
View File

@@ -16,4 +16,15 @@ def test_spark_shell(container):
c.wait(timeout=30) c.wait(timeout=30)
logs = c.logs(stdout=True).decode('utf-8') logs = c.logs(stdout=True).decode('utf-8')
LOGGER.debug(logs) LOGGER.debug(logs)
assert 'res0: Int = 2' in logs assert 'res0: Int = 2' in logs
def test_pyspark(container):
"""PySpark should be in the Python path"""
c = container.run(
tty=True,
command=['start.sh', 'python', '-c', '"import pyspark"']
)
rv = c.wait(timeout=30)
assert rv == 0 or rv["StatusCode"] == 0
logs = c.logs(stdout=True).decode('utf-8')
LOGGER.debug(logs)