diff --git a/pyspark-notebook/Dockerfile b/pyspark-notebook/Dockerfile index 08cf2958..8e3d5a70 100644 --- a/pyspark-notebook/Dockerfile +++ b/pyspark-notebook/Dockerfile @@ -15,7 +15,7 @@ RUN apt-get -y update && \ apt-get install --no-install-recommends -y openjdk-8-jre-headless ca-certificates-java && \ rm -rf /var/lib/apt/lists/* -# Using the preferred mirror to download the file +# Using the preferred mirror to download Spark RUN cd /tmp && \ wget -q $(wget -qO- https://www.apache.org/dyn/closer.lua/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz\?as_json | \ python -c "import sys, json; content=json.load(sys.stdin); print(content['preferred']+content['path_info'])") && \ @@ -24,7 +24,7 @@ RUN cd /tmp && \ rm spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark -# Spark and Mesos config +# Configure Spark ENV SPARK_HOME=/usr/local/spark ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip \ SPARK_OPTS="--driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info" \