hazza/docker-stacks
1
0
Fork 0
mirror of https://github.com/jupyter/docker-stacks.git synced 2025-10-13 21:12:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1198 from romainx/fix_spark_java11

Browse Source 
Fix spark installation for Java 11 and Arrow
This commit is contained in:
Romain
2020-12-26 17:22:44 +01:00
committed by GitHub
parent a0a544e6dc 1dd95bad2c
commit d113a601db
3 changed files with 79 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
pyspark-notebook/Dockerfile
View File

@@ -49,6 +49,12 @@ RUN ln -s "spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" spark && \
mkdir -p /usr/local/bin/before-notebook.d && \
ln -s "${SPARK_HOME}/sbin/spark-config.sh" /usr/local/bin/before-notebook.d/spark-config.sh
# Fix Spark installation for Java 11 and Apache Arrow library
# see: https://github.com/apache/spark/pull/27356, https://spark.apache.org/docs/latest/#downloading
RUN cp -p "$SPARK_HOME/conf/spark-defaults.conf.template" "$SPARK_HOME/conf/spark-defaults.conf" && \
echo 'spark.driver.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true"' >> $SPARK_HOME/conf/spark-defaults.conf && \
echo 'spark.executor.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true"' >> $SPARK_HOME/conf/spark-defaults.conf
USER $NB_UID
# Install pyarrow