diff --git a/all-spark-notebook/Dockerfile b/all-spark-notebook/Dockerfile index 872bdb7a..ff5958b6 100644 --- a/all-spark-notebook/Dockerfile +++ b/all-spark-notebook/Dockerfile @@ -6,7 +6,7 @@ MAINTAINER Jupyter Project USER root # Spark dependencies -ENV APACHE_SPARK_VERSION 1.4.1 +ENV APACHE_SPARK_VERSION 1.5.1 RUN apt-get -y update && \ apt-get install -y --no-install-recommends openjdk-7-jre-headless && \ apt-get clean diff --git a/all-spark-notebook/README.md b/all-spark-notebook/README.md index f7d88506..84a5d502 100644 --- a/all-spark-notebook/README.md +++ b/all-spark-notebook/README.md @@ -8,7 +8,7 @@ * Scala 2.10.x * pyspark, pandas, matplotlib, scipy, seaborn, scikit-learn pre-installed for Python * ggplot2, rcurl preinstalled for R -* Spark 1.4.1 for use in local mode or to connect to a cluster of Spark workers +* Spark 1.5.1 for use in local mode or to connect to a cluster of Spark workers * Mesos client 0.22 binary that can communicate with a Mesos master * Unprivileged user `jovyan` (uid=1000, configurable, see options) in group `users` (gid=100) with ownership over `/home/jovyan` and `/opt/conda` * [tini](https://github.com/krallin/tini) as the container entrypoint and [start-notebook.sh](../minimal-notebook/start-notebook.sh) as the default command @@ -114,8 +114,8 @@ conf = pyspark.SparkConf() # point to mesos master or zookeeper entry (e.g., zk://10.10.10.10:2181/mesos) conf.setMaster("mesos://10.10.10.10:5050") # point to spark binary package in HDFS or on local filesystem on all slave -# nodes (e.g., file:///opt/spark/spark-1.4.1-bin-hadoop2.6.tgz) -conf.set("spark.executor.uri", "hdfs://10.10.10.10/spark/spark-1.4.1-bin-hadoop2.6.tgz") +# nodes (e.g., file:///opt/spark/spark-1.5.1-bin-hadoop2.6.tgz) +conf.set("spark.executor.uri", "hdfs://10.10.10.10/spark/spark-1.5.1-bin-hadoop2.6.tgz") # set other options as desired conf.set("spark.executor.memory", "8g") conf.set("spark.core.connection.ack.wait.timeout", "1200") @@ -147,10 +147,10 @@ library(SparkR) # point to mesos master or zookeeper entry (e.g., zk://10.10.10.10:2181/mesos)\ # as the first argument # point to spark binary package in HDFS or on local filesystem on all slave -# nodes (e.g., file:///opt/spark/spark-1.4.1-bin-hadoop2.6.tgz) in sparkEnvir +# nodes (e.g., file:///opt/spark/spark-1.5.1-bin-hadoop2.6.tgz) in sparkEnvir # set other options in sparkEnvir sc <- sparkR.init("mesos://10.10.10.10:5050", sparkEnvir=list( - spark.executor.uri="hdfs://10.10.10.10/spark/spark-1.4.1-bin-hadoop2.6.tgz", + spark.executor.uri="hdfs://10.10.10.10/spark/spark-1.5.1-bin-hadoop2.6.tgz", spark.executor.memory="8g" ) ) @@ -184,7 +184,7 @@ For instance, a kernel spec file with information about a Mesos master, Spark bi "--master=mesos://10.10.10.10:5050" ], "env": { - "SPARK_CONFIGURATION": "spark.executor.memory=8g,spark.executor.uri=hdfs://10.10.10.10/spark/spark-1.4.1-bin-hadoop2.6.tgz" + "SPARK_CONFIGURATION": "spark.executor.memory=8g,spark.executor.uri=hdfs://10.10.10.10/spark/spark-1.5.1-bin-hadoop2.6.tgz" } } ``` diff --git a/pyspark-notebook/Dockerfile b/pyspark-notebook/Dockerfile index c29dbef4..0eb777bc 100644 --- a/pyspark-notebook/Dockerfile +++ b/pyspark-notebook/Dockerfile @@ -6,7 +6,7 @@ MAINTAINER Jupyter Project USER root # Spark dependencies -ENV APACHE_SPARK_VERSION 1.4.1 +ENV APACHE_SPARK_VERSION 1.5.1 RUN apt-get -y update && \ apt-get install -y --no-install-recommends openjdk-7-jre-headless && \ apt-get clean diff --git a/pyspark-notebook/README.md b/pyspark-notebook/README.md index ede3711f..fee77bec 100644 --- a/pyspark-notebook/README.md +++ b/pyspark-notebook/README.md @@ -5,7 +5,7 @@ * Jupyter Notebook 4.0.x * Conda Python 3.x and Python 2.7.x environments * pyspark, pandas, matplotlib, scipy, seaborn, scikit-learn pre-installed -* Spark 1.4.1 for use in local mode or to connect to a cluster of Spark workers +* Spark 1.5.1 for use in local mode or to connect to a cluster of Spark workers * Mesos client 0.22 binary that can communicate with a Mesos master * Unprivileged user `jovyan` (uid=1000, configurable, see options) in group `users` (gid=100) with ownership over `/home/jovyan` and `/opt/conda` * [tini](https://github.com/krallin/tini) as the container entrypoint and [start-notebook.sh](../minimal-notebook/start-notebook.sh) as the default command @@ -72,8 +72,8 @@ conf = pyspark.SparkConf() # point to mesos master or zookeeper entry (e.g., zk://10.10.10.10:2181/mesos) conf.setMaster("mesos://10.10.10.10:5050") # point to spark binary package in HDFS or on local filesystem on all slave -# nodes (e.g., file:///opt/spark/spark-1.4.1-bin-hadoop2.6.tgz) -conf.set("spark.executor.uri", "hdfs://10.122.193.209/spark/spark-1.4.1-bin-hadoop2.6.tgz") +# nodes (e.g., file:///opt/spark/spark-1.5.1-bin-hadoop2.6.tgz) +conf.set("spark.executor.uri", "hdfs://10.122.193.209/spark/spark-1.5.1-bin-hadoop2.6.tgz") # set other options as desired conf.set("spark.executor.memory", "8g") conf.set("spark.core.connection.ack.wait.timeout", "1200")