hazza/docker-stacks
1
0
Fork 0
mirror of https://github.com/jupyter/docker-stacks.git synced 2025-10-18 07:22:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

update for Spark 1.5.1

Browse Source
This commit is contained in:
Kai Wang
2015-10-21 15:22:57 +00:00
parent 2819b2b393
commit 817550198f
4 changed files with 11 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
all-spark-notebook/Dockerfile
View File

@@ -6,7 +6,7 @@ MAINTAINER Jupyter Project <jupyter@googlegroups.com>
USER root USER root
# Spark dependencies # Spark dependencies
ENV APACHE_SPARK_VERSION 1.4.1 ENV APACHE_SPARK_VERSION 1.5.1
RUN apt-get -y update && \ RUN apt-get -y update && \
apt-get install -y --no-install-recommends openjdk-7-jre-headless && \ apt-get install -y --no-install-recommends openjdk-7-jre-headless && \
apt-get clean apt-get clean

12
all-spark-notebook/README.md
View File

@@ -8,7 +8,7 @@
* Scala 2.10.x * Scala 2.10.x
* pyspark, pandas, matplotlib, scipy, seaborn, scikit-learn pre-installed for Python * pyspark, pandas, matplotlib, scipy, seaborn, scikit-learn pre-installed for Python
* ggplot2, rcurl preinstalled for R * ggplot2, rcurl preinstalled for R
* Spark 1.4.1 for use in local mode or to connect to a cluster of Spark workers * Spark 1.5.1 for use in local mode or to connect to a cluster of Spark workers
* Mesos client 0.22 binary that can communicate with a Mesos master * Mesos client 0.22 binary that can communicate with a Mesos master
* Unprivileged user `jovyan` (uid=1000, configurable, see options) in group `users` (gid=100) with ownership over `/home/jovyan` and `/opt/conda` * Unprivileged user `jovyan` (uid=1000, configurable, see options) in group `users` (gid=100) with ownership over `/home/jovyan` and `/opt/conda`
* [tini](https://github.com/krallin/tini) as the container entrypoint and [start-notebook.sh](../minimal-notebook/start-notebook.sh) as the default command * [tini](https://github.com/krallin/tini) as the container entrypoint and [start-notebook.sh](../minimal-notebook/start-notebook.sh) as the default command
@@ -114,8 +114,8 @@ conf = pyspark.SparkConf()
# point to mesos master or zookeeper entry (e.g., zk://10.10.10.10:2181/mesos) # point to mesos master or zookeeper entry (e.g., zk://10.10.10.10:2181/mesos)
conf.setMaster("mesos://10.10.10.10:5050") conf.setMaster("mesos://10.10.10.10:5050")
# point to spark binary package in HDFS or on local filesystem on all slave # point to spark binary package in HDFS or on local filesystem on all slave
# nodes (e.g., file:///opt/spark/spark-1.4.1-bin-hadoop2.6.tgz) # nodes (e.g., file:///opt/spark/spark-1.5.1-bin-hadoop2.6.tgz)
conf.set("spark.executor.uri", "hdfs://10.10.10.10/spark/spark-1.4.1-bin-hadoop2.6.tgz") conf.set("spark.executor.uri", "hdfs://10.10.10.10/spark/spark-1.5.1-bin-hadoop2.6.tgz")
# set other options as desired # set other options as desired
conf.set("spark.executor.memory", "8g") conf.set("spark.executor.memory", "8g")
conf.set("spark.core.connection.ack.wait.timeout", "1200") conf.set("spark.core.connection.ack.wait.timeout", "1200")
@@ -147,10 +147,10 @@ library(SparkR)
# point to mesos master or zookeeper entry (e.g., zk://10.10.10.10:2181/mesos)\ # point to mesos master or zookeeper entry (e.g., zk://10.10.10.10:2181/mesos)\
# as the first argument # as the first argument
# point to spark binary package in HDFS or on local filesystem on all slave # point to spark binary package in HDFS or on local filesystem on all slave
# nodes (e.g., file:///opt/spark/spark-1.4.1-bin-hadoop2.6.tgz) in sparkEnvir # nodes (e.g., file:///opt/spark/spark-1.5.1-bin-hadoop2.6.tgz) in sparkEnvir
# set other options in sparkEnvir # set other options in sparkEnvir
sc <- sparkR.init("mesos://10.10.10.10:5050", sparkEnvir=list( sc <- sparkR.init("mesos://10.10.10.10:5050", sparkEnvir=list(
spark.executor.uri="hdfs://10.10.10.10/spark/spark-1.4.1-bin-hadoop2.6.tgz", spark.executor.uri="hdfs://10.10.10.10/spark/spark-1.5.1-bin-hadoop2.6.tgz",
spark.executor.memory="8g" spark.executor.memory="8g"
) )
) )
@@ -184,7 +184,7 @@ For instance, a kernel spec file with information about a Mesos master, Spark bi
"--master=mesos://10.10.10.10:5050" "--master=mesos://10.10.10.10:5050"
], ],
"env": { "env": {
"SPARK_CONFIGURATION": "spark.executor.memory=8g,spark.executor.uri=hdfs://10.10.10.10/spark/spark-1.4.1-bin-hadoop2.6.tgz" "SPARK_CONFIGURATION": "spark.executor.memory=8g,spark.executor.uri=hdfs://10.10.10.10/spark/spark-1.5.1-bin-hadoop2.6.tgz"
} }
} }
``` ```

2
pyspark-notebook/Dockerfile
View File

@@ -6,7 +6,7 @@ MAINTAINER Jupyter Project <jupyter@googlegroups.com>
USER root USER root
# Spark dependencies # Spark dependencies
ENV APACHE_SPARK_VERSION 1.4.1 ENV APACHE_SPARK_VERSION 1.5.1
RUN apt-get -y update && \ RUN apt-get -y update && \
apt-get install -y --no-install-recommends openjdk-7-jre-headless && \ apt-get install -y --no-install-recommends openjdk-7-jre-headless && \
apt-get clean apt-get clean

6
pyspark-notebook/README.md
View File

@@ -5,7 +5,7 @@
* Jupyter Notebook 4.0.x * Jupyter Notebook 4.0.x
* Conda Python 3.x and Python 2.7.x environments * Conda Python 3.x and Python 2.7.x environments
* pyspark, pandas, matplotlib, scipy, seaborn, scikit-learn pre-installed * pyspark, pandas, matplotlib, scipy, seaborn, scikit-learn pre-installed
* Spark 1.4.1 for use in local mode or to connect to a cluster of Spark workers * Spark 1.5.1 for use in local mode or to connect to a cluster of Spark workers
* Mesos client 0.22 binary that can communicate with a Mesos master * Mesos client 0.22 binary that can communicate with a Mesos master
* Unprivileged user `jovyan` (uid=1000, configurable, see options) in group `users` (gid=100) with ownership over `/home/jovyan` and `/opt/conda` * Unprivileged user `jovyan` (uid=1000, configurable, see options) in group `users` (gid=100) with ownership over `/home/jovyan` and `/opt/conda`
* [tini](https://github.com/krallin/tini) as the container entrypoint and [start-notebook.sh](../minimal-notebook/start-notebook.sh) as the default command * [tini](https://github.com/krallin/tini) as the container entrypoint and [start-notebook.sh](../minimal-notebook/start-notebook.sh) as the default command
@@ -72,8 +72,8 @@ conf = pyspark.SparkConf()
# point to mesos master or zookeeper entry (e.g., zk://10.10.10.10:2181/mesos) # point to mesos master or zookeeper entry (e.g., zk://10.10.10.10:2181/mesos)
conf.setMaster("mesos://10.10.10.10:5050") conf.setMaster("mesos://10.10.10.10:5050")
# point to spark binary package in HDFS or on local filesystem on all slave # point to spark binary package in HDFS or on local filesystem on all slave
# nodes (e.g., file:///opt/spark/spark-1.4.1-bin-hadoop2.6.tgz) # nodes (e.g., file:///opt/spark/spark-1.5.1-bin-hadoop2.6.tgz)
conf.set("spark.executor.uri", "hdfs://10.122.193.209/spark/spark-1.4.1-bin-hadoop2.6.tgz") conf.set("spark.executor.uri", "hdfs://10.122.193.209/spark/spark-1.5.1-bin-hadoop2.6.tgz")
# set other options as desired # set other options as desired
conf.set("spark.executor.memory", "8g") conf.set("spark.executor.memory", "8g")
conf.set("spark.core.connection.ack.wait.timeout", "1200") conf.set("spark.core.connection.ack.wait.timeout", "1200")