From 78ac19aefb2ce3f7944b999aae36992cc0133215 Mon Sep 17 00:00:00 2001 From: DracoBlue Date: Thu, 2 Mar 2017 12:16:36 +0100 Subject: [PATCH 1/2] Update to Spark 1.6.3 --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7fbcaf0..2dd4ea7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,13 +2,13 @@ FROM sequenceiq/hadoop-docker:2.6.0 MAINTAINER SequenceIQ #support for Hadoop 2.6.0 -RUN curl -s http://d3kbcqa49mib13.cloudfront.net/spark-1.6.1-bin-hadoop2.6.tgz | tar -xz -C /usr/local/ -RUN cd /usr/local && ln -s spark-1.6.1-bin-hadoop2.6 spark +RUN curl -s http://d3kbcqa49mib13.cloudfront.net/spark-1.6.3-bin-hadoop2.6.tgz | tar -xz -C /usr/local/ +RUN cd /usr/local && ln -s spark-1.6.3-bin-hadoop2.6 spark ENV SPARK_HOME /usr/local/spark RUN mkdir $SPARK_HOME/yarn-remote-client ADD yarn-remote-client $SPARK_HOME/yarn-remote-client -RUN $BOOTSTRAP && $HADOOP_PREFIX/bin/hadoop dfsadmin -safemode leave && $HADOOP_PREFIX/bin/hdfs dfs -put $SPARK_HOME-1.6.1-bin-hadoop2.6/lib /spark +RUN $BOOTSTRAP && $HADOOP_PREFIX/bin/hadoop dfsadmin -safemode leave && $HADOOP_PREFIX/bin/hdfs dfs -put $SPARK_HOME-1.6.3-bin-hadoop2.6/lib /spark ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop ENV PATH $PATH:$SPARK_HOME/bin:$HADOOP_PREFIX/bin From 53948eee02492d49ddbebd64e518b20de9a9e40b Mon Sep 17 00:00:00 2001 From: DracoBlue Date: Thu, 2 Mar 2017 12:18:00 +0100 Subject: [PATCH 2/2] Use Spark 1.6.3 for spark-defaults.conf --- bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap.sh b/bootstrap.sh index c01eeda..a2b9e86 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -13,7 +13,7 @@ cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; sed s/HOSTNAME/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml # setting spark defaults -echo spark.yarn.jar hdfs:///spark/spark-assembly-1.6.0-hadoop2.6.0.jar > $SPARK_HOME/conf/spark-defaults.conf +echo spark.yarn.jar hdfs:///spark/spark-assembly-1.6.3-hadoop2.6.0.jar > $SPARK_HOME/conf/spark-defaults.conf cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties service sshd start