diff --git a/Dockerfile b/Dockerfile index 7fbcaf0..2dd4ea7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,13 +2,13 @@ FROM sequenceiq/hadoop-docker:2.6.0 MAINTAINER SequenceIQ #support for Hadoop 2.6.0 -RUN curl -s http://d3kbcqa49mib13.cloudfront.net/spark-1.6.1-bin-hadoop2.6.tgz | tar -xz -C /usr/local/ -RUN cd /usr/local && ln -s spark-1.6.1-bin-hadoop2.6 spark +RUN curl -s http://d3kbcqa49mib13.cloudfront.net/spark-1.6.3-bin-hadoop2.6.tgz | tar -xz -C /usr/local/ +RUN cd /usr/local && ln -s spark-1.6.3-bin-hadoop2.6 spark ENV SPARK_HOME /usr/local/spark RUN mkdir $SPARK_HOME/yarn-remote-client ADD yarn-remote-client $SPARK_HOME/yarn-remote-client -RUN $BOOTSTRAP && $HADOOP_PREFIX/bin/hadoop dfsadmin -safemode leave && $HADOOP_PREFIX/bin/hdfs dfs -put $SPARK_HOME-1.6.1-bin-hadoop2.6/lib /spark +RUN $BOOTSTRAP && $HADOOP_PREFIX/bin/hadoop dfsadmin -safemode leave && $HADOOP_PREFIX/bin/hdfs dfs -put $SPARK_HOME-1.6.3-bin-hadoop2.6/lib /spark ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop ENV PATH $PATH:$SPARK_HOME/bin:$HADOOP_PREFIX/bin diff --git a/bootstrap.sh b/bootstrap.sh index c01eeda..a2b9e86 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -13,7 +13,7 @@ cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; sed s/HOSTNAME/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml # setting spark defaults -echo spark.yarn.jar hdfs:///spark/spark-assembly-1.6.0-hadoop2.6.0.jar > $SPARK_HOME/conf/spark-defaults.conf +echo spark.yarn.jar hdfs:///spark/spark-assembly-1.6.3-hadoop2.6.0.jar > $SPARK_HOME/conf/spark-defaults.conf cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties service sshd start