From 0e23a953b02920a657939df64d673391462ac2cc Mon Sep 17 00:00:00 2001 From: robertpangborn Date: Thu, 11 Feb 2021 16:23:05 -0800 Subject: [PATCH 1/4] add version for kafka tag --- container/setup/setup-container.sh | 5 ++++- container/setup/setup-kafka.sh | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/container/setup/setup-container.sh b/container/setup/setup-container.sh index 05c5479..4a23b74 100755 --- a/container/setup/setup-container.sh +++ b/container/setup/setup-container.sh @@ -7,6 +7,9 @@ else ES_VERSION=\"${1}\" fi +# construct tag for Kafka using ES version +KAFKA_TAG=v$ES_VERSION + # set user volume USER_VOLUME=/root/user_volume @@ -18,7 +21,7 @@ SPARK_CLIENT=$(jq -r .release.${ES_VERSION}.sparkclient $USER_VOLUME/es-releases ${SETUP_AREA}/setup-java.sh ${SETUP_AREA}/setup-scala.sh ${SETUP_AREA}/setup-python.sh -${SETUP_AREA}/setup-kafka.sh +${SETUP_AREA}/setup-kafka.sh $KAFKA_TAG ${IOT_REPO_PATH}/rest/install.sh ${SETUP_AREA}/setup-spark.sh $SPARK_VERSION diff --git a/container/setup/setup-kafka.sh b/container/setup/setup-kafka.sh index 3b7ef85..38ebfe7 100755 --- a/container/setup/setup-kafka.sh +++ b/container/setup/setup-kafka.sh @@ -1,12 +1,12 @@ #!/bin/bash ## -## kafka +## kafka git repository has been tagged with release to ensure correct ibm-event.jar file is used ## cd ~ -git clone https://github.com/IBMProjectEventStore/db2eventstore-kafka.git +git clone -b $KAFKA_TAG --single-branch https://github.com/IBMProjectEventStore/db2eventstore-kafka.git cd ~/db2eventstore-kafka sbt clean sbt compile sbt package assembly -echo "Kafka setup finished." \ No newline at end of file +echo "Kafka setup finished." From 387f92b00122d45c8840e39d6dee5fbf998f256d Mon Sep 17 00:00:00 2001 From: robertpangborn Date: Mon, 22 Feb 2021 10:17:06 -0800 Subject: [PATCH 2/4] add kakfa tag to json and as parameter --- container/es-releases.json | 15 ++++++++++++--- container/setup/setup-container.sh | 6 ++---- container/setup/setup-kafka.sh | 4 ++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/container/es-releases.json b/container/es-releases.json index d60c7ac..4b06c31 100755 --- a/container/es-releases.json +++ b/container/es-releases.json @@ -4,13 +4,22 @@ "sparkversion": "2.4.6", "sparkclient": "repo1.maven.org/maven2/com/ibm/event/ibm-db2-eventstore-client-spark-2.4.6/2.0.1.0/ibm-db2-eventstore-client-spark-2.4.6-2.0.1.0.jar", "hadoopversion":"2.6", - "scopt211version":"3.5.0" + "scopt211version":"3.5.0", + "kafkatag":"v2.0.1.0" } , "2.0.0.5" : { "sparkversion": "2.2.1", "sparkclient": "repo1.maven.org/maven2/com/ibm/event/ibm-db2-eventstore-client-spark-2.2.1/2.0.0/ibm-db2-eventstore-client-spark-2.2.1-2.0.0.jar", "hadoopversion":"2.6", - "scopt211version":"3.5.0" - } + "scopt211version":"3.5.0", + "kafkatag":"v2.0..0.5" + } , + "dev" :{ + "sparkversion": "2.4.6", + "sparkclient": "repo1.maven.org/maven2/com/ibm/event/ibm-db2-eventstore-client-spark-2.4.6/2.0.1.0/ibm-db2-eventstore-client-spark-2.4.6-2.0.1.0.jar", + "hadoopversion":"2.6", + "scopt211version":"3.5.0", + "kafkatag":"dev" + ) } } diff --git a/container/setup/setup-container.sh b/container/setup/setup-container.sh index 4a23b74..72d157d 100755 --- a/container/setup/setup-container.sh +++ b/container/setup/setup-container.sh @@ -1,4 +1,4 @@ -#!/bin/bash +KAFKA_TAG#!/bin/bash # test if ES version supplied if [ -z $1 ]; then @@ -7,15 +7,13 @@ else ES_VERSION=\"${1}\" fi -# construct tag for Kafka using ES version -KAFKA_TAG=v$ES_VERSION - # set user volume USER_VOLUME=/root/user_volume # set the version variables SPARK_VERSION=$(jq -r .release.${ES_VERSION}.sparkversion $USER_VOLUME/es-releases.json) SPARK_CLIENT=$(jq -r .release.${ES_VERSION}.sparkclient $USER_VOLUME/es-releases.json) +KAFKA_TAG=$(jq -r .release.${ES_VERSION}.kafkatag $USER_VOLUME/es-releases.json) # Setup java/scala/python/kafka/rest/spark ${SETUP_AREA}/setup-java.sh diff --git a/container/setup/setup-kafka.sh b/container/setup/setup-kafka.sh index 38ebfe7..b7e80f1 100755 --- a/container/setup/setup-kafka.sh +++ b/container/setup/setup-kafka.sh @@ -3,6 +3,10 @@ ## kafka git repository has been tagged with release to ensure correct ibm-event.jar file is used ## + +# set passed kafka tag value +KAFKA_TAG=${1} + cd ~ git clone -b $KAFKA_TAG --single-branch https://github.com/IBMProjectEventStore/db2eventstore-kafka.git cd ~/db2eventstore-kafka From a49d674c642d678fedcdb2714167cdfcaf213ed9 Mon Sep 17 00:00:00 2001 From: robertpangborn Date: Mon, 22 Feb 2021 10:51:16 -0800 Subject: [PATCH 3/4] updated json --- container/es-releases.json | 2 +- container/setup/setup-container.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/container/es-releases.json b/container/es-releases.json index 4b06c31..e810b13 100755 --- a/container/es-releases.json +++ b/container/es-releases.json @@ -12,7 +12,7 @@ "sparkclient": "repo1.maven.org/maven2/com/ibm/event/ibm-db2-eventstore-client-spark-2.2.1/2.0.0/ibm-db2-eventstore-client-spark-2.2.1-2.0.0.jar", "hadoopversion":"2.6", "scopt211version":"3.5.0", - "kafkatag":"v2.0..0.5" + "kafkatag":"v2.0.0.5" } , "dev" :{ "sparkversion": "2.4.6", diff --git a/container/setup/setup-container.sh b/container/setup/setup-container.sh index 72d157d..e96fac1 100755 --- a/container/setup/setup-container.sh +++ b/container/setup/setup-container.sh @@ -1,4 +1,4 @@ -KAFKA_TAG#!/bin/bash +#!/bin/bash # test if ES version supplied if [ -z $1 ]; then From f484b98ad6935018cba068a6d2452eb1c7334773 Mon Sep 17 00:00:00 2001 From: robertpangborn Date: Mon, 22 Feb 2021 13:33:45 -0800 Subject: [PATCH 4/4] fixed json format error --- container/es-releases.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/container/es-releases.json b/container/es-releases.json index e810b13..7f00623 100755 --- a/container/es-releases.json +++ b/container/es-releases.json @@ -1,6 +1,6 @@ { "release": { - "2.0.1.0" :{ + "2.0.1.0" : { "sparkversion": "2.4.6", "sparkclient": "repo1.maven.org/maven2/com/ibm/event/ibm-db2-eventstore-client-spark-2.4.6/2.0.1.0/ibm-db2-eventstore-client-spark-2.4.6-2.0.1.0.jar", "hadoopversion":"2.6", @@ -14,12 +14,12 @@ "scopt211version":"3.5.0", "kafkatag":"v2.0.0.5" } , - "dev" :{ + "dev" : { "sparkversion": "2.4.6", "sparkclient": "repo1.maven.org/maven2/com/ibm/event/ibm-db2-eventstore-client-spark-2.4.6/2.0.1.0/ibm-db2-eventstore-client-spark-2.4.6-2.0.1.0.jar", "hadoopversion":"2.6", "scopt211version":"3.5.0", "kafkatag":"dev" - ) - } + } + } }