From f1334d6a003eb902816d6ce7c299b132936d8d37 Mon Sep 17 00:00:00 2001 From: Michael Criscolo Date: Wed, 26 Jul 2023 14:03:00 -0500 Subject: [PATCH 1/2] FAB-5988 ensure needed dependencies are available for main:run --- main-app/build.gradle.kts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/main-app/build.gradle.kts b/main-app/build.gradle.kts index a8dd992..06ff4e4 100644 --- a/main-app/build.gradle.kts +++ b/main-app/build.gradle.kts @@ -56,6 +56,13 @@ dependencies { implementation("info.picocli:picocli:4.6.3") annotationProcessor("info.picocli:picocli-codegen:4.6.3") + // Hadoop AWS - for Spark S3a connector, in sdk image but needed for main-app:run + runtimeOnly("org.apache.hadoop:hadoop-aws:3.3.4") + testRuntimeOnly("org.apache.hadoop:hadoop-aws:3.3.4") + // Hadoop GCS - spark connector, in sdk image but needed for main-app:run + runtimeOnly("com.google.cloud.bigdataoss:gcs-connector:hadoop3-2.2.15") + testRuntimeOnly("com.google.cloud.bigdataoss:gcs-connector:hadoop3-2.2.15") + // test dependencies testImplementation(platform("org.junit:junit-bom:5.8.2")) testImplementation("org.junit.jupiter:junit-jupiter") From 80fef74e2965e06c2c28d4724dfca3664e9a996b Mon Sep 17 00:00:00 2001 From: Michael Criscolo Date: Fri, 28 Jul 2023 10:47:07 -0500 Subject: [PATCH 2/2] FAB-5988 rename property streamReadDir to correct value --- main-app/src/main/resources/spec/streaming-connections.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main-app/src/main/resources/spec/streaming-connections.yml b/main-app/src/main/resources/spec/streaming-connections.yml index 5abd687..2ba5446 100644 --- a/main-app/src/main/resources/spec/streaming-connections.yml +++ b/main-app/src/main/resources/spec/streaming-connections.yml @@ -24,7 +24,7 @@ spec: value: true - name: csv/header value: true - - name: streamReadDir + - name: stream_read_dir value: s3a://cortex-examples/data/data-streaming - name: isTriggered value: false