diff --git a/main-app/build.gradle.kts b/main-app/build.gradle.kts index a8dd992..06ff4e4 100644 --- a/main-app/build.gradle.kts +++ b/main-app/build.gradle.kts @@ -56,6 +56,13 @@ dependencies { implementation("info.picocli:picocli:4.6.3") annotationProcessor("info.picocli:picocli-codegen:4.6.3") + // Hadoop AWS - for Spark S3a connector, in sdk image but needed for main-app:run + runtimeOnly("org.apache.hadoop:hadoop-aws:3.3.4") + testRuntimeOnly("org.apache.hadoop:hadoop-aws:3.3.4") + // Hadoop GCS - spark connector, in sdk image but needed for main-app:run + runtimeOnly("com.google.cloud.bigdataoss:gcs-connector:hadoop3-2.2.15") + testRuntimeOnly("com.google.cloud.bigdataoss:gcs-connector:hadoop3-2.2.15") + // test dependencies testImplementation(platform("org.junit:junit-bom:5.8.2")) testImplementation("org.junit.jupiter:junit-jupiter") diff --git a/main-app/src/main/resources/spec/streaming-connections.yml b/main-app/src/main/resources/spec/streaming-connections.yml index 5abd687..2ba5446 100644 --- a/main-app/src/main/resources/spec/streaming-connections.yml +++ b/main-app/src/main/resources/spec/streaming-connections.yml @@ -24,7 +24,7 @@ spec: value: true - name: csv/header value: true - - name: streamReadDir + - name: stream_read_dir value: s3a://cortex-examples/data/data-streaming - name: isTriggered value: false