Skip to content

Commit 22a3f31

Browse files
Pietro Brunettiagilelab-tmnd1991
Pietro Brunetti
authored and
agilelab-tmnd1991
committed
[#550] Parallel Write Plugin - improving error handling when sending streamed data
# New features and improvements - Parallel Write writer throws exception if the http response status is not 2xx, therefore now spark queries will fail if the response of data/stream is not successful. - Add more logs around the parallel write http calls, to see them, enable logger category `it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel` at INFO level - Avoid performing parallel write when there are no partitions to be written (streaming query first "run") - Update image used to build `waspctl` - Automatically generate openapi during the build instead of using the committed one - Update `de.flapdoodle.embed:de.flapdoodle.embed.mongo` from `3.4.6` to `3.5.4` - Change restlet repo from `https://maven.restlet.com/` to `https://maven.restlet.talend.com` since the SSL certificate was expired # Breaking changes None. # Migration None. # Bug fixes None. # How this feature was tested Existing unit tests. # Related issue Closes #550
1 parent 87be53e commit 22a3f31

File tree

10 files changed

+100
-140
lines changed

10 files changed

+100
-140
lines changed

‎.gitlab-ci.yml‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ test-kernel:
3030
image: "hseeberger/scala-sbt:8u312_1.6.2_2.12.15"
3131
parallel:
3232
matrix:
33-
- WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "CDP717" , "EMR_2_12"]
33+
- WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "EMR_2_12", "CDP717"]
3434
script:
3535
- ./run-sbt-unprivileged.sh "clean coverageOn wasp-kernel/test coverageOff wasp-kernel/coverageReport wasp-kernel/coverageAggregate"
3636
after_script:
@@ -56,7 +56,7 @@ test-plugin:
5656
image: "hseeberger/scala-sbt:8u312_1.6.2_2.12.15"
5757
parallel:
5858
matrix:
59-
- WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "CDP717" , "EMR_2_12"]
59+
- WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "EMR_2_12", "CDP717"]
6060
script:
6161
- ./run-sbt-unprivileged.sh "clean coverageOn wasp-plugin/test coverageOff wasp-plugin/coverageReport wasp-plugin/coverageAggregate"
6262
after_script:
@@ -83,7 +83,7 @@ test-repo:
8383
image: "hseeberger/scala-sbt:8u312_1.6.2_2.12.15"
8484
parallel:
8585
matrix:
86-
- WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "CDP717" , "EMR_2_12"]
86+
- WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "EMR_2_12", "CDP717"]
8787
script:
8888
- ./run-sbt-unprivileged.sh "clean coverageOn wasp-repository/test coverageOff wasp-repository/coverageReport wasp-repository/coverageAggregate"
8989
after_script:
@@ -109,7 +109,7 @@ compile-whitelabel:
109109
image: "hseeberger/scala-sbt:11.0.14.1_1.6.2_2.12.15"
110110
parallel:
111111
matrix:
112-
- WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "CDP717" , "EMR_2_12"]
112+
- WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "EMR_2_12", "CDP717"]
113113
script:
114114
- ./run-sbt-unprivileged.sh "clean wasp-whitelabel/Test/compile"
115115
tags:
@@ -128,11 +128,27 @@ censor:
128128
script:
129129
- ./censor.sh
130130

131+
generate-open-api:
132+
stage: test
133+
interruptible: true
134+
image: "hseeberger/scala-sbt:11.0.14.1_1.6.2_2.12.15"
135+
script:
136+
- ./run-sbt-unprivileged.sh "generateOpenApi"
137+
tags:
138+
- gitlab-org
139+
only:
140+
- /^develop$/
141+
- /(^release\/v([0-9]+)\.([0-9]+))/
142+
- merge_requests
143+
artifacts:
144+
paths: ['documentation/wasp-openapi.yaml']
145+
needs: []
146+
131147

132148
waspctl:
133149
interruptible: true
134150
stage: deploy
135-
image: "registry.gitlab.com/agilefactory/agile.wasp2/waspctl-build:0.1.0"
151+
image: "registry.gitlab.com/agilefactory/agile.wasp2/waspctl-build:0.2.0"
136152
script:
137153
- bash waspctl/build-for-all-archs.bash
138154
artifacts:
@@ -142,14 +158,18 @@ waspctl:
142158
- /(^release\/v([0-9]+)\.([0-9]+))/
143159
- merge_requests
144160
- tags
161+
needs:
162+
- generate-open-api
163+
dependencies:
164+
- generate-open-api
145165

146166
snapshot:
147167
interruptible: false
148168
stage: deploy
149169
image: "hseeberger/scala-sbt:11.0.14.1_1.6.2_2.12.15"
150170
parallel:
151171
matrix:
152-
- WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "CDP717" , "EMR_2_12"]
172+
- WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "EMR_2_12", "CDP717"]
153173
tags:
154174
- gitlab-org
155175
script:
@@ -173,7 +193,7 @@ release:
173193
image: "hseeberger/scala-sbt:11.0.14.1_1.6.2_2.12.15"
174194
parallel:
175195
matrix:
176-
- WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "CDP717" , "EMR_2_12"]
196+
- WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "EMR_2_12", "CDP717"]
177197
script:
178198
- "mkdir -p $HOME/.sbt/gpg"
179199
- "echo $GPG_PUB | base64 -d > $HOME/.sbt/gpg/pubring.asc"

‎documentation/wasp-openapi.yaml‎

Lines changed: 20 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
openapi: 3.0.1
22
info:
33
title: wasp-api
4-
version: 2.27.0-436-refactor-plugin-in-order-to-allow-plugin-defined-models-SNAPSHOT
4+
version: 2.31.0-550-parallel-write-plugin-improving-error-handling-when-sending-streamed-data-SNAPSHOT
55
servers:
66
- url: http://localhost:2891
77
description: default development server, beware of CORS
@@ -1114,9 +1114,7 @@ paths:
11141114
- owner
11151115
- isSystem
11161116
- creationTime
1117-
- legacyStreamingComponents
11181117
- structuredStreamingComponents
1119-
- rtComponents
11201118
- labels
11211119
- enrichmentSources
11221120
type: object
@@ -1132,18 +1130,10 @@ paths:
11321130
creationTime:
11331131
type: integer
11341132
format: int64
1135-
legacyStreamingComponents:
1136-
type: array
1137-
items:
1138-
$ref: '#/components/schemas/LegacyStreamingETLModel'
11391133
structuredStreamingComponents:
11401134
type: array
11411135
items:
11421136
$ref: '#/components/schemas/StructuredStreamingETLModel'
1143-
rtComponents:
1144-
type: array
1145-
items:
1146-
$ref: '#/components/schemas/RTModel'
11471137
dashboard:
11481138
$ref: '#/components/schemas/DashboardModel'
11491139
labels:
@@ -1194,9 +1184,7 @@ paths:
11941184
- owner
11951185
- isSystem
11961186
- creationTime
1197-
- legacyStreamingComponents
11981187
- structuredStreamingComponents
1199-
- rtComponents
12001188
- labels
12011189
- enrichmentSources
12021190
type: object
@@ -1212,18 +1200,10 @@ paths:
12121200
creationTime:
12131201
type: integer
12141202
format: int64
1215-
legacyStreamingComponents:
1216-
type: array
1217-
items:
1218-
$ref: '#/components/schemas/LegacyStreamingETLModel'
12191203
structuredStreamingComponents:
12201204
type: array
12211205
items:
12221206
$ref: '#/components/schemas/StructuredStreamingETLModel'
1223-
rtComponents:
1224-
type: array
1225-
items:
1226-
$ref: '#/components/schemas/RTModel'
12271207
dashboard:
12281208
$ref: '#/components/schemas/DashboardModel'
12291209
labels:
@@ -2620,40 +2600,6 @@ components:
26202600
xml:
26212601
name: CountEntry
26222602
namespace: java://it.agilelab.bigdata.wasp.models
2623-
LegacyStreamingETLModel:
2624-
required:
2625-
- name
2626-
- inputs
2627-
- output
2628-
- mlModels
2629-
- kafkaAccessType
2630-
- group
2631-
- isActive
2632-
type: object
2633-
properties:
2634-
name:
2635-
type: string
2636-
inputs:
2637-
type: array
2638-
items:
2639-
$ref: '#/components/schemas/ReaderModel'
2640-
output:
2641-
$ref: '#/components/schemas/WriterModel'
2642-
mlModels:
2643-
type: array
2644-
items:
2645-
$ref: '#/components/schemas/MlModelOnlyInfo'
2646-
strategy:
2647-
$ref: '#/components/schemas/StrategyModel'
2648-
kafkaAccessType:
2649-
type: string
2650-
group:
2651-
type: string
2652-
isActive:
2653-
type: boolean
2654-
xml:
2655-
name: LegacyStreamingETLModel
2656-
namespace: java://it.agilelab.bigdata.wasp.models
26572603
Sources:
26582604
required:
26592605
- found
@@ -2978,17 +2924,17 @@ components:
29782924
GenericModel:
29792925
required:
29802926
- name
2981-
- kind
29822927
- value
2928+
- product
29832929
- options
29842930
type: object
29852931
properties:
29862932
name:
29872933
type: string
2988-
kind:
2989-
type: string
29902934
value:
29912935
$ref: '#/components/schemas/BsonDocument'
2936+
product:
2937+
$ref: '#/components/schemas/GenericProduct'
29922938
options:
29932939
$ref: '#/components/schemas/GenericOptions'
29942940
xml:
@@ -3509,7 +3455,6 @@ components:
35093455
type: object
35103456
additionalProperties:
35113457
type: string
3512-
nullable: true
35133458
xml:
35143459
name: WriterModel
35153460
namespace: java://it.agilelab.bigdata.wasp.models
@@ -3520,9 +3465,7 @@ components:
35203465
- owner
35213466
- isSystem
35223467
- creationTime
3523-
- legacyStreamingComponents
35243468
- structuredStreamingComponents
3525-
- rtComponents
35263469
- labels
35273470
- enrichmentSources
35283471
type: object
@@ -3538,18 +3481,10 @@ components:
35383481
creationTime:
35393482
type: integer
35403483
format: int64
3541-
legacyStreamingComponents:
3542-
type: array
3543-
items:
3544-
$ref: '#/components/schemas/LegacyStreamingETLModel'
35453484
structuredStreamingComponents:
35463485
type: array
35473486
items:
35483487
$ref: '#/components/schemas/StructuredStreamingETLModel'
3549-
rtComponents:
3550-
type: array
3551-
items:
3552-
$ref: '#/components/schemas/RTModel'
35533488
dashboard:
35543489
$ref: '#/components/schemas/DashboardModel'
35553490
labels:
@@ -3622,6 +3557,7 @@ components:
36223557
- kryoSerializer
36233558
- streamingBatchIntervalMs
36243559
- checkpointDir
3560+
- enableHiveSupport
36253561
- others
36263562
- schedulingStrategy
36273563
- name
@@ -3660,6 +3596,8 @@ components:
36603596
format: int32
36613597
checkpointDir:
36623598
type: string
3599+
enableHiveSupport:
3600+
type: boolean
36633601
triggerIntervalMs:
36643602
type: integer
36653603
format: int64
@@ -4190,28 +4128,6 @@ components:
41904128
xml:
41914129
name: SolrConfigModel
41924130
namespace: java://it.agilelab.bigdata.wasp.models.configuration
4193-
RTModel:
4194-
required:
4195-
- name
4196-
- inputs
4197-
- isActive
4198-
type: object
4199-
properties:
4200-
name:
4201-
type: string
4202-
inputs:
4203-
type: array
4204-
items:
4205-
$ref: '#/components/schemas/ReaderModel'
4206-
isActive:
4207-
type: boolean
4208-
strategy:
4209-
$ref: '#/components/schemas/StrategyModel'
4210-
endpoint:
4211-
$ref: '#/components/schemas/WriterModel'
4212-
xml:
4213-
name: RTModel
4214-
namespace: java://it.agilelab.bigdata.wasp.models
42154131
Events:
42164132
required:
42174133
- found
@@ -4465,6 +4381,19 @@ components:
44654381
xml:
44664382
name: KeyValueOption
44674383
namespace: java://it.agilelab.bigdata.wasp.models
4384+
GenericProduct:
4385+
required:
4386+
- categoryName
4387+
type: object
4388+
properties:
4389+
categoryName:
4390+
type: string
4391+
productName:
4392+
type: string
4393+
nullable: true
4394+
xml:
4395+
name: GenericProduct
4396+
namespace: java://it.agilelab.bigdata.wasp.datastores
44684397
PipegraphStatus:
44694398
type: string
44704399
enum:

0 commit comments

Comments
 (0)