Skip to content

Commit 23d3f4c

Browse files
authored
fix: align Vector component schemas with v0.53.0 spec (#80)
* fix: align Vector component schemas with official spec (112/114 clean) Bump Vector to 0.53.0 and audit all VectorFlow component schemas against `vector generate-schema` to fix mismatches: - Bump Vector from 0.44.0 to 0.53.0 in agent and server Dockerfiles - Add encoding to required arrays for 25 sinks - Remove invalid TLS fields from 12 components (implicit via HTTPS/rediss) - Remove deprecated fields (http headers, elasticsearch suppress_type_name, splunk_hec token, databend tls) - Remove invalid fields (mqtt qos, pulsar dead_letter_queue_topic, etc.) - Fix required arrays for 8 components (kafka group_id, nats connection_name, vector address, opentelemetry grpc/http, throttle window_secs, etc.) - Fix aws_sns queue_url → topic_arn (Vector docs have copy-paste bug from SQS) - Add dependsOn for 6 modal components (syslog, socket, fluent, dnstap, statsd sources + socket sink) so fields show/hide based on selected mode * fix: align encoding codecs, compression, enums, and defaults with Vector spec Deep value-level validation against `vector generate-schema`: Encoding codecs: - Remove invalid `ndjson` codec from all sinks and shared.ts default (NDJSON in Vector = codec:json + framing:newline_delimited) - Add missing `otlp` and `syslog` codecs to 20+ sinks - Standardize all sinks to Vector's canonical 13-codec list Compression options: - Add missing algorithms to AWS, GCP, network, and search-db sinks (snappy, zlib, zstd where missing) - Keep Kafka's unique list (lz4 instead of zlib) Enum value fixes: - source:http_server method — add DELETE, HEAD, OPTIONS, PATCH - source:http_client method — add DELETE, OPTIONS, PATCH - sink:http method — add trace - sink:clickhouse format — add arrow_stream - source:host_metrics collectors — add tcp - source:vector version — remove deprecated v1 - source:docker_logs multiline mode — halt_after → halt_with - sink:gcp_chronicle region — remove invalid australia-southeast1 Default value fixes: - source:aws_sqs visibility_timeout_secs — 30 → 300 (10x too low) * feat: add Vector docs link icon to component config header Small ExternalLink icon next to the component name in the detail panel that opens the Vector documentation for that component type. URL is computed from kind/type: vector.dev/docs/reference/configuration/{kind}s/{type}/
1 parent 4164f15 commit 23d3f4c

File tree

19 files changed

+190
-239
lines changed

19 files changed

+190
-239
lines changed

agent/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ RUN CGO_ENABLED=0 go build -ldflags="-s -w -X github.com/TerrifiedBug/vectorflow
99

1010
# Stage 2: Download Vector
1111
FROM alpine:3.21 AS vector
12-
ARG VECTOR_VERSION=0.44.0
12+
ARG VECTOR_VERSION=0.53.0
1313
RUN apk add --no-cache curl && \
1414
curl -sSfL -o /tmp/vector.tar.gz \
1515
"https://packages.timber.io/vector/${VECTOR_VERSION}/vector-${VECTOR_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \

agent/install.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ VECTOR_DATA_DIR="/var/lib/vector"
1111
CONFIG_DIR="/etc/vectorflow"
1212
ENV_FILE="${CONFIG_DIR}/agent.env"
1313
SERVICE_NAME="vf-agent"
14-
VECTOR_VERSION="0.44.0"
14+
VECTOR_VERSION="0.53.0"
1515

1616
# Defaults
1717
VF_URL=""

docker/server/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ RUN --mount=type=cache,target=/root/.local/share/pnpm/store \
1212

1313
# ---- Stage 2: Download Vector binary (cached unless VECTOR_VERSION changes) ----
1414
FROM alpine:3.21 AS vector
15-
ARG VECTOR_VERSION=0.44.0
15+
ARG VECTOR_VERSION=0.53.0
1616
RUN apk add --no-cache curl && \
1717
curl -sSfL -o /tmp/vector.tar.gz \
1818
"https://packages.timber.io/vector/${VECTOR_VERSION}/vector-${VECTOR_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \

src/components/flow/detail-panel.tsx

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
"use client";
22

33
import { useCallback, useEffect, useMemo, useState } from "react";
4-
import { Copy, Trash2, Lock, Info, MousePointerClick } from "lucide-react";
4+
import { Copy, Trash2, Lock, Info, MousePointerClick, ExternalLink } from "lucide-react";
55
import { useFlowStore } from "@/stores/flow-store";
66
import { SchemaForm } from "@/components/config-forms/schema-form";
77
import { VrlEditor } from "@/components/vrl-editor/vrl-editor";
@@ -255,8 +255,17 @@ export function DetailPanel({ pipelineId, isDeployed }: DetailPanelProps) {
255255
<Card className="gap-4 py-4">
256256
<CardHeader className="pb-0">
257257
<div className="flex items-center justify-between gap-2">
258-
<CardTitle className="truncate text-base">
258+
<CardTitle className="flex items-center gap-1.5 truncate text-base">
259259
{componentDef.displayName}
260+
<a
261+
href={`https://vector.dev/docs/reference/configuration/${componentDef.kind}s/${componentDef.type}/`}
262+
target="_blank"
263+
rel="noopener noreferrer"
264+
className="inline-flex text-muted-foreground hover:text-foreground"
265+
aria-label="Open Vector docs"
266+
>
267+
<ExternalLink className="h-3.5 w-3.5" />
268+
</a>
260269
</CardTitle>
261270
<div className="flex items-center gap-1.5">
262271
<Badge

src/lib/vector/schemas/shared.ts

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -175,14 +175,19 @@ export function requestSchema() {
175175

176176
export function encodingSchema(
177177
codecs: string[] = [
178+
"avro",
179+
"cef",
180+
"csv",
181+
"gelf",
178182
"json",
179-
"ndjson",
180-
"text",
181183
"logfmt",
182-
"csv",
183-
"avro",
184-
"raw_message",
184+
"native",
185185
"native_json",
186+
"otlp",
187+
"protobuf",
188+
"raw_message",
189+
"syslog",
190+
"text",
186191
],
187192
) {
188193
return {

src/lib/vector/schemas/sinks/aws.ts

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -129,8 +129,8 @@ export const awsSinks: VectorComponentDef[] = [
129129
type: "string",
130130
description: "Timezone for date specifiers in templates (default: UTC)",
131131
},
132-
...encodingSchema(["avro", "cef", "csv", "gelf", "json", "logfmt", "native", "native_json", "protobuf", "raw_message", "text"]),
133-
...compressionSchema(["none", "gzip"], "gzip"),
132+
...encodingSchema(["avro", "cef", "csv", "gelf", "json", "logfmt", "native", "native_json", "otlp", "protobuf", "raw_message", "syslog", "text"]),
133+
...compressionSchema(["gzip", "none", "snappy", "zlib", "zstd"], "gzip"),
134134
...authAwsSchema(),
135135
...tlsSchema(),
136136
...batchSchema({ max_bytes: "10MB", timeout_secs: "300" }),
@@ -197,15 +197,15 @@ export const awsSinks: VectorComponentDef[] = [
197197
type: "string",
198198
description: "KMS key ARN for encrypting logs",
199199
},
200-
...encodingSchema(["avro", "cef", "csv", "gelf", "json", "logfmt", "native", "native_json", "protobuf", "raw_message", "text"]),
201-
...compressionSchema(["none", "gzip"], "none"),
200+
...encodingSchema(["avro", "cef", "csv", "gelf", "json", "logfmt", "native", "native_json", "otlp", "protobuf", "raw_message", "syslog", "text"]),
201+
...compressionSchema(["gzip", "none", "snappy", "zlib", "zstd"], "none"),
202202
...authAwsSchema(),
203203
...tlsSchema(),
204204
...batchSchema({ max_bytes: "1MB", timeout_secs: "1" }),
205205
...bufferSchema(),
206206
...requestSchema(),
207207
},
208-
required: ["group_name", "stream_name"],
208+
required: ["group_name", "stream_name", "encoding"],
209209
},
210210
},
211211
{
@@ -232,7 +232,7 @@ export const awsSinks: VectorComponentDef[] = [
232232
type: "string",
233233
description: "Custom endpoint for use with AWS-compatible services",
234234
},
235-
...compressionSchema(["none", "gzip"], "none"),
235+
...compressionSchema(["gzip", "none", "snappy", "zlib", "zstd"], "none"),
236236
...authAwsSchema(),
237237
...tlsSchema(),
238238
...batchSchema({ timeout_secs: "1" }),
@@ -269,15 +269,15 @@ export const awsSinks: VectorComponentDef[] = [
269269
type: "string",
270270
description: "Log field to use as the partition key for Kinesis records",
271271
},
272-
...encodingSchema(["avro", "cef", "csv", "gelf", "json", "logfmt", "native", "native_json", "protobuf", "raw_message", "text"]),
273-
...compressionSchema(["none", "gzip"], "none"),
272+
...encodingSchema(["avro", "cef", "csv", "gelf", "json", "logfmt", "native", "native_json", "otlp", "protobuf", "raw_message", "syslog", "text"]),
273+
...compressionSchema(["gzip", "none", "snappy", "zlib", "zstd"], "none"),
274274
...authAwsSchema(),
275275
...tlsSchema(),
276276
...batchSchema({ max_bytes: "4MB", timeout_secs: "1" }),
277277
...bufferSchema(),
278278
...requestSchema(),
279279
},
280-
required: ["stream_name"],
280+
required: ["stream_name", "encoding"],
281281
},
282282
},
283283
{
@@ -308,15 +308,15 @@ export const awsSinks: VectorComponentDef[] = [
308308
type: "string",
309309
description: "Custom endpoint for use with AWS-compatible services",
310310
},
311-
...encodingSchema(["avro", "cef", "csv", "gelf", "json", "logfmt", "native", "native_json", "protobuf", "raw_message", "text"]),
312-
...compressionSchema(["none", "gzip"], "none"),
311+
...encodingSchema(["avro", "cef", "csv", "gelf", "json", "logfmt", "native", "native_json", "otlp", "protobuf", "raw_message", "syslog", "text"]),
312+
...compressionSchema(["gzip", "none", "snappy", "zlib", "zstd"], "none"),
313313
...authAwsSchema(),
314314
...tlsSchema(),
315315
...batchSchema({ max_bytes: "5MB", timeout_secs: "1" }),
316316
...bufferSchema(),
317317
...requestSchema(),
318318
},
319-
required: ["stream_name"],
319+
required: ["stream_name", "encoding"],
320320
},
321321
},
322322
{
@@ -351,13 +351,13 @@ export const awsSinks: VectorComponentDef[] = [
351351
type: "string",
352352
description: "Deduplication ID for FIFO queues (template-enabled)",
353353
},
354-
...encodingSchema(["avro", "cef", "csv", "gelf", "json", "logfmt", "native", "native_json", "protobuf", "raw_message", "text"]),
354+
...encodingSchema(["avro", "cef", "csv", "gelf", "json", "logfmt", "native", "native_json", "otlp", "protobuf", "raw_message", "syslog", "text"]),
355355
...authAwsSchema(),
356356
...tlsSchema(),
357357
...bufferSchema(),
358358
...requestSchema(),
359359
},
360-
required: ["queue_url"],
360+
required: ["queue_url", "encoding"],
361361
},
362362
},
363363
{
@@ -372,9 +372,9 @@ export const awsSinks: VectorComponentDef[] = [
372372
configSchema: {
373373
type: "object",
374374
properties: {
375-
queue_url: {
375+
topic_arn: {
376376
type: "string",
377-
description: "SQS queue URL for SNS publishing",
377+
description: "The ARN of the SNS topic to publish to",
378378
},
379379
region: {
380380
type: "string",
@@ -392,13 +392,13 @@ export const awsSinks: VectorComponentDef[] = [
392392
type: "string",
393393
description: "Deduplication ID for FIFO queues (template-enabled)",
394394
},
395-
...encodingSchema(["avro", "cef", "csv", "gelf", "json", "logfmt", "native", "native_json", "protobuf", "raw_message", "text"]),
395+
...encodingSchema(["avro", "cef", "csv", "gelf", "json", "logfmt", "native", "native_json", "otlp", "protobuf", "raw_message", "syslog", "text"]),
396396
...authAwsSchema(),
397397
...tlsSchema(),
398398
...bufferSchema(),
399399
...requestSchema(),
400400
},
401-
required: ["queue_url"],
401+
required: ["topic_arn", "encoding"],
402402
},
403403
},
404404
];

src/lib/vector/schemas/sinks/azure.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,10 @@ export const azureSinks: VectorComponentDef[] = [
5656
"logfmt",
5757
"native",
5858
"native_json",
59+
"otlp",
5960
"protobuf",
6061
"raw_message",
62+
"syslog",
6163
"text",
6264
]),
6365
...compressionSchema(
@@ -68,7 +70,7 @@ export const azureSinks: VectorComponentDef[] = [
6870
...bufferSchema(),
6971
...requestSchema(),
7072
},
71-
required: ["connection_string", "container_name"],
73+
required: ["connection_string", "container_name", "encoding"],
7274
},
7375
},
7476
{

src/lib/vector/schemas/sinks/gcp.ts

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,10 @@ export const gcpSinks: VectorComponentDef[] = [
108108
"logfmt",
109109
"native",
110110
"native_json",
111+
"otlp",
111112
"protobuf",
112113
"raw_message",
114+
"syslog",
113115
"text",
114116
]),
115117
...compressionSchema(
@@ -121,7 +123,7 @@ export const gcpSinks: VectorComponentDef[] = [
121123
...bufferSchema(),
122124
...requestSchema(),
123125
},
124-
required: ["bucket"],
126+
required: ["bucket", "encoding"],
125127
},
126128
},
127129
{
@@ -225,7 +227,7 @@ export const gcpSinks: VectorComponentDef[] = [
225227
...bufferSchema(),
226228
...requestSchema(),
227229
},
228-
required: ["log_id"],
230+
required: ["log_id", "resource"],
229231
},
230232
},
231233
{
@@ -269,7 +271,7 @@ export const gcpSinks: VectorComponentDef[] = [
269271
...bufferSchema(),
270272
...requestSchema(),
271273
},
272-
required: ["project_id"],
274+
required: ["project_id", "resource"],
273275
},
274276
},
275277
{
@@ -308,16 +310,18 @@ export const gcpSinks: VectorComponentDef[] = [
308310
"logfmt",
309311
"native",
310312
"native_json",
313+
"otlp",
311314
"protobuf",
312315
"raw_message",
316+
"syslog",
313317
"text",
314318
]),
315319
...tlsSchema(),
316320
...batchSchema({ max_bytes: "10MB", timeout_secs: "1" }),
317321
...bufferSchema(),
318322
...requestSchema(),
319323
},
320-
required: ["project", "topic"],
324+
required: ["project", "topic", "encoding"],
321325
},
322326
},
323327
{
@@ -357,7 +361,6 @@ export const gcpSinks: VectorComponentDef[] = [
357361
"us",
358362
"eu",
359363
"asia",
360-
"australia-southeast1",
361364
"canada",
362365
"dammam",
363366
"doha",
@@ -396,17 +399,19 @@ export const gcpSinks: VectorComponentDef[] = [
396399
"logfmt",
397400
"native",
398401
"native_json",
402+
"otlp",
399403
"protobuf",
400404
"raw_message",
405+
"syslog",
401406
"text",
402407
]),
403-
...compressionSchema(["gzip", "none"], "none"),
408+
...compressionSchema(["gzip", "none", "snappy", "zlib", "zstd"], "none"),
404409
...tlsSchema(),
405410
...batchSchema({ max_bytes: "1MB", timeout_secs: "15" }),
406411
...bufferSchema(),
407412
...requestSchema(),
408413
},
409-
required: ["customer_id", "log_type"],
414+
required: ["customer_id", "log_type", "encoding"],
410415
},
411416
},
412417
];

src/lib/vector/schemas/sinks/logging.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ export const loggingSinks: VectorComponentDef[] = [
9191
...bufferSchema(),
9292
...requestSchema(),
9393
},
94-
required: ["endpoint"],
94+
required: ["endpoint", "encoding"],
9595
},
9696
},
9797
{
@@ -146,7 +146,7 @@ export const loggingSinks: VectorComponentDef[] = [
146146
...tlsSchema(),
147147
...bufferSchema(),
148148
},
149-
required: ["endpoint"],
149+
required: ["endpoint", "encoding"],
150150
},
151151
},
152152
{
@@ -236,7 +236,7 @@ export const loggingSinks: VectorComponentDef[] = [
236236
...bufferSchema(),
237237
...requestSchema(),
238238
},
239-
required: ["endpoint", "default_token"],
239+
required: ["endpoint", "default_token", "encoding"],
240240
},
241241
},
242242
{

0 commit comments

Comments
 (0)