From 4701c1391b6e5902abbfa77f2be11a582774f47d Mon Sep 17 00:00:00 2001 From: Andy Georges Date: Wed, 25 Sep 2024 15:09:16 +0200 Subject: [PATCH 01/10] feat: add optional argument forcing syslog parcing to one of both RFCs Allows giving priority to the older RFC 3164 over the newer RFC 5424. Defaults to the original behaviour of using the newer format over the older. --- src/stdlib/parse_linux_authorization.rs | 6 ++- src/stdlib/parse_syslog.rs | 59 ++++++++++++++++++------- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/stdlib/parse_linux_authorization.rs b/src/stdlib/parse_linux_authorization.rs index adc53ef0c7..aaa97ae445 100644 --- a/src/stdlib/parse_linux_authorization.rs +++ b/src/stdlib/parse_linux_authorization.rs @@ -41,6 +41,10 @@ impl Function for ParseLinuxAuthorization { let value = arguments.required("value"); // The parse_linux_authorization function is just an alias for parse_syslog - Ok(ParseSyslogFn { value }.as_expr()) + Ok(ParseSyslogFn { + value: value, + variant: "either".into(), + } + .as_expr()) } } diff --git a/src/stdlib/parse_syslog.rs b/src/stdlib/parse_syslog.rs index c227514653..abb7a88297 100644 --- a/src/stdlib/parse_syslog.rs +++ b/src/stdlib/parse_syslog.rs @@ -1,20 +1,17 @@ use crate::compiler::prelude::*; use chrono::{DateTime, Datelike, Utc}; +use nom::AsBytes; use std::collections::BTreeMap; use syslog_loose::{IncompleteDate, Message, ProcId, Protocol, Variant}; -pub(crate) fn parse_syslog(value: Value, ctx: &Context) -> Resolved { +pub(crate) fn parse_syslog(value: Value, variant: Variant, ctx: &Context) -> Resolved { let message = value.try_bytes_utf8_lossy()?; let timezone = match ctx.timezone() { TimeZone::Local => None, TimeZone::Named(tz) => Some(*tz), }; - let parsed = syslog_loose::parse_message_with_year_exact_tz( - &message, - resolve_year, - timezone, - Variant::Either, - )?; + let parsed = + syslog_loose::parse_message_with_year_exact_tz(&message, resolve_year, timezone, variant)?; Ok(message_to_value(parsed)) } @@ -27,11 +24,18 @@ impl Function for ParseSyslog { } fn parameters(&self) -> &'static [Parameter] { - &[Parameter { - keyword: "value", - kind: kind::BYTES, - required: true, - }] + &[ + Parameter { + keyword: "value", + kind: kind::BYTES, + required: true, + }, + Parameter { + keyword: "variant", + kind: kind::BYTES, + required: false, + }, + ] } fn examples(&self) -> &'static [Example] { @@ -59,26 +63,39 @@ impl Function for ParseSyslog { fn compile( &self, - _state: &state::TypeState, + state: &state::TypeState, _ctx: &mut FunctionCompileContext, arguments: ArgumentList, ) -> Compiled { + let variants = vec!["either".into(), "rfc3164".into(), "rfc5424".into()]; + let value = arguments.required("value"); + let variant = arguments + .optional_enum("variant", &variants, state)? + .unwrap_or_else(|| "either".into()) + .try_bytes() + .expect("syslog variant is not bytes"); - Ok(ParseSyslogFn { value }.as_expr()) + Ok(ParseSyslogFn { value, variant }.as_expr()) } } #[derive(Debug, Clone)] pub(crate) struct ParseSyslogFn { pub(crate) value: Box, + pub(crate) variant: Bytes, } impl FunctionExpression for ParseSyslogFn { fn resolve(&self, ctx: &mut Context) -> Resolved { let value = self.value.resolve(ctx)?; + let variant = match self.variant.as_bytes() { + b"rfc3164" => Variant::RFC3164, + b"rfc5424" => Variant::RFC5424, + _ => Variant::Either, + }; - parse_syslog(value, ctx) + parse_syslog(value, variant, ctx) } fn type_def(&self, _: &state::TypeState) -> TypeDef { @@ -356,5 +373,17 @@ mod tests { }), tdef: TypeDef::object(inner_kind()).fallible(), } + + force_rfc_3164 { + args: func_args![value: "2024-09-19T15:39:45.469435+02:00 node1234 slurmstepd[548422]: [65684352.batch] done with job", variant: "rfc3164"], + want: Ok(btreemap!{ + "appname" => "slurmstepd", + "hostname" => "node3521", + "message" => "[65684352.batch] done with job", + "procid" => 548422, + "timestamp" => "2024-09-19T13:39:45.469435Z" + }), + tdef: TypeDef::object(inner_kind()).fallible(), + } ]; } From 894a3a320016c5ab8735a46c31fc0598ccc7560b Mon Sep 17 00:00:00 2001 From: Andy Georges Date: Wed, 25 Sep 2024 15:48:10 +0200 Subject: [PATCH 02/10] fix: move catchall case to panic, should never occur here. --- src/stdlib/parse_syslog.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/stdlib/parse_syslog.rs b/src/stdlib/parse_syslog.rs index abb7a88297..88a993ade1 100644 --- a/src/stdlib/parse_syslog.rs +++ b/src/stdlib/parse_syslog.rs @@ -92,7 +92,8 @@ impl FunctionExpression for ParseSyslogFn { let variant = match self.variant.as_bytes() { b"rfc3164" => Variant::RFC3164, b"rfc5424" => Variant::RFC5424, - _ => Variant::Either, + b"either" => Variant::Either, + _ => panic!("No such variant for syslog RFC"), }; parse_syslog(value, variant, ctx) From d4e7f89712f9082a3f4a3a02bf3a290fbdbae59b Mon Sep 17 00:00:00 2001 From: Andy Georges Date: Wed, 25 Sep 2024 23:13:12 +0200 Subject: [PATCH 03/10] fix: use patched syslog_loose Ditch support for structured data inside RFC3164 messages --- Cargo.lock | 3 +- Cargo.toml | 340 +++++++++++++++++++++++++------------ src/stdlib/parse_syslog.rs | 6 +- 3 files changed, 239 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a82e86f5f8..333a8e4b84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3029,8 +3029,7 @@ dependencies = [ [[package]] name = "syslog_loose" version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "161028c00842709450114c39db3b29f44c898055ed8833bb9b535aba7facf30e" +source = "git+https://github.com/itkovian/syslog-loose?rev=7acfd39be5bd#7acfd39be5bdac043dee9955a9e59d839b0ce950" dependencies = [ "chrono", "nom", diff --git a/Cargo.toml b/Cargo.toml index e6f5a6fad7..5a2bfc42f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,39 +10,117 @@ repository = "https://github.com/vectordotdev/vrl" readme = "README.md" keywords = ["vector", "datadog", "compiler"] categories = ["compilers"] -rust-version = "1.79" # msrv +rust-version = "1.79" # msrv [workspace] -members = [ - ".", - "lib/cli", - "lib/tests", - "lib/fuzz" -] +members = [".", "lib/cli", "lib/tests", "lib/fuzz"] [features] -default = ["compiler", "value", "diagnostic", "path", "parser", "stdlib", "datadog", "core"] +default = [ + "compiler", + "value", + "diagnostic", + "path", + "parser", + "stdlib", + "datadog", + "core", +] # Main features (on by default) -compiler = ["diagnostic", "path", "parser", "value", "dep:paste", "dep:chrono", "dep:serde", "dep:regex", "dep:bytes", "dep:ordered-float", "dep:chrono-tz", "dep:snafu", "dep:thiserror", "dep:dyn-clone", "dep:indoc", "dep:thiserror", "dep:lalrpop-util"] -value = ["path", "dep:bytes", "dep:regex", "dep:ordered-float", "dep:chrono", "dep:serde_json"] +compiler = [ + "diagnostic", + "path", + "parser", + "value", + "dep:paste", + "dep:chrono", + "dep:serde", + "dep:regex", + "dep:bytes", + "dep:ordered-float", + "dep:chrono-tz", + "dep:snafu", + "dep:thiserror", + "dep:dyn-clone", + "dep:indoc", + "dep:thiserror", + "dep:lalrpop-util", +] +value = [ + "path", + "dep:bytes", + "dep:regex", + "dep:ordered-float", + "dep:chrono", + "dep:serde_json", +] diagnostic = ["dep:codespan-reporting", "dep:termcolor"] path = ["value", "dep:once_cell", "dep:serde", "dep:snafu", "dep:regex"] -parser = ["path", "diagnostic", "value", "dep:thiserror", "dep:ordered-float", "dep:lalrpop-util"] -parsing = ["value", "compiler", "dep:url", "dep:nom", "dep:regex", "dep:roxmltree", "dep:rust_decimal"] +parser = [ + "path", + "diagnostic", + "value", + "dep:thiserror", + "dep:ordered-float", + "dep:lalrpop-util", +] +parsing = [ + "value", + "compiler", + "dep:url", + "dep:nom", + "dep:regex", + "dep:roxmltree", + "dep:rust_decimal", +] core = ["value", "dep:snafu", "dep:nom"] string_path = [] # Datadog related features (on by default) datadog = ["datadog_filter", "datadog_grok", "datadog_search"] datadog_filter = ["path", "datadog_search", "dep:regex", "dep:dyn-clone"] -datadog_grok = ["value", "parsing", "dep:nom", "dep:peeking_take_while", "dep:serde_json", "dep:onig", "dep:lalrpop-util", "dep:thiserror", "dep:chrono", "dep:chrono-tz", "dep:percent-encoding", "dep:fancy-regex"] -datadog_search = ["dep:pest", "dep:pest_derive", "dep:itertools", "dep:once_cell", "dep:regex", "dep:serde"] +datadog_grok = [ + "value", + "parsing", + "dep:nom", + "dep:peeking_take_while", + "dep:serde_json", + "dep:onig", + "dep:lalrpop-util", + "dep:thiserror", + "dep:chrono", + "dep:chrono-tz", + "dep:percent-encoding", + "dep:fancy-regex", +] +datadog_search = [ + "dep:pest", + "dep:pest_derive", + "dep:itertools", + "dep:once_cell", + "dep:regex", + "dep:serde", +] # Features that aren't used as often (default off) -cli = ["stdlib", "dep:serde_json", "dep:thiserror", "dep:clap", "dep:exitcode", "dep:webbrowser", "dep:rustyline", "dep:prettytable-rs"] -test_framework = ["compiler", "dep:prettydiff", "dep:serde_json", "dep:ansi_term"] +cli = [ + "stdlib", + "dep:serde_json", + "dep:thiserror", + "dep:clap", + "dep:exitcode", + "dep:webbrowser", + "dep:rustyline", + "dep:prettytable-rs", +] +test_framework = [ + "compiler", + "dep:prettydiff", + "dep:serde_json", + "dep:ansi_term", +] arbitrary = ["dep:quickcheck", "dep:arbitrary"] lua = ["dep:mlua"] proptest = ["dep:proptest", "dep:proptest-derive"] @@ -53,135 +131,164 @@ test = ["string_path"] # All stdlib functions stdlib = [ - "compiler", - "core", - "datadog", - "parsing", - "dep:aes", - "dep:base16", - "dep:base62", - "dep:base64", - "dep:cbc", - "dep:cfb-mode", - "dep:chacha20poly1305", - "dep:charset", - "dep:convert_case", - "dep:cidr-utils", - "dep:community-id", - "dep:crypto_secretbox", - "dep:csv", - "dep:ctr", - "dep:data-encoding", - "dep:digest", - "dep:domain", - "dep:dns-lookup", - "dep:flate2", - "dep:grok", - "dep:hex", - "dep:hmac", - "dep:hostname", - "dep:iana-time-zone", - "dep:idna", - "dep:indexmap", - "dep:influxdb-line-protocol", - "dep:md-5", - "dep:nom", - "dep:ofb", - "dep:once_cell", - "dep:percent-encoding", - "dep:prost", - "dep:prost-reflect", - "dep:psl", - "dep:psl-types", - "dep:publicsuffix", - "dep:quoted_printable", - "dep:rand", - "dep:roxmltree", - "dep:rust_decimal", - "dep:seahash", - "dep:sha-1", - "dep:sha-2", - "dep:sha-3", - "dep:snap", - "dep:strip-ansi-escapes", - "dep:syslog_loose", - "dep:tokio", - "dep:uaparser", - "dep:url", - "dep:utf8-width", - "dep:uuid", - "dep:woothee", - "dep:zstd", - ] + "compiler", + "core", + "datadog", + "parsing", + "dep:aes", + "dep:base16", + "dep:base62", + "dep:base64", + "dep:cbc", + "dep:cfb-mode", + "dep:chacha20poly1305", + "dep:charset", + "dep:convert_case", + "dep:cidr-utils", + "dep:community-id", + "dep:crypto_secretbox", + "dep:csv", + "dep:ctr", + "dep:data-encoding", + "dep:digest", + "dep:domain", + "dep:dns-lookup", + "dep:flate2", + "dep:grok", + "dep:hex", + "dep:hmac", + "dep:hostname", + "dep:iana-time-zone", + "dep:idna", + "dep:indexmap", + "dep:influxdb-line-protocol", + "dep:md-5", + "dep:nom", + "dep:ofb", + "dep:once_cell", + "dep:percent-encoding", + "dep:prost", + "dep:prost-reflect", + "dep:psl", + "dep:psl-types", + "dep:publicsuffix", + "dep:quoted_printable", + "dep:rand", + "dep:roxmltree", + "dep:rust_decimal", + "dep:seahash", + "dep:sha-1", + "dep:sha-2", + "dep:sha-3", + "dep:snap", + "dep:strip-ansi-escapes", + "dep:syslog_loose", + "dep:tokio", + "dep:uaparser", + "dep:url", + "dep:utf8-width", + "dep:uuid", + "dep:woothee", + "dep:zstd", +] [dependencies] cfg-if = "1" # Optional dependencies -ansi_term = {version = "0.12", optional = true } +ansi_term = { version = "0.12", optional = true } arbitrary = { version = "1", optional = true, features = ["derive"] } base16 = { version = "0.2", optional = true } base62 = { version = "2.0.2", optional = true } base64 = { version = "0.22", optional = true } bytes = { version = "1", default-features = false, optional = true } charset = { version = "0.1", optional = true } -chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"], optional = true } +chrono = { version = "0.4", default-features = false, features = [ + "clock", + "serde", + "wasmbind", +], optional = true } chrono-tz = { version = "0.10", default-features = false, optional = true } cidr-utils = { version = "0.6", optional = true } csv = { version = "1", optional = true } clap = { version = "4", features = ["derive"], optional = true } -codespan-reporting = {version = "0.11", optional = true } +codespan-reporting = { version = "0.11", optional = true } convert_case = { version = "0.6.0", optional = true } data-encoding = { version = "2", optional = true } digest = { version = "0.10", optional = true } dyn-clone = { version = "1", default-features = false, optional = true } -exitcode = {version = "1", optional = true } -flate2 = { version = "1", default-features = false, features = ["default"], optional = true } +exitcode = { version = "1", optional = true } +flate2 = { version = "1", default-features = false, features = [ + "default", +], optional = true } hex = { version = "0.4", optional = true } hmac = { version = "0.12", optional = true } iana-time-zone = { version = "0.1", optional = true } idna = { version = "0.5", optional = true } -indexmap = { version = "2", default-features = false, features = ["std"], optional = true} +indexmap = { version = "2", default-features = false, features = [ + "std", +], optional = true } influxdb-line-protocol = { version = "2.0.0", optional = true } -indoc = {version = "2", optional = true } -itertools = { version = "0.13", default-features = false, features=["use_std"], optional = true } +indoc = { version = "2", optional = true } +itertools = { version = "0.13", default-features = false, features = [ + "use_std", +], optional = true } lalrpop-util = { version = "0.21", optional = true } -mlua = { version = "0.9", default-features = false, features = ["lua54", "send", "vendored"], optional = true} -nom = { version = "7", default-features = false, features = ["std"], optional = true } -once_cell = { version = "1", default-features = false, features = ["std"], optional = true } +mlua = { version = "0.9", default-features = false, features = [ + "lua54", + "send", + "vendored", +], optional = true } +nom = { version = "7", default-features = false, features = [ + "std", +], optional = true } +once_cell = { version = "1", default-features = false, features = [ + "std", +], optional = true } ordered-float = { version = "4", default-features = false, optional = true } md-5 = { version = "0.10", optional = true } paste = { version = "1", default-features = false, optional = true } peeking_take_while = { version = "1", default-features = false, optional = true } percent-encoding = { version = "2", optional = true } -pest = { version = "2", default-features = false, optional = true, features = ["std"] } -pest_derive = { version = "2", default-features = false, optional = true, features = ["std"] } +pest = { version = "2", default-features = false, optional = true, features = [ + "std", +] } +pest_derive = { version = "2", default-features = false, optional = true, features = [ + "std", +] } proptest = { version = "1", optional = true } proptest-derive = { version = "0.4", optional = true } -prettydiff = {version = "0.7", default-features = false, optional = true } +prettydiff = { version = "0.7", default-features = false, optional = true } prettytable-rs = { version = "0.10", default-features = false, optional = true } quickcheck = { version = "1", optional = true } -quoted_printable = {version = "0.5", optional = true } +quoted_printable = { version = "0.5", optional = true } psl = { version = "2", optional = true } psl-types = { version = "2", optional = true } publicsuffix = { version = "2", optional = true } rand = { version = "0.8", optional = true } -regex = { version = "1", default-features = false, optional = true, features = ["std", "perf", "unicode"] } +regex = { version = "1", default-features = false, optional = true, features = [ + "std", + "perf", + "unicode", +] } roxmltree = { version = "0.20", optional = true } rustyline = { version = "14", default-features = false, optional = true } rust_decimal = { version = "1", optional = true } seahash = { version = "4", optional = true } serde = { version = "1", features = ["derive"], optional = true } -serde_json = { version = "1", default-features = false, optional = true, features = ["std", "raw_value"] } +serde_json = { version = "1", default-features = false, optional = true, features = [ + "std", + "raw_value", +] } fancy-regex = { version = "0.13.0", default-features = false, optional = true } sha-1 = { version = "0.10", optional = true } sha-2 = { package = "sha2", version = "0.10", optional = true } sha-3 = { package = "sha3", version = "0.10", optional = true } strip-ansi-escapes = { version = "0.2", optional = true } snap = { version = "1", optional = true } -syslog_loose = { version = "0.21", optional = true } -termcolor = {version = "1", optional = true } -thiserror ={ version = "1", optional = true } +syslog_loose = { git = "https://github.com/itkovian/syslog-loose", rev = "7acfd39be5bd", optional = true } +termcolor = { version = "1", optional = true } +thiserror = { version = "1", optional = true } tracing = { version = "0.1", default-features = false } uaparser = { version = "0.6", default-features = false, optional = true } utf8-width = { version = "0.1", optional = true } @@ -189,9 +296,11 @@ url = { version = "2", optional = true } snafu = { version = "0.8", optional = true } webbrowser = { version = "1.0", default-features = false, optional = true } woothee = { version = "0.13", optional = true } -community-id = { version = "0.2", optional = true} +community-id = { version = "0.2", optional = true } -zstd = { version = "0.13", default-features = false, features = ["wasm"], optional = true } +zstd = { version = "0.13", default-features = false, features = [ + "wasm", +], optional = true } # Cryptography aes = { version = "0.8", optional = true } @@ -205,17 +314,30 @@ cfb-mode = { version = "0.8", optional = true } ofb = { version = "0.6", optional = true } # Protobuf support. -prost = { version = "0.13", default-features = false, optional = true, features = ["std"]} -prost-reflect = { version = "0.14", default-features = false, optional = true} +prost = { version = "0.13", default-features = false, optional = true, features = [ + "std", +] } +prost-reflect = { version = "0.14", default-features = false, optional = true } # Dependencies used for non-WASM [target.'cfg(not(target_arch = "wasm32"))'.dependencies] dns-lookup = { version = "2", optional = true } -domain = { version = "0.10.1", optional = true, features = ["resolv-sync", "serde"] } +domain = { version = "0.10.1", optional = true, features = [ + "resolv-sync", + "serde", +] } hostname = { version = "0.4", optional = true } grok = { version = "2", optional = true } onig = { version = "6", default-features = false, optional = true } -tokio = { version = "1.38", optional = true, features = ["io-util", "macros", "net", "time", "sync", "rt", "rt-multi-thread" ] } +tokio = { version = "1.38", optional = true, features = [ + "io-util", + "macros", + "net", + "time", + "sync", + "rt", + "rt-multi-thread", +] } uuid = { version = "1", features = ["v4", "v7"], optional = true } # Dependencies used for WASM @@ -230,9 +352,17 @@ serde_json = "1" indoc = "2" tracing-test = { version = "0.2", default-features = false } toml = { version = "0.8", default-features = false } -mlua = { version = "0.9", default-features = false, features = ["lua54", "send", "vendored"]} -quickcheck = { version = "1"} -regex = { version = "1", default-features = false, features = ["std", "perf", "unicode"] } +mlua = { version = "0.9", default-features = false, features = [ + "lua54", + "send", + "vendored", +] } +quickcheck = { version = "1" } +regex = { version = "1", default-features = false, features = [ + "std", + "perf", + "unicode", +] } paste = { version = "1", default-features = false } proptest = { version = "1" } proptest-derive = { version = "0.4" } diff --git a/src/stdlib/parse_syslog.rs b/src/stdlib/parse_syslog.rs index 88a993ade1..0e85a87218 100644 --- a/src/stdlib/parse_syslog.rs +++ b/src/stdlib/parse_syslog.rs @@ -376,13 +376,13 @@ mod tests { } force_rfc_3164 { - args: func_args![value: "2024-09-19T15:39:45.469435+02:00 node1234 slurmstepd[548422]: [65684352.batch] done with job", variant: "rfc3164"], + args: func_args![value: "2024-09-19T15:39:45.469+02:00 node1234 slurmstepd[548422]: [65684352.batch] done with job", variant: "rfc3164"], want: Ok(btreemap!{ "appname" => "slurmstepd", - "hostname" => "node3521", + "hostname" => "node1234", "message" => "[65684352.batch] done with job", "procid" => 548422, - "timestamp" => "2024-09-19T13:39:45.469435Z" + "timestamp" => chrono::Utc.ymd(2024, 09, 19).and_hms_milli(13,39,45,469), }), tdef: TypeDef::object(inner_kind()).fallible(), } From 858e6d5a37adf2e51cea16c49a2bb9b1b31b927d Mon Sep 17 00:00:00 2001 From: Andy Georges Date: Wed, 7 May 2025 17:10:33 +0200 Subject: [PATCH 04/10] fix: deps --- Cargo.toml | 338 ++++++++++++++++++++++++++--------------------------- 1 file changed, 169 insertions(+), 169 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index da143bb35d..b3486dcaf4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,62 +20,62 @@ clap = { version = "4", features = ["derive"] } [features] default = [ - "compiler", - "value", - "diagnostic", - "path", - "parser", - "stdlib", - "datadog", - "core", + "compiler", + "value", + "diagnostic", + "path", + "parser", + "stdlib", + "datadog", + "core", ] # Main features (on by default) compiler = [ - "diagnostic", - "path", - "parser", - "value", - "dep:paste", - "dep:chrono", - "dep:serde", - "dep:regex", - "dep:bytes", - "dep:ordered-float", - "dep:chrono-tz", - "dep:snafu", - "dep:thiserror", - "dep:dyn-clone", - "dep:indoc", - "dep:thiserror", - "dep:lalrpop-util", + "diagnostic", + "path", + "parser", + "value", + "dep:paste", + "dep:chrono", + "dep:serde", + "dep:regex", + "dep:bytes", + "dep:ordered-float", + "dep:chrono-tz", + "dep:snafu", + "dep:thiserror", + "dep:dyn-clone", + "dep:indoc", + "dep:thiserror", + "dep:lalrpop-util", ] value = [ - "path", - "dep:bytes", - "dep:regex", - "dep:ordered-float", - "dep:chrono", - "dep:serde_json", + "path", + "dep:bytes", + "dep:regex", + "dep:ordered-float", + "dep:chrono", + "dep:serde_json", ] diagnostic = ["dep:codespan-reporting", "dep:termcolor"] path = ["value", "dep:once_cell", "dep:serde", "dep:snafu", "dep:regex"] parser = [ - "path", - "diagnostic", - "value", - "dep:thiserror", - "dep:ordered-float", - "dep:lalrpop-util", + "path", + "diagnostic", + "value", + "dep:thiserror", + "dep:ordered-float", + "dep:lalrpop-util", ] parsing = [ - "value", - "compiler", - "dep:url", - "dep:nom", - "dep:regex", - "dep:roxmltree", - "dep:rust_decimal", + "value", + "compiler", + "dep:url", + "dep:nom", + "dep:regex", + "dep:roxmltree", + "dep:rust_decimal", ] core = ["value", "dep:snafu", "dep:nom"] string_path = [] @@ -84,44 +84,44 @@ string_path = [] datadog = ["datadog_filter", "datadog_grok", "datadog_search"] datadog_filter = ["path", "datadog_search", "dep:regex", "dep:dyn-clone"] datadog_grok = [ - "value", - "parsing", - "dep:nom", - "dep:peeking_take_while", - "dep:serde_json", - "dep:onig", - "dep:lalrpop-util", - "dep:thiserror", - "dep:chrono", - "dep:chrono-tz", - "dep:percent-encoding", - "dep:fancy-regex", + "value", + "parsing", + "dep:nom", + "dep:peeking_take_while", + "dep:serde_json", + "dep:onig", + "dep:lalrpop-util", + "dep:thiserror", + "dep:chrono", + "dep:chrono-tz", + "dep:percent-encoding", + "dep:fancy-regex", ] datadog_search = [ - "dep:pest", - "dep:pest_derive", - "dep:itertools", - "dep:once_cell", - "dep:regex", - "dep:serde", + "dep:pest", + "dep:pest_derive", + "dep:itertools", + "dep:once_cell", + "dep:regex", + "dep:serde", ] # Features that aren't used as often (default off) cli = [ - "stdlib", - "dep:serde_json", - "dep:thiserror", - "dep:clap", - "dep:exitcode", - "dep:webbrowser", - "dep:rustyline", - "dep:prettytable-rs", + "stdlib", + "dep:serde_json", + "dep:thiserror", + "dep:clap", + "dep:exitcode", + "dep:webbrowser", + "dep:rustyline", + "dep:prettytable-rs", ] test_framework = [ - "compiler", - "dep:prettydiff", - "dep:serde_json", - "dep:ansi_term", + "compiler", + "dep:prettydiff", + "dep:serde_json", + "dep:ansi_term", ] arbitrary = ["dep:quickcheck", "dep:arbitrary"] lua = ["dep:mlua"] @@ -134,65 +134,65 @@ test = ["string_path"] # All stdlib functions stdlib = [ - "compiler", - "core", - "datadog", - "parsing", - "dep:aes", - "dep:base16", - "dep:base62", - "dep:base64", - "dep:cbc", - "dep:cfb-mode", - "dep:chacha20poly1305", - "dep:charset", - "dep:convert_case", - "dep:cidr-utils", - "dep:community-id", - "dep:crypto_secretbox", - "dep:csv", - "dep:ctr", - "dep:data-encoding", - "dep:digest", - "dep:domain", - "dep:dns-lookup", - "dep:flate2", - "dep:grok", - "dep:hex", - "dep:hmac", - "dep:hostname", - "dep:iana-time-zone", - "dep:idna", - "dep:indexmap", - "dep:influxdb-line-protocol", - "dep:md-5", - "dep:nom", - "dep:ofb", - "dep:once_cell", - "dep:percent-encoding", - "dep:prost", - "dep:prost-reflect", - "dep:psl", - "dep:psl-types", - "dep:publicsuffix", - "dep:quoted_printable", - "dep:rand", - "dep:roxmltree", - "dep:rust_decimal", - "dep:seahash", - "dep:sha-1", - "dep:sha-2", - "dep:sha-3", - "dep:snap", - "dep:strip-ansi-escapes", - "dep:syslog_loose", - "dep:tokio", - "dep:uaparser", - "dep:url", - "dep:utf8-width", - "dep:uuid", - "dep:woothee", - "dep:zstd", + "compiler", + "core", + "datadog", + "parsing", + "dep:aes", + "dep:base16", + "dep:base62", + "dep:base64", + "dep:cbc", + "dep:cfb-mode", + "dep:chacha20poly1305", + "dep:charset", + "dep:convert_case", + "dep:cidr", + "dep:community-id", + "dep:crypto_secretbox", + "dep:csv", + "dep:ctr", + "dep:data-encoding", + "dep:digest", + "dep:domain", + "dep:dns-lookup", + "dep:flate2", + "dep:grok", + "dep:hex", + "dep:hmac", + "dep:hostname", + "dep:iana-time-zone", + "dep:idna", + "dep:indexmap", + "dep:influxdb-line-protocol", + "dep:md-5", + "dep:nom", + "dep:ofb", + "dep:once_cell", + "dep:percent-encoding", + "dep:prost", + "dep:prost-reflect", + "dep:psl", + "dep:psl-types", + "dep:publicsuffix", + "dep:quoted_printable", + "dep:rand", + "dep:roxmltree", + "dep:rust_decimal", + "dep:seahash", + "dep:sha-1", + "dep:sha-2", + "dep:sha-3", + "dep:snap", + "dep:strip-ansi-escapes", + "dep:syslog_loose", + "dep:tokio", + "dep:ua-parser", + "dep:url", + "dep:utf8-width", + "dep:uuid", + "dep:woothee", + "dep:zstd", ] [dependencies] @@ -207,9 +207,9 @@ base64 = { version = "0.22", optional = true } bytes = { version = "1", default-features = false, optional = true } charset = { version = "0.1", optional = true } chrono = { version = "0.4", default-features = false, features = [ - "clock", - "serde", - "wasmbind", + "clock", + "serde", + "wasmbind", ], optional = true } chrono-tz = { version = "0.10", default-features = false, optional = true } ciborium = { version = "0.2.2", default-features = false, optional = true } @@ -223,42 +223,42 @@ digest = { version = "0.10", optional = true } dyn-clone = { version = "1", default-features = false, optional = true } exitcode = { version = "1", optional = true } flate2 = { version = "1", default-features = false, features = [ - "default", + "default", ], optional = true } hex = { version = "0.4", optional = true } hmac = { version = "0.12", optional = true } iana-time-zone = { version = "0.1", optional = true } idna = { version = "0.5", optional = true } indexmap = { version = "2", default-features = false, features = [ - "std", + "std", ], optional = true } influxdb-line-protocol = { version = "2.0.0", optional = true } indoc = { version = "2", optional = true } itertools = { version = "0.13", default-features = false, features = [ - "use_std", + "use_std", ], optional = true } lalrpop-util = { version = "0.21", optional = true } mlua = { version = "0.9", default-features = false, features = [ - "lua54", - "send", - "vendored", + "lua54", + "send", + "vendored", ], optional = true } nom = { version = "7", default-features = false, features = [ - "std", + "std", ], optional = true } once_cell = { version = "1", default-features = false, features = [ - "std", + "std", ], optional = true } ordered-float = { version = "4", default-features = false, optional = true } md-5 = { version = "0.10", optional = true } -parse-size = { version = "1.1.0", optional = true } +parse-size = { version = "1.1.0", optional = true } peeking_take_while = { version = "1", default-features = false, optional = true } percent-encoding = { version = "2", optional = true } pest = { version = "2", default-features = false, optional = true, features = [ - "std", + "std", ] } pest_derive = { version = "2", default-features = false, optional = true, features = [ - "std", + "std", ] } proptest = { version = "1", optional = true } proptest-derive = { version = "0.4", optional = true } @@ -271,9 +271,9 @@ psl-types = { version = "2", optional = true } publicsuffix = { version = "2", optional = true } rand = { version = "0.8", optional = true } regex = { version = "1", default-features = false, optional = true, features = [ - "std", - "perf", - "unicode", + "std", + "perf", + "unicode", ] } roxmltree = { version = "0.20", optional = true } rustyline = { version = "15", default-features = false, optional = true } @@ -281,8 +281,8 @@ rust_decimal = { version = "1", optional = true, default-features = false } seahash = { version = "4", optional = true } serde = { version = "1", features = ["derive"], optional = true } serde_json = { version = "1", default-features = false, optional = true, features = [ - "std", - "raw_value", + "std", + "raw_value", ] } fancy-regex = { version = "0.13.0", default-features = false, optional = true } sha-1 = { version = "0.10", optional = true } @@ -303,7 +303,7 @@ woothee = { version = "0.13", optional = true } community-id = { version = "0.2", optional = true } zstd = { version = "0.13", default-features = false, features = [ - "wasm", + "wasm", ], optional = true } # Cryptography @@ -320,7 +320,7 @@ ofb = { version = "0.6", optional = true } # Protobuf support. prost = { version = "0.13", default-features = false, optional = true, features = [ - "std", + "std", ] } prost-reflect = { version = "0.14", default-features = false, optional = true } @@ -328,20 +328,20 @@ prost-reflect = { version = "0.14", default-features = false, optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] dns-lookup = { version = "2", optional = true } domain = { version = "0.10.1", optional = true, features = [ - "resolv-sync", - "serde", + "resolv-sync", + "serde", ] } hostname = { version = "0.4", optional = true } grok = { version = "2", optional = true } onig = { version = "6", default-features = false, optional = true } tokio = { version = "1.38", optional = true, features = [ - "io-util", - "macros", - "net", - "time", - "sync", - "rt", - "rt-multi-thread", + "io-util", + "macros", + "net", + "time", + "sync", + "rt", + "rt-multi-thread", ] } uuid = { version = "1", features = ["v4", "v7"], optional = true } @@ -358,15 +358,15 @@ indoc = "2" tracing-test = { version = "0.2", default-features = false } toml = { version = "0.8", default-features = false } mlua = { version = "0.9", default-features = false, features = [ - "lua54", - "send", - "vendored", + "lua54", + "send", + "vendored", ] } quickcheck = { version = "1" } regex = { version = "1", default-features = false, features = [ - "std", - "perf", - "unicode", + "std", + "perf", + "unicode", ] } paste = { version = "1", default-features = false } proptest = { version = "1" } From f97c4f292153a35053a847e44b17da66d33a6fd5 Mon Sep 17 00:00:00 2001 From: Andy Georges Date: Wed, 7 May 2025 22:33:45 +0200 Subject: [PATCH 05/10] fix: Cargo.lock --- Cargo.lock | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 734fa67d6d..9a11f2e82a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2297,7 +2297,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.90", @@ -2305,10 +2305,11 @@ dependencies = [ [[package]] name = "prost-reflect" -version = "0.15.2" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebb644dd3ad12d7cf62a18234d385ea5511ac6abb208704c18098e7e3a5e1b69" +checksum = "7b5edd582b62f5cde844716e66d92565d7faf7ab1445c8cebce6e00fba83ddb2" dependencies = [ + "once_cell", "prost", "prost-types", ] @@ -2934,7 +2935,8 @@ dependencies = [ [[package]] name = "syslog_loose" version = "0.21.0" -source = "git+https://github.com/itkovian/syslog-loose?rev=7acfd39be5bd#7acfd39be5bdac043dee9955a9e59d839b0ce950" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "161028c00842709450114c39db3b29f44c898055ed8833bb9b535aba7facf30e" dependencies = [ "chrono", "nom", From 42a78bed01ba4733838b7cf523c74aa6005c39dc Mon Sep 17 00:00:00 2001 From: Andy Georges Date: Wed, 7 May 2025 22:37:40 +0200 Subject: [PATCH 06/10] bump: version for syslog-loose to match 0.21.0 --- Cargo.lock | 3 +- Cargo.toml | 231 +++++++++++++---------------------------------------- 2 files changed, 55 insertions(+), 179 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a11f2e82a..45eb1c2f71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2935,8 +2935,7 @@ dependencies = [ [[package]] name = "syslog_loose" version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "161028c00842709450114c39db3b29f44c898055ed8833bb9b535aba7facf30e" +source = "git+https://github.com/itkovian/syslog-loose?rev=e221a9c647ee6822d39a423ef85caf30114e3923#e221a9c647ee6822d39a423ef85caf30114e3923" dependencies = [ "chrono", "nom", diff --git a/Cargo.toml b/Cargo.toml index b3486dcaf4..2df1448059 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,119 +10,41 @@ repository = "https://github.com/vectordotdev/vrl" readme = "README.md" keywords = ["vector", "datadog", "compiler"] categories = ["compilers"] -rust-version = "1.79" # msrv +rust-version = "1.81" # msrv [workspace] -members = [".", "lib/cli", "lib/tests", "lib/fuzz"] +members = [ + ".", + "lib/cli", + "lib/tests", + "lib/fuzz" +] [workspace.dependencies] clap = { version = "4", features = ["derive"] } [features] -default = [ - "compiler", - "value", - "diagnostic", - "path", - "parser", - "stdlib", - "datadog", - "core", -] +default = ["compiler", "value", "diagnostic", "path", "parser", "stdlib", "datadog", "core"] # Main features (on by default) -compiler = [ - "diagnostic", - "path", - "parser", - "value", - "dep:paste", - "dep:chrono", - "dep:serde", - "dep:regex", - "dep:bytes", - "dep:ordered-float", - "dep:chrono-tz", - "dep:snafu", - "dep:thiserror", - "dep:dyn-clone", - "dep:indoc", - "dep:thiserror", - "dep:lalrpop-util", -] -value = [ - "path", - "dep:bytes", - "dep:regex", - "dep:ordered-float", - "dep:chrono", - "dep:serde_json", -] +compiler = ["diagnostic", "path", "parser", "value", "dep:chrono", "dep:serde", "dep:regex", "dep:bytes", "dep:ordered-float", "dep:chrono-tz", "dep:snafu", "dep:thiserror", "dep:dyn-clone", "dep:indoc", "dep:thiserror", "dep:lalrpop-util"] +value = ["path", "dep:bytes", "dep:regex", "dep:ordered-float", "dep:chrono", "dep:serde_json", "dep:simdutf8"] diagnostic = ["dep:codespan-reporting", "dep:termcolor"] -path = ["value", "dep:once_cell", "dep:serde", "dep:snafu", "dep:regex"] -parser = [ - "path", - "diagnostic", - "value", - "dep:thiserror", - "dep:ordered-float", - "dep:lalrpop-util", -] -parsing = [ - "value", - "compiler", - "dep:url", - "dep:nom", - "dep:regex", - "dep:roxmltree", - "dep:rust_decimal", -] +path = ["value", "dep:serde", "dep:snafu", "dep:regex"] +parser = ["path", "diagnostic", "value", "dep:thiserror", "dep:ordered-float", "dep:lalrpop-util"] +parsing = ["value", "compiler", "dep:url", "dep:nom", "dep:regex", "dep:roxmltree", "dep:rust_decimal"] core = ["value", "dep:snafu", "dep:nom"] string_path = [] # Datadog related features (on by default) datadog = ["datadog_filter", "datadog_grok", "datadog_search"] datadog_filter = ["path", "datadog_search", "dep:regex", "dep:dyn-clone"] -datadog_grok = [ - "value", - "parsing", - "dep:nom", - "dep:peeking_take_while", - "dep:serde_json", - "dep:onig", - "dep:lalrpop-util", - "dep:thiserror", - "dep:chrono", - "dep:chrono-tz", - "dep:percent-encoding", - "dep:fancy-regex", -] -datadog_search = [ - "dep:pest", - "dep:pest_derive", - "dep:itertools", - "dep:once_cell", - "dep:regex", - "dep:serde", -] +datadog_grok = ["value", "parsing", "dep:nom", "dep:peeking_take_while", "dep:serde_json", "dep:onig", "dep:lalrpop-util", "dep:thiserror", "dep:chrono", "dep:chrono-tz", "dep:percent-encoding", "dep:fancy-regex"] +datadog_search = ["dep:pest", "dep:pest_derive", "dep:itertools", "dep:regex", "dep:serde"] # Features that aren't used as often (default off) -cli = [ - "stdlib", - "dep:serde_json", - "dep:thiserror", - "dep:clap", - "dep:exitcode", - "dep:webbrowser", - "dep:rustyline", - "dep:prettytable-rs", -] -test_framework = [ - "compiler", - "dep:prettydiff", - "dep:serde_json", - "dep:ansi_term", -] +cli = ["stdlib", "dep:serde_json", "dep:thiserror", "dep:exitcode", "dep:webbrowser", "dep:rustyline", "dep:prettytable-rs"] +test_framework = ["compiler", "dep:prettydiff", "dep:serde_json", "dep:ansi_term"] arbitrary = ["dep:quickcheck", "dep:arbitrary"] lua = ["dep:mlua"] proptest = ["dep:proptest", "dep:proptest-derive"] @@ -139,6 +61,7 @@ stdlib = [ "datadog", "parsing", "dep:aes", + "dep:aes-siv", "dep:base16", "dep:base62", "dep:base64", @@ -147,8 +70,10 @@ stdlib = [ "dep:chacha20poly1305", "dep:charset", "dep:convert_case", + "dep:ciborium", "dep:cidr", "dep:community-id", + "dep:crc", "dep:crypto_secretbox", "dep:csv", "dep:ctr", @@ -168,7 +93,7 @@ stdlib = [ "dep:md-5", "dep:nom", "dep:ofb", - "dep:once_cell", + "dep:parse-size", "dep:percent-encoding", "dep:prost", "dep:prost-reflect", @@ -193,6 +118,7 @@ stdlib = [ "dep:uuid", "dep:woothee", "dep:zstd", + "dep:encoding_rs" ] [dependencies] @@ -206,63 +132,42 @@ base62 = { version = "2.2.1", optional = true } base64 = { version = "0.22", optional = true } bytes = { version = "1", default-features = false, optional = true } charset = { version = "0.1", optional = true } -chrono = { version = "0.4", default-features = false, features = [ - "clock", - "serde", - "wasmbind", -], optional = true } +encoding_rs = { version = "0.8.35", optional = true } +chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"], optional = true } chrono-tz = { version = "0.10", default-features = false, optional = true } ciborium = { version = "0.2.2", default-features = false, optional = true } cidr = { version = "0.3", optional = true } csv = { version = "1", optional = true } -clap = { version = "4", features = ["derive"], optional = true } -codespan-reporting = { version = "0.11", optional = true } -convert_case = { version = "0.6.0", optional = true } +clap.workspace = true +codespan-reporting = { version = "0.12", optional = true } +convert_case = { version = "0.7.1", optional = true } +crc = { version = "3.2.1", optional = true } data-encoding = { version = "2", optional = true } digest = { version = "0.10", optional = true } dyn-clone = { version = "1", default-features = false, optional = true } exitcode = { version = "1", optional = true } -flate2 = { version = "1", default-features = false, features = [ - "default", -], optional = true } +flate2 = { version = "1.1.1", default-features = false, features = ["zlib-rs"], optional = true } hex = { version = "0.4", optional = true } hmac = { version = "0.12", optional = true } iana-time-zone = { version = "0.1", optional = true } -idna = { version = "0.5", optional = true } -indexmap = { version = "2", default-features = false, features = [ - "std", -], optional = true } +idna = { version = "1.0", optional = true } +indexmap = { version = "2", default-features = false, features = ["std"], optional = true } influxdb-line-protocol = { version = "2.0.0", optional = true } indoc = { version = "2", optional = true } -itertools = { version = "0.13", default-features = false, features = [ - "use_std", -], optional = true } -lalrpop-util = { version = "0.21", optional = true } -mlua = { version = "0.9", default-features = false, features = [ - "lua54", - "send", - "vendored", -], optional = true } -nom = { version = "7", default-features = false, features = [ - "std", -], optional = true } -once_cell = { version = "1", default-features = false, features = [ - "std", -], optional = true } +itertools = { version = "0.14", default-features = false, features = ["use_std"], optional = true } +lalrpop-util = { version = "0.22", optional = true } +mlua = { version = "0.10", default-features = false, features = ["lua54", "send", "vendored"], optional = true } +nom = { version = "7", default-features = false, features = ["std"], optional = true } ordered-float = { version = "4", default-features = false, optional = true } md-5 = { version = "0.10", optional = true } -parse-size = { version = "1.1.0", optional = true } +parse-size = { version = "1.1.0", optional = true } peeking_take_while = { version = "1", default-features = false, optional = true } percent-encoding = { version = "2", optional = true } -pest = { version = "2", default-features = false, optional = true, features = [ - "std", -] } -pest_derive = { version = "2", default-features = false, optional = true, features = [ - "std", -] } +pest = { version = "2", default-features = false, optional = true, features = ["std"] } +pest_derive = { version = "2", default-features = false, optional = true, features = ["std"] } proptest = { version = "1", optional = true } -proptest-derive = { version = "0.4", optional = true } -prettydiff = { version = "0.7", default-features = false, optional = true } +proptest-derive = { version = "0.5", optional = true } +prettydiff = { version = "0.8", default-features = false, optional = true } prettytable-rs = { version = "0.10", default-features = false, optional = true } quickcheck = { version = "1", optional = true } quoted_printable = { version = "0.5", optional = true } @@ -270,29 +175,23 @@ psl = { version = "2", optional = true } psl-types = { version = "2", optional = true } publicsuffix = { version = "2", optional = true } rand = { version = "0.8", optional = true } -regex = { version = "1", default-features = false, optional = true, features = [ - "std", - "perf", - "unicode", -] } +regex = { version = "1", default-features = false, optional = true, features = ["std", "perf", "unicode"] } roxmltree = { version = "0.20", optional = true } rustyline = { version = "15", default-features = false, optional = true } rust_decimal = { version = "1", optional = true, default-features = false } seahash = { version = "4", optional = true } serde = { version = "1", features = ["derive"], optional = true } -serde_json = { version = "1", default-features = false, optional = true, features = [ - "std", - "raw_value", -] } -fancy-regex = { version = "0.13.0", default-features = false, optional = true } +serde_json = { version = "1", default-features = false, optional = true, features = ["std", "raw_value"] } +simdutf8 = { version = "0.1.5", optional = true } +fancy-regex = { version = "0.14.0", default-features = false, optional = true } sha-1 = { version = "0.10", optional = true } sha-2 = { package = "sha2", version = "0.10", optional = true } sha-3 = { package = "sha3", version = "0.10", optional = true } strip-ansi-escapes = { version = "0.2", optional = true } snap = { version = "1", optional = true } -syslog_loose = { git = "https://github.com/itkovian/syslog-loose", rev = "7acfd39be5bd", optional = true } +syslog_loose = { git = "https://github.com/itkovian/syslog-loose", rev = "e221a9c647ee6822d39a423ef85caf30114e3923", optional = true } termcolor = { version = "1", optional = true } -thiserror = { version = "1", optional = true } +thiserror = { version = "2", optional = true } tracing = { version = "0.1", default-features = false } ua-parser = { version = "0.2", optional = true } utf8-width = { version = "0.1", optional = true } @@ -302,9 +201,7 @@ webbrowser = { version = "1.0", default-features = false, optional = true } woothee = { version = "0.13", optional = true } community-id = { version = "0.2", optional = true } -zstd = { version = "0.13", default-features = false, features = [ - "wasm", -], optional = true } +zstd = { version = "0.13", default-features = false, features = ["wasm"], optional = true } # Cryptography aes = { version = "0.8", optional = true } @@ -319,30 +216,19 @@ cfb-mode = { version = "0.8", optional = true } ofb = { version = "0.6", optional = true } # Protobuf support. -prost = { version = "0.13", default-features = false, optional = true, features = [ - "std", -] } +prost = { version = "0.13", default-features = false, optional = true, features = ["std"] } prost-reflect = { version = "0.14", default-features = false, optional = true } +unicode-segmentation = "1.12.0" +lz4_flex = "0.11.3" # Dependencies used for non-WASM [target.'cfg(not(target_arch = "wasm32"))'.dependencies] dns-lookup = { version = "2", optional = true } -domain = { version = "0.10.1", optional = true, features = [ - "resolv-sync", - "serde", -] } +domain = { version = "0.10.4", optional = true, features = ["resolv-sync", "serde"] } hostname = { version = "0.4", optional = true } grok = { version = "2", optional = true } onig = { version = "6", default-features = false, optional = true } -tokio = { version = "1.38", optional = true, features = [ - "io-util", - "macros", - "net", - "time", - "sync", - "rt", - "rt-multi-thread", -] } +tokio = { version = "1.44", optional = true, features = ["io-util", "macros", "net", "time", "sync", "rt", "rt-multi-thread"] } uuid = { version = "1", features = ["v4", "v7"], optional = true } # Dependencies used for WASM @@ -356,18 +242,9 @@ chrono-tz = "0.10" serde_json = "1" indoc = "2" tracing-test = { version = "0.2", default-features = false } -toml = { version = "0.8", default-features = false } -mlua = { version = "0.9", default-features = false, features = [ - "lua54", - "send", - "vendored", -] } +mlua = { version = "0.10", default-features = false, features = ["lua54", "send", "vendored"] } quickcheck = { version = "1" } -regex = { version = "1", default-features = false, features = [ - "std", - "perf", - "unicode", -] } +regex = { version = "1", default-features = false, features = ["std", "perf", "unicode"] } paste = { version = "1", default-features = false } proptest = { version = "1" } proptest-derive = { version = "0.5" } From 9e2d37bbc267be844d4d3b71730ede770035ff97 Mon Sep 17 00:00:00 2001 From: Andy Georges Date: Thu, 8 May 2025 11:23:26 +0200 Subject: [PATCH 07/10] fmt: back to original formatting --- src/stdlib/parse_syslog.rs | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/stdlib/parse_syslog.rs b/src/stdlib/parse_syslog.rs index 15061c3e95..d38fd5fc9b 100644 --- a/src/stdlib/parse_syslog.rs +++ b/src/stdlib/parse_syslog.rs @@ -10,8 +10,12 @@ pub(crate) fn parse_syslog(value: Value, variant: Variant, ctx: &Context) -> Res TimeZone::Local => None, TimeZone::Named(tz) => Some(*tz), }; - let parsed = - syslog_loose::parse_message_with_year_exact_tz(&message, resolve_year, timezone, variant)?; + let parsed = syslog_loose::parse_message_with_year_exact_tz( + &message, + resolve_year, + timezone, + variant + )?; Ok(message_to_value(parsed)) } @@ -24,18 +28,16 @@ impl Function for ParseSyslog { } fn parameters(&self) -> &'static [Parameter] { - &[ - Parameter { - keyword: "value", - kind: kind::BYTES, - required: true, - }, - Parameter { - keyword: "variant", - kind: kind::BYTES, - required: false, - }, - ] + &[Parameter { + keyword: "value", + kind: kind::BYTES, + required: true, + }, + Parameter { + keyword: "variant", + kind: kind::BYTES, + required: false, + }] } fn examples(&self) -> &'static [Example] { From 2b56cd02fe88d7759af9e5366828554100853353 Mon Sep 17 00:00:00 2001 From: Andy Georges Date: Wed, 21 May 2025 12:14:53 +0200 Subject: [PATCH 08/10] fix: syslog-loose without extraeneous printed messages --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 45eb1c2f71..34c15c6457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2935,7 +2935,7 @@ dependencies = [ [[package]] name = "syslog_loose" version = "0.21.0" -source = "git+https://github.com/itkovian/syslog-loose?rev=e221a9c647ee6822d39a423ef85caf30114e3923#e221a9c647ee6822d39a423ef85caf30114e3923" +source = "git+https://github.com/itkovian/syslog-loose?rev=72bb2608ff942112d688626143b9bf8187a2f24f#72bb2608ff942112d688626143b9bf8187a2f24f" dependencies = [ "chrono", "nom", diff --git a/Cargo.toml b/Cargo.toml index 2df1448059..55c9507a73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -189,7 +189,7 @@ sha-2 = { package = "sha2", version = "0.10", optional = true } sha-3 = { package = "sha3", version = "0.10", optional = true } strip-ansi-escapes = { version = "0.2", optional = true } snap = { version = "1", optional = true } -syslog_loose = { git = "https://github.com/itkovian/syslog-loose", rev = "e221a9c647ee6822d39a423ef85caf30114e3923", optional = true } +syslog_loose = { git = "https://github.com/itkovian/syslog-loose", rev = "72bb2608ff942112d688626143b9bf8187a2f24f", optional = true } termcolor = { version = "1", optional = true } thiserror = { version = "2", optional = true } tracing = { version = "0.1", default-features = false } From b558b4eec6070a0c97e7ed3b0858d8d62eb1bbff Mon Sep 17 00:00:00 2001 From: Andy Georges Date: Fri, 13 Jun 2025 17:11:25 +0200 Subject: [PATCH 09/10] changelog: add entry for RFC variant PR --- changelog.d/1051.enhancement.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog.d/1051.enhancement.md diff --git a/changelog.d/1051.enhancement.md b/changelog.d/1051.enhancement.md new file mode 100644 index 0000000000..ab6e12a1b8 --- /dev/null +++ b/changelog.d/1051.enhancement.md @@ -0,0 +1,7 @@ +Update the syslog parser with a variant choice, forcing to more strictly follow the RFC when +parsing. + +Choices are `either`, `rfc3164`, or `rfc5424`. `either` is the behaviour +exhibited prior to this PR. + +authors: itkovian (Andy Georges) From 5037a02de99552e750b8ad873cbcb1ba13a4f8c5 Mon Sep 17 00:00:00 2001 From: Andy Georges Date: Fri, 13 Jun 2025 17:33:44 +0200 Subject: [PATCH 10/10] fix: clippy --- src/stdlib/parse_linux_authorization.rs | 2 +- src/stdlib/parse_syslog.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/stdlib/parse_linux_authorization.rs b/src/stdlib/parse_linux_authorization.rs index 8f5bf5a4b1..1580fafbe8 100644 --- a/src/stdlib/parse_linux_authorization.rs +++ b/src/stdlib/parse_linux_authorization.rs @@ -42,7 +42,7 @@ impl Function for ParseLinuxAuthorization { // The parse_linux_authorization function is just an alias for parse_syslog Ok(ParseSyslogFn { - value: value, + value, variant: "either".into(), } .as_expr()) diff --git a/src/stdlib/parse_syslog.rs b/src/stdlib/parse_syslog.rs index d38fd5fc9b..0434f15b2c 100644 --- a/src/stdlib/parse_syslog.rs +++ b/src/stdlib/parse_syslog.rs @@ -4,7 +4,7 @@ use nom::AsBytes; use std::collections::BTreeMap; use syslog_loose::{IncompleteDate, Message, ProcId, Protocol, Variant}; -pub(crate) fn parse_syslog(value: Value, variant: Variant, ctx: &Context) -> Resolved { +pub(crate) fn parse_syslog(value: &Value, variant: Variant, ctx: &Context) -> Resolved { let message = value.try_bytes_utf8_lossy()?; let timezone = match ctx.timezone() { TimeZone::Local => None, @@ -98,7 +98,7 @@ impl FunctionExpression for ParseSyslogFn { _ => panic!("No such variant for syslog RFC"), }; - parse_syslog(value, variant, ctx) + parse_syslog(&value, variant, ctx) } fn type_def(&self, _: &state::TypeState) -> TypeDef {