From f414ef3b47309f4b7ae301be3a1f405fa4e62fa1 Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Wed, 26 Nov 2025 10:12:42 -0500 Subject: [PATCH 01/21] line log generator --- lading/src/generator/file_gen/traditional.rs | 37 ++++ lading_payload/src/block.rs | 43 +++++ lading_payload/src/lib.rs | 34 ++++ lading_payload/src/statik_line_rate.rs | 178 ++++++++++++++++++ lading_payload/src/statik_second.rs | 182 +++++++++++++++++++ 5 files changed, 474 insertions(+) create mode 100644 lading_payload/src/statik_line_rate.rs create mode 100644 lading_payload/src/statik_second.rs diff --git a/lading/src/generator/file_gen/traditional.rs b/lading/src/generator/file_gen/traditional.rs index 72ea23fd3..ba3c6ab88 100644 --- a/lading/src/generator/file_gen/traditional.rs +++ b/lading/src/generator/file_gen/traditional.rs @@ -30,6 +30,7 @@ use tokio::{ fs, io::{AsyncWriteExt, BufWriter}, task::{JoinError, JoinSet}, + time::{Duration, Instant}, }; use tracing::{error, info}; @@ -120,6 +121,13 @@ pub struct Config { rotate: bool, /// The load throttle configuration pub throttle: Option, + /// Optional fixed interval between blocks. When set, the generator waits + /// this duration before emitting the next block, regardless of byte size. + pub block_interval_millis: Option, + /// Flush after each block. Useful when block intervals are large and the + /// buffered writer would otherwise delay writes to disk. + #[serde(default)] + pub flush_each_block: bool, } #[derive(Debug)] @@ -195,6 +203,10 @@ impl Server { file_index: Arc::clone(&file_index), rotate: config.rotate, shutdown: shutdown.clone(), + block_interval: config + .block_interval_millis + .map(Duration::from_millis), + flush_each_block: config.flush_each_block, }; handles.spawn(child.spin()); @@ -269,6 +281,8 @@ struct Child { rotate: bool, file_index: Arc, shutdown: lading_signal::Watcher, + block_interval: Option, + flush_each_block: bool, } impl Child { @@ -303,6 +317,10 @@ impl Child { let shutdown_wait = self.shutdown.recv(); tokio::pin!(shutdown_wait); + let mut next_tick = self + .block_interval + .as_ref() + .map(|dur| Instant::now() + *dur); loop { let total_bytes = self.block_cache.peek_next_size(&handle); @@ -310,6 +328,22 @@ impl Child { result = self.throttle.wait_for(total_bytes) => { match result { Ok(()) => { + if let Some(dur) = self.block_interval { + if let Some(deadline) = next_tick { + tokio::select! { + _ = tokio::time::sleep_until(deadline) => {}, + () = &mut shutdown_wait => { + fp.flush().await?; + info!("shutdown signal received"); + return Ok(()); + }, + } + next_tick = Some(deadline + dur); + } else { + next_tick = Some(Instant::now() + dur); + } + } + let block = self.block_cache.advance(&mut handle); let total_bytes = u64::from(total_bytes.get()); @@ -318,6 +352,9 @@ impl Child { counter!("bytes_written").increment(total_bytes); total_bytes_written += total_bytes; } + if self.flush_each_block { + fp.flush().await?; + } if total_bytes_written > maximum_bytes_per_file { fp.flush().await?; diff --git a/lading_payload/src/block.rs b/lading_payload/src/block.rs index ab2c4f99d..84d52e75c 100644 --- a/lading_payload/src/block.rs +++ b/lading_payload/src/block.rs @@ -26,6 +26,12 @@ pub enum SpinError { /// `StaticChunks` payload creation error #[error(transparent)] StaticChunks(#[from] crate::static_chunks::Error), + /// Static line-rate payload creation error + #[error(transparent)] + StaticLinesPerSecond(#[from] crate::statik_line_rate::Error), + /// Static second-grouped payload creation error + #[error(transparent)] + StaticSecond(#[from] crate::statik_second::Error), /// rng slice is Empty #[error("RNG slice is empty")] EmptyRng, @@ -61,6 +67,12 @@ pub enum Error { /// `StaticChunks` payload creation error #[error(transparent)] StaticChunks(#[from] crate::static_chunks::Error), + /// Static line-rate payload creation error + #[error(transparent)] + StaticLinesPerSecond(#[from] crate::statik_line_rate::Error), + /// Static second-grouped payload creation error + #[error(transparent)] + StaticSecond(#[from] crate::statik_second::Error), /// Error for crate deserialization #[error("Deserialization error: {0}")] Deserialize(#[from] crate::Error), @@ -354,6 +366,37 @@ impl Cache { total_bytes.get(), )? } + crate::Config::StaticLinesPerSecond { + static_path, + lines_per_second, + } => { + let span = span!(Level::INFO, "fixed", payload = "static-lines-per-second"); + let _guard = span.enter(); + let mut serializer = + crate::StaticLinesPerSecond::new(static_path, *lines_per_second)?; + construct_block_cache_inner( + &mut rng, + &mut serializer, + maximum_block_bytes, + total_bytes.get(), + )? + } + crate::Config::StaticSecond { + static_path, + timestamp_format, + emit_placeholder, + } => { + let span = span!(Level::INFO, "fixed", payload = "static-second"); + let _guard = span.enter(); + let mut serializer = + crate::StaticSecond::new(static_path, ×tamp_format, *emit_placeholder)?; + construct_block_cache_inner( + &mut rng, + &mut serializer, + maximum_block_bytes, + total_bytes.get(), + )? + } crate::Config::OpentelemetryTraces(config) => { let mut pyld = crate::OpentelemetryTraces::with_config(*config, &mut rng)?; let span = span!(Level::INFO, "fixed", payload = "otel-traces"); diff --git a/lading_payload/src/lib.rs b/lading_payload/src/lib.rs index 034f182b6..a962ad8a7 100644 --- a/lading_payload/src/lib.rs +++ b/lading_payload/src/lib.rs @@ -28,6 +28,8 @@ pub use opentelemetry::trace::OpentelemetryTraces; pub use splunk_hec::SplunkHec; pub use static_chunks::StaticChunks; pub use statik::Static; +pub use statik_line_rate::StaticLinesPerSecond; +pub use statik_second::StaticSecond; pub use syslog::Syslog5424; pub mod apache_common; @@ -42,6 +44,8 @@ pub mod procfs; pub mod splunk_hec; pub mod static_chunks; pub mod statik; +pub mod statik_line_rate; +pub mod statik_second; pub mod syslog; pub mod trace_agent; @@ -139,6 +143,28 @@ pub enum Config { /// all files under it (non-recursively) will be read line by line. static_path: PathBuf, }, + /// Generates static data but limits the number of lines emitted per block + StaticLinesPerSecond { + /// Defines the file path to read static variant data from. Content is + /// assumed to be line-oriented but no other claim is made on the file. + static_path: PathBuf, + /// Number of lines to emit in each generated block + lines_per_second: u32, + }, + /// Generates static data grouped by second; each block contains one + /// second's worth of logs as determined by a parsed timestamp prefix. + StaticSecond { + /// Defines the file path to read static variant data from. Content is + /// assumed to be line-oriented. + static_path: PathBuf, + /// Chrono-compatible timestamp format string used to parse the leading + /// timestamp in each line. + timestamp_format: String, + /// Emit a minimal placeholder block (single newline) for seconds with + /// no lines. When false, empty seconds are skipped. + #[serde(default)] + emit_placeholder: bool, + }, /// Generates a line of printable ascii characters Ascii, /// Generates a json encoded line @@ -179,6 +205,10 @@ pub enum Payload { Static(Static), /// Static file content, chunked into lines that fill blocks as closely as possible. StaticChunks(StaticChunks), + /// Static file content with a fixed number of lines emitted per block + StaticLinesPerSecond(StaticLinesPerSecond), + /// Static file content grouped into one-second blocks based on timestamps + StaticSecond(StaticSecond), /// Syslog RFC 5424 format Syslog(Syslog5424), /// OpenTelemetry traces @@ -208,6 +238,8 @@ impl Serialize for Payload { Payload::SplunkHec(ser) => ser.to_bytes(rng, max_bytes, writer), Payload::Static(ser) => ser.to_bytes(rng, max_bytes, writer), Payload::StaticChunks(ser) => ser.to_bytes(rng, max_bytes, writer), + Payload::StaticLinesPerSecond(ser) => ser.to_bytes(rng, max_bytes, writer), + Payload::StaticSecond(ser) => ser.to_bytes(rng, max_bytes, writer), Payload::Syslog(ser) => ser.to_bytes(rng, max_bytes, writer), Payload::OtelTraces(ser) => ser.to_bytes(rng, max_bytes, writer), Payload::OtelLogs(ser) => ser.to_bytes(rng, max_bytes, writer), @@ -220,6 +252,8 @@ impl Serialize for Payload { fn data_points_generated(&self) -> Option { match self { Payload::OtelMetrics(ser) => ser.data_points_generated(), + Payload::StaticLinesPerSecond(ser) => ser.data_points_generated(), + Payload::StaticSecond(ser) => ser.data_points_generated(), // Other implementations use the default None _ => None, } diff --git a/lading_payload/src/statik_line_rate.rs b/lading_payload/src/statik_line_rate.rs new file mode 100644 index 000000000..25ff0ee6f --- /dev/null +++ b/lading_payload/src/statik_line_rate.rs @@ -0,0 +1,178 @@ +//! Static file payload that replays a limited number of lines per block. + +use std::{ + fs::{self, OpenOptions}, + io::{BufRead, BufReader, Write}, + num::NonZeroU32, + path::Path, +}; + +use rand::{Rng, seq::IndexedMutRandom}; +use tracing::debug; + +#[derive(Debug)] +struct Source { + lines: Vec>, + next_idx: usize, +} + +#[derive(Debug)] +/// Static payload that emits a fixed number of lines each time it is asked to +/// serialize. +pub struct StaticLinesPerSecond { + sources: Vec, + lines_per_block: NonZeroU32, + last_lines_generated: u64, +} + +#[derive(thiserror::Error, Debug)] +/// Errors produced by [`StaticLinesPerSecond`]. +pub enum Error { + /// IO error + #[error(transparent)] + Io(#[from] std::io::Error), + /// No lines were discovered in the provided path + #[error("No lines found in static path")] + NoLines, + /// The provided lines_per_second value was zero + #[error("lines_per_second must be greater than zero")] + ZeroLinesPerSecond, +} + +impl StaticLinesPerSecond { + /// Create a new instance of `StaticLinesPerSecond` + /// + /// # Errors + /// + /// See documentation for [`Error`] + pub fn new(path: &Path, lines_per_second: u32) -> Result { + let lines_per_block = + NonZeroU32::new(lines_per_second).ok_or(Error::ZeroLinesPerSecond)?; + + let mut sources = Vec::with_capacity(16); + + let metadata = fs::metadata(path)?; + if metadata.is_file() { + debug!("Static path {} is a file.", path.display()); + let lines = read_lines(path)?; + sources.push(Source { next_idx: 0, lines }); + } else if metadata.is_dir() { + debug!("Static path {} is a directory.", path.display()); + for entry in fs::read_dir(path)? { + let entry = entry?; + let entry_pth = entry.path(); + debug!("Attempting to open {} as file.", entry_pth.display()); + if let Ok(file) = OpenOptions::new().read(true).open(&entry_pth) { + let lines = read_lines_from_reader(file)?; + sources.push(Source { next_idx: 0, lines }); + } + } + } + + if sources.iter().all(|s| s.lines.is_empty()) { + return Err(Error::NoLines); + } + + Ok(Self { + sources, + lines_per_block, + last_lines_generated: 0, + }) + } +} + +impl crate::Serialize for StaticLinesPerSecond { + fn to_bytes( + &mut self, + mut rng: R, + max_bytes: usize, + writer: &mut W, + ) -> Result<(), crate::Error> + where + R: Rng + Sized, + W: Write, + { + self.last_lines_generated = 0; + + let Some(source) = self.sources.choose_mut(&mut rng) else { + return Ok(()); + }; + if source.lines.is_empty() { + return Ok(()); + } + + let mut bytes_written = 0usize; + for _ in 0..self.lines_per_block.get() { + let line = &source.lines[source.next_idx % source.lines.len()]; + let needed = line.len() + 1; // newline + if bytes_written + needed > max_bytes { + break; + } + + writer.write_all(line)?; + writer.write_all(b"\n")?; + bytes_written += needed; + self.last_lines_generated += 1; + source.next_idx = (source.next_idx + 1) % source.lines.len(); + } + + Ok(()) + } + + fn data_points_generated(&self) -> Option { + Some(self.last_lines_generated) + } +} + +fn read_lines(path: &Path) -> Result>, std::io::Error> { + let file = OpenOptions::new().read(true).open(path)?; + read_lines_from_reader(file) +} + +fn read_lines_from_reader(reader: R) -> Result>, std::io::Error> { + let mut out = Vec::new(); + let mut reader = BufReader::new(reader); + let mut buf = String::new(); + while { + buf.clear(); + reader.read_line(&mut buf)? + } != 0 + { + if buf.ends_with('\n') { + buf.pop(); + if buf.ends_with('\r') { + buf.pop(); + } + } + out.push(buf.as_bytes().to_vec()); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::{SeedableRng, rngs::StdRng}; + use std::{env, fs::File, io::Write as IoWrite}; + + #[test] + fn writes_requested_number_of_lines() { + let mut path = env::temp_dir(); + path.push("static_line_rate_test.txt"); + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "alpha").unwrap(); + writeln!(f, "beta").unwrap(); + writeln!(f, "gamma").unwrap(); + } + + let mut serializer = StaticLinesPerSecond::new(&path, 2).unwrap(); + let mut buf = Vec::new(); + let mut rng = StdRng::seed_from_u64(42); + + serializer.to_bytes(&mut rng, 1024, &mut buf).unwrap(); + assert_eq!(buf, b"alpha\nbeta\n"); + // Clean up + let _ = std::fs::remove_file(&path); + } +} diff --git a/lading_payload/src/statik_second.rs b/lading_payload/src/statik_second.rs new file mode 100644 index 000000000..e21c68033 --- /dev/null +++ b/lading_payload/src/statik_second.rs @@ -0,0 +1,182 @@ +//! Static file payload that emits one second of log lines per block, based on +//! parsing a timestamp at the start of each line. + +use std::{ + fs::File, + io::{BufRead, BufReader, Write}, + path::Path, +}; + +use chrono::{NaiveDateTime, TimeZone, Utc}; +use rand::Rng; +use tracing::debug; + +#[derive(Debug)] +struct BlockLines { + lines: Vec>, +} + +#[derive(thiserror::Error, Debug)] +/// Errors produced by [`StaticSecond`]. +pub enum Error { + /// IO error + #[error(transparent)] + Io(#[from] std::io::Error), + /// No lines were discovered in the provided path + #[error("No lines found in static path")] + NoLines, + /// Timestamp parsing failed for a line + #[error("Failed to parse timestamp from line: {0}")] + Timestamp(String), +} + +#[derive(Debug)] +/// Static payload grouped by second boundaries. +pub struct StaticSecond { + blocks: Vec, + idx: usize, + last_lines_generated: u64, + emit_placeholder: bool, +} + +impl StaticSecond { + /// Create a new instance of `StaticSecond` + /// + /// Lines are grouped into blocks by the second of their timestamp. The + /// timestamp is parsed from the start of the line up to the first + /// whitespace, using `timestamp_format` (chrono strftime syntax). + pub fn new(path: &Path, timestamp_format: &str, emit_placeholder: bool) -> Result { + let file = File::open(path)?; + let reader = BufReader::new(file); + + let mut blocks: Vec = Vec::new(); + let mut current_sec: Option = None; + let mut current_lines: Vec> = Vec::new(); + + for line_res in reader.lines() { + let line = line_res?; + if line.trim().is_empty() { + continue; + } + + // Take prefix until first whitespace as the timestamp segment. + let ts_token = line.split_whitespace().next().unwrap_or(""); + let ts = NaiveDateTime::parse_from_str(ts_token, timestamp_format) + .map_err(|_| Error::Timestamp(line.clone()))?; + let sec = Utc.from_utc_datetime(&ts).timestamp(); + + match current_sec { + Some(s) if s == sec => { + current_lines.push(line.as_bytes().to_vec()); + } + Some(s) if s < sec => { + // Close out the previous second. + blocks.push(BlockLines { + lines: current_lines, + }); + // Fill missing seconds with empty buckets when placeholders + // are requested. + if emit_placeholder { + let mut missing = s + 1; + while missing < sec { + blocks.push(BlockLines { lines: Vec::new() }); + missing += 1; + } + } + current_lines = vec![line.as_bytes().to_vec()]; + current_sec = Some(sec); + } + Some(s) => { + // Unexpected time travel backwards; treat as new bucket to + // preserve ordering. + blocks.push(BlockLines { + lines: current_lines, + }); + current_lines = vec![line.as_bytes().to_vec()]; + current_sec = Some(sec); + debug!("Encountered out-of-order timestamp: current {s}, new {sec}"); + } + None => { + current_sec = Some(sec); + current_lines.push(line.as_bytes().to_vec()); + } + } + } + + if !current_lines.is_empty() { + blocks.push(BlockLines { + lines: current_lines, + }); + } else if emit_placeholder && current_sec.is_some() { + // If the file ended right after emitting placeholders, ensure the + // last bucket is represented. + blocks.push(BlockLines { lines: Vec::new() }); + } + + if blocks.is_empty() { + return Err(Error::NoLines); + } + + debug!( + "StaticSecond loaded {} second-buckets from {}", + blocks.len(), + path.display() + ); + + Ok(Self { + blocks, + idx: 0, + last_lines_generated: 0, + emit_placeholder, + }) + } +} + +impl crate::Serialize for StaticSecond { + fn to_bytes( + &mut self, + _rng: R, + max_bytes: usize, + writer: &mut W, + ) -> Result<(), crate::Error> + where + R: Rng + Sized, + W: Write, + { + self.last_lines_generated = 0; + if self.blocks.is_empty() { + return Ok(()); + } + + // Choose blocks strictly sequentially to preserve chronological replay (no rng based on seed) + let block = &self.blocks[self.idx]; + + let mut bytes_written = 0usize; + if block.lines.is_empty() { + // When requested, emit a minimal placeholder (one newline) for + // empty seconds to preserve timing gaps without breaking the + // non-zero block invariant. + if self.emit_placeholder && max_bytes > 0 { + writer.write_all(b"\n")?; + } + } else { + for line in &block.lines { + let needed = line.len() + 1; // newline + if bytes_written + needed > max_bytes { + break; + } + writer.write_all(line)?; + writer.write_all(b"\n")?; + bytes_written += needed; + self.last_lines_generated += 1; + } + } + + self.idx = (self.idx + 1) % self.blocks.len(); + Ok(()) + } + + fn data_points_generated(&self) -> Option { + Some(self.last_lines_generated) + } +} From 79a2098a7fbdd5041bce772b3067345d3ceac22b Mon Sep 17 00:00:00 2001 From: Joy Zhang Date: Mon, 1 Dec 2025 17:32:35 +0100 Subject: [PATCH 02/21] Added start-up wrap so we can tcpdump --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 54893cefb..9c26b7f8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,9 +26,9 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* # Download pre-built sccache binary RUN case "$(uname -m)" in \ - x86_64) ARCH=x86_64-unknown-linux-musl ;; \ - aarch64) ARCH=aarch64-unknown-linux-musl ;; \ - *) echo "Unsupported architecture" && exit 1 ;; \ + x86_64) ARCH=x86_64-unknown-linux-musl ;; \ + aarch64) ARCH=aarch64-unknown-linux-musl ;; \ + *) echo "Unsupported architecture" && exit 1 ;; \ esac && \ curl -L https://github.com/mozilla/sccache/releases/download/v0.8.2/sccache-v0.8.2-${ARCH}.tar.gz | tar xz && \ mv sccache-v0.8.2-${ARCH}/sccache /usr/local/cargo/bin/ && \ From 542f2225d7391c4f67616f3973e28a57605ac2a8 Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Fri, 12 Dec 2025 13:08:10 -0500 Subject: [PATCH 03/21] omit timestamp --- Cargo.lock | 1 + Dockerfile | 8 ++ lading/src/generator/file_gen/traditional.rs | 25 +++++- lading_payload/Cargo.toml | 5 ++ lading_payload/src/block.rs | 17 +++- lading_payload/src/lib.rs | 3 + lading_payload/src/statik_line_rate.rs | 1 + lading_payload/src/statik_second.rs | 88 +++++++++++++++++--- 8 files changed, 135 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d65c6066b..f57e4737a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1966,6 +1966,7 @@ dependencies = [ "serde", "serde_json", "serde_tuple", + "tempfile", "thiserror 2.0.17", "time", "tokio", diff --git a/Dockerfile b/Dockerfile index 9c26b7f8f..8091d9695 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,11 @@ +FROM docker.io/rust:1.90.0-bookworm AS chef +RUN cargo install cargo-chef +WORKDIR /app + +FROM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + # Update the rust version in-sync with the version in rust-toolchain.toml # Stage 0: Planner - Extract dependency metadata diff --git a/lading/src/generator/file_gen/traditional.rs b/lading/src/generator/file_gen/traditional.rs index ba3c6ab88..76cee1cf7 100644 --- a/lading/src/generator/file_gen/traditional.rs +++ b/lading/src/generator/file_gen/traditional.rs @@ -128,6 +128,11 @@ pub struct Config { /// buffered writer would otherwise delay writes to disk. #[serde(default)] pub flush_each_block: bool, + /// Optional starting line offset into the block cache. This advances + /// through blocks until the cumulative line count reaches this value, + /// then begins emitting from that block. If data point counts are not + /// available for a payload, this setting is effectively ignored. + pub start_line_index: Option, } #[derive(Debug)] @@ -207,6 +212,7 @@ impl Server { .block_interval_millis .map(Duration::from_millis), flush_each_block: config.flush_each_block, + start_line_index: config.start_line_index.unwrap_or(0), }; handles.spawn(child.spin()); @@ -283,6 +289,7 @@ struct Child { shutdown: lading_signal::Watcher, block_interval: Option, flush_each_block: bool, + start_line_index: u64, } impl Child { @@ -314,7 +321,23 @@ impl Child { ); let mut handle = self.block_cache.handle(); - + if self.start_line_index > 0 { + let mut remaining = self.start_line_index; + // Walk blocks until we reach or surpass the requested line offset. + // If metadata is missing, assume at least one data point to ensure progress. + loop { + let md = self.block_cache.peek_next_metadata(&handle); + let mut lines = md.data_points.unwrap_or(1); + if lines == 0 { + lines = 1; + } + if lines >= remaining { + break; + } + remaining = remaining.saturating_sub(lines); + let _ = self.block_cache.advance(&mut handle); + } + } let shutdown_wait = self.shutdown.recv(); tokio::pin!(shutdown_wait); let mut next_tick = self diff --git a/lading_payload/Cargo.toml b/lading_payload/Cargo.toml index a761f2369..601c5e8da 100644 --- a/lading_payload/Cargo.toml +++ b/lading_payload/Cargo.toml @@ -31,7 +31,12 @@ arbitrary = { version = "1", optional = true, features = ["derive"] } [dev-dependencies] proptest = { workspace = true } proptest-derive = { workspace = true } +<<<<<<< HEAD criterion = { version = "0.8", features = ["html_reports"] } +======= +criterion = { version = "0.7", features = ["html_reports"] } +tempfile = { workspace = true } +>>>>>>> ac30b30 (omit timestamp) [features] default = [] diff --git a/lading_payload/src/block.rs b/lading_payload/src/block.rs index 84d52e75c..ef1e2e398 100644 --- a/lading_payload/src/block.rs +++ b/lading_payload/src/block.rs @@ -385,11 +385,16 @@ impl Cache { static_path, timestamp_format, emit_placeholder, + start_line_index, } => { let span = span!(Level::INFO, "fixed", payload = "static-second"); let _guard = span.enter(); - let mut serializer = - crate::StaticSecond::new(static_path, ×tamp_format, *emit_placeholder)?; + let mut serializer = crate::StaticSecond::new( + static_path, + ×tamp_format, + *emit_placeholder, + start_line_index.unwrap_or(0), + )?; construct_block_cache_inner( &mut rng, &mut serializer, @@ -469,6 +474,14 @@ impl Cache { } } + /// Get the number of blocks in the cache. + #[must_use] + pub fn len(&self) -> usize { + match self { + Self::Fixed { blocks, .. } => blocks.len(), + } + } + /// Get metadata of the next block without advancing. #[must_use] pub fn peek_next_metadata(&self, handle: &Handle) -> BlockMetadata { diff --git a/lading_payload/src/lib.rs b/lading_payload/src/lib.rs index a962ad8a7..0df39a4a6 100644 --- a/lading_payload/src/lib.rs +++ b/lading_payload/src/lib.rs @@ -164,6 +164,9 @@ pub enum Config { /// no lines. When false, empty seconds are skipped. #[serde(default)] emit_placeholder: bool, + /// Optional starting line offset; lines before this index are skipped. + #[serde(default)] + start_line_index: Option, }, /// Generates a line of printable ascii characters Ascii, diff --git a/lading_payload/src/statik_line_rate.rs b/lading_payload/src/statik_line_rate.rs index 25ff0ee6f..e37139bc1 100644 --- a/lading_payload/src/statik_line_rate.rs +++ b/lading_payload/src/statik_line_rate.rs @@ -152,6 +152,7 @@ fn read_lines_from_reader(reader: R) -> Result>, s #[cfg(test)] mod tests { use super::*; + use crate::Serialize; use rand::{SeedableRng, rngs::StdRng}; use std::{env, fs::File, io::Write as IoWrite}; diff --git a/lading_payload/src/statik_second.rs b/lading_payload/src/statik_second.rs index e21c68033..f34e4a489 100644 --- a/lading_payload/src/statik_second.rs +++ b/lading_payload/src/statik_second.rs @@ -1,5 +1,6 @@ //! Static file payload that emits one second of log lines per block, based on -//! parsing a timestamp at the start of each line. +//! parsing a timestamp at the start of each line. The parsed timestamp is +//! stripped from emitted lines; only the message body is replayed. use std::{ fs::File, @@ -44,8 +45,15 @@ impl StaticSecond { /// /// Lines are grouped into blocks by the second of their timestamp. The /// timestamp is parsed from the start of the line up to the first - /// whitespace, using `timestamp_format` (chrono strftime syntax). - pub fn new(path: &Path, timestamp_format: &str, emit_placeholder: bool) -> Result { + /// whitespace, using `timestamp_format` (chrono strftime syntax). The + /// parsed timestamp is removed from the emitted line, leaving only the + /// remainder of the message. + pub fn new( + path: &Path, + timestamp_format: &str, + emit_placeholder: bool, + start_line_index: u64, + ) -> Result { let file = File::open(path)?; let reader = BufReader::new(file); @@ -59,15 +67,18 @@ impl StaticSecond { continue; } - // Take prefix until first whitespace as the timestamp segment. - let ts_token = line.split_whitespace().next().unwrap_or(""); + // Take prefix until first whitespace as the timestamp segment and + // drop it from the payload we store. + let mut parts = line.splitn(2, char::is_whitespace); + let ts_token = parts.next().unwrap_or(""); + let payload = parts.next().unwrap_or("").trim_start().as_bytes().to_vec(); let ts = NaiveDateTime::parse_from_str(ts_token, timestamp_format) .map_err(|_| Error::Timestamp(line.clone()))?; let sec = Utc.from_utc_datetime(&ts).timestamp(); match current_sec { Some(s) if s == sec => { - current_lines.push(line.as_bytes().to_vec()); + current_lines.push(payload); } Some(s) if s < sec => { // Close out the previous second. @@ -83,7 +94,7 @@ impl StaticSecond { missing += 1; } } - current_lines = vec![line.as_bytes().to_vec()]; + current_lines = vec![payload]; current_sec = Some(sec); } Some(s) => { @@ -92,13 +103,13 @@ impl StaticSecond { blocks.push(BlockLines { lines: current_lines, }); - current_lines = vec![line.as_bytes().to_vec()]; + current_lines = vec![payload]; current_sec = Some(sec); debug!("Encountered out-of-order timestamp: current {s}, new {sec}"); } None => { current_sec = Some(sec); - current_lines.push(line.as_bytes().to_vec()); + current_lines.push(payload); } } } @@ -117,6 +128,30 @@ impl StaticSecond { return Err(Error::NoLines); } + // Apply starting line offset by trimming leading lines across buckets. + let total_lines: u64 = blocks.iter().map(|b| b.lines.len() as u64).sum(); + let mut start_idx = 0usize; + if total_lines > 0 && start_line_index > 0 { + let mut remaining = start_line_index % total_lines; + if remaining > 0 { + for (idx, block) in blocks.iter_mut().enumerate() { + let len = block.lines.len() as u64; + if len == 0 { + continue; + } + if remaining >= len { + remaining -= len; + continue; + } else { + let cut = remaining as usize; + block.lines.drain(0..cut); + start_idx = idx; + break; + } + } + } + } + debug!( "StaticSecond loaded {} second-buckets from {}", blocks.len(), @@ -125,7 +160,7 @@ impl StaticSecond { Ok(Self { blocks, - idx: 0, + idx: start_idx, last_lines_generated: 0, emit_placeholder, }) @@ -180,3 +215,36 @@ impl crate::Serialize for StaticSecond { Some(self.last_lines_generated) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::Serialize; + use rand::{SeedableRng, rngs::StdRng}; + use std::{fs::File, io::Write as IoWrite}; + use tempfile::tempdir; + + #[test] + fn removes_timestamp_from_output() { + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("static_second_test.log"); + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "2024-01-01T00:00:00 first").unwrap(); + writeln!(f, "2024-01-01T00:00:00 second").unwrap(); + writeln!(f, "2024-01-01T00:00:01 third").unwrap(); + } + + let mut serializer = + StaticSecond::new(&path, "%Y-%m-%dT%H:%M:%S", /* emit_placeholder */ false).unwrap(); + let mut rng = StdRng::seed_from_u64(7); + let mut buf = Vec::new(); + + serializer.to_bytes(&mut rng, 1024, &mut buf).unwrap(); + assert_eq!(buf, b"first\nsecond\n"); + + buf.clear(); + serializer.to_bytes(&mut rng, 1024, &mut buf).unwrap(); + assert_eq!(buf, b"third\n"); + } +} From 07682a9273fbdd849fb2ad931902735218596ea1 Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Wed, 17 Dec 2025 14:13:54 -0500 Subject: [PATCH 04/21] cargot fmt --- lading/src/generator/file_gen/traditional.rs | 4 +--- lading_payload/src/statik_line_rate.rs | 3 +-- lading_payload/src/statik_second.rs | 8 ++++++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lading/src/generator/file_gen/traditional.rs b/lading/src/generator/file_gen/traditional.rs index 76cee1cf7..1d6157484 100644 --- a/lading/src/generator/file_gen/traditional.rs +++ b/lading/src/generator/file_gen/traditional.rs @@ -208,9 +208,7 @@ impl Server { file_index: Arc::clone(&file_index), rotate: config.rotate, shutdown: shutdown.clone(), - block_interval: config - .block_interval_millis - .map(Duration::from_millis), + block_interval: config.block_interval_millis.map(Duration::from_millis), flush_each_block: config.flush_each_block, start_line_index: config.start_line_index.unwrap_or(0), }; diff --git a/lading_payload/src/statik_line_rate.rs b/lading_payload/src/statik_line_rate.rs index e37139bc1..92ad4ed4e 100644 --- a/lading_payload/src/statik_line_rate.rs +++ b/lading_payload/src/statik_line_rate.rs @@ -46,8 +46,7 @@ impl StaticLinesPerSecond { /// /// See documentation for [`Error`] pub fn new(path: &Path, lines_per_second: u32) -> Result { - let lines_per_block = - NonZeroU32::new(lines_per_second).ok_or(Error::ZeroLinesPerSecond)?; + let lines_per_block = NonZeroU32::new(lines_per_second).ok_or(Error::ZeroLinesPerSecond)?; let mut sources = Vec::with_capacity(16); diff --git a/lading_payload/src/statik_second.rs b/lading_payload/src/statik_second.rs index f34e4a489..c23ffb0f4 100644 --- a/lading_payload/src/statik_second.rs +++ b/lading_payload/src/statik_second.rs @@ -235,8 +235,12 @@ mod tests { writeln!(f, "2024-01-01T00:00:01 third").unwrap(); } - let mut serializer = - StaticSecond::new(&path, "%Y-%m-%dT%H:%M:%S", /* emit_placeholder */ false).unwrap(); + let mut serializer = StaticSecond::new( + &path, + "%Y-%m-%dT%H:%M:%S", + /* emit_placeholder */ false, + ) + .unwrap(); let mut rng = StdRng::seed_from_u64(7); let mut buf = Vec::new(); From 14f24a27010bbed338dd7ac25a0a0ee8bbef8001 Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Fri, 19 Dec 2025 12:45:40 -0500 Subject: [PATCH 05/21] changes --- lading/src/generator/common.rs | 63 ++++++++++++++++++ lading/src/generator/file_gen/logrotate.rs | 34 +++++++--- lading/src/generator/file_gen/logrotate_fs.rs | 26 +++++++- lading/src/generator/file_gen/traditional.rs | 66 +++++++++++-------- lading_payload/src/block.rs | 2 +- lading_payload/src/statik_second.rs | 62 ++++++++++++++++- 6 files changed, 214 insertions(+), 39 deletions(-) diff --git a/lading/src/generator/common.rs b/lading/src/generator/common.rs index eeee3d86c..0c5d0c0e2 100644 --- a/lading/src/generator/common.rs +++ b/lading/src/generator/common.rs @@ -45,6 +45,37 @@ pub enum ThrottleConversionError { ConflictingConfig, } +/// Indicates how a throttle should interpret its token units. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ThrottleMode { + /// Throttle tokens represent bytes. + Bytes, + /// Throttle tokens represent block counts. + Blocks, +} + +impl ThrottleMode { + /// Return the number of tokens required for a block of the given byte size. + /// + /// Bytes mode consumes the actual block size. Blocks mode consumes a single + /// token per block regardless of its byte length. + pub(super) fn tokens_for_block(&self, block_size: NonZeroU32) -> NonZeroU32 { + match self { + ThrottleMode::Bytes => block_size, + ThrottleMode::Blocks => NonZeroU32::new(1).expect("non-zero"), + } + } +} + +/// Wrapper around a throttle and how its tokens should be interpreted. +#[derive(Debug)] +pub(super) struct ThroughputThrottle { + /// Underlying throttle instance. + pub throttle: lading_throttle::Throttle, + /// Token interpretation mode. + pub mode: ThrottleMode, +} + /// Create a throttle from optional config and `bytes_per_second` fallback /// /// This function implements the standard throttle creation logic for @@ -92,6 +123,38 @@ pub(super) fn create_throttle( Ok(lading_throttle::Throttle::new_with_config(throttle_config)) } +/// Create a throttle that can be interpreted either as bytes-per-second +/// or blocks-per-second. +/// +/// Blocks-based throttling conflicts with byte-based throttling; providing both +/// will result in [`ThrottleConversionError::ConflictingConfig`]. +pub(super) fn create_block_or_byte_throttle( + config: Option<&BytesThrottleConfig>, + bytes_per_second: Option<&byte_unit::Byte>, + blocks_per_second: Option, +) -> Result { + if let Some(blocks) = blocks_per_second { + if config.is_some() || bytes_per_second.is_some() { + return Err(ThrottleConversionError::ConflictingConfig); + } + let throttle = + lading_throttle::Throttle::new_with_config(lading_throttle::Config::Stable { + maximum_capacity: blocks, + timeout_micros: 0, + }); + return Ok(ThroughputThrottle { + throttle, + mode: ThrottleMode::Blocks, + }); + } + + let throttle = create_throttle(config, bytes_per_second)?; + Ok(ThroughputThrottle { + throttle, + mode: ThrottleMode::Bytes, + }) +} + impl TryFrom<&BytesThrottleConfig> for lading_throttle::Config { type Error = ThrottleConversionError; diff --git a/lading/src/generator/file_gen/logrotate.rs b/lading/src/generator/file_gen/logrotate.rs index 5efdbbbb5..5212c5e09 100644 --- a/lading/src/generator/file_gen/logrotate.rs +++ b/lading/src/generator/file_gen/logrotate.rs @@ -36,7 +36,8 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - BytesThrottleConfig, MetricsBuilder, ThrottleConversionError, create_throttle, + BytesThrottleConfig, MetricsBuilder, ThrottleConversionError, ThrottleMode, + create_block_or_byte_throttle, }; /// An enum to allow us to determine what operation caused an IO errror as the @@ -149,6 +150,9 @@ pub struct Config { block_cache_method: block::CacheMethod, /// The load throttle configuration pub throttle: Option, + /// Optional blocks-per-second throttle. Conflicts with byte-based throttles. + #[serde(default)] + pub blocks_per_second: Option, } #[derive(Debug)] @@ -214,8 +218,11 @@ impl Server { let mut handles = Vec::new(); for idx in 0..config.concurrent_logs { - let throttle = - create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; + let throughput_throttle = create_block_or_byte_throttle( + config.throttle.as_ref(), + config.bytes_per_second.as_ref(), + config.blocks_per_second, + )?; let mut dir_path = config.root.clone(); let depth = rng.random_range(0..config.max_depth); @@ -235,7 +242,8 @@ impl Server { config.total_rotations, maximum_bytes_per_log, Arc::clone(&block_cache), - throttle, + throughput_throttle.throttle, + throughput_throttle.mode, shutdown.clone(), child_labels, ); @@ -285,6 +293,7 @@ struct Child { maximum_bytes_per_log: NonZeroU32, block_cache: Arc, throttle: lading_throttle::Throttle, + throttle_mode: ThrottleMode, shutdown: lading_signal::Watcher, labels: Vec<(String, String)>, } @@ -297,6 +306,7 @@ impl Child { maximum_bytes_per_log: NonZeroU32, block_cache: Arc, throttle: lading_throttle::Throttle, + throttle_mode: ThrottleMode, shutdown: lading_signal::Watcher, labels: Vec<(String, String)>, ) -> Self { @@ -318,13 +328,18 @@ impl Child { maximum_bytes_per_log, block_cache, throttle, + throttle_mode, shutdown, labels, } } async fn spin(mut self) -> Result<(), Error> { - let buffer_capacity = self.throttle.maximum_capacity() as usize; + let mut handle = self.block_cache.handle(); + let buffer_capacity = match self.throttle_mode { + ThrottleMode::Bytes => self.throttle.maximum_capacity() as usize, + ThrottleMode::Blocks => self.block_cache.peek_next_size(&handle).get() as usize, + }; let mut total_bytes_written: u64 = 0; let maximum_bytes_per_log: u64 = u64::from(self.maximum_bytes_per_log.get()); @@ -357,17 +372,16 @@ impl Child { })?, ); - let mut handle = self.block_cache.handle(); - let shutdown_wait = self.shutdown.recv(); tokio::pin!(shutdown_wait); loop { // SAFETY: By construction the block cache will never be empty // except in the event of a catastrophic failure. let total_bytes = self.block_cache.peek_next_size(&handle); + let tokens = self.throttle_mode.tokens_for_block(total_bytes); tokio::select! { - result = self.throttle.wait_for(total_bytes) => { + result = self.throttle.wait_for(tokens) => { match result { Ok(()) => { let block = self.block_cache.advance(&mut handle); @@ -382,7 +396,9 @@ impl Child { &self.labels).await?; } Err(err) => { - error!("Throttle request of {} is larger than throttle capacity. Block will be discarded. Error: {}", total_bytes, err); + error!( + "Throttle request of {tokens} token(s) for block size {total_bytes} is larger than throttle capacity. Block will be discarded. Error: {err}" + ); } } } diff --git a/lading/src/generator/file_gen/logrotate_fs.rs b/lading/src/generator/file_gen/logrotate_fs.rs index c07d58ffc..4a9bbe34c 100644 --- a/lading/src/generator/file_gen/logrotate_fs.rs +++ b/lading/src/generator/file_gen/logrotate_fs.rs @@ -57,6 +57,10 @@ pub struct Config { mount_point: PathBuf, /// The load profile, controlling bytes per second as a function of time. load_profile: LoadProfile, + /// Optional blocks-per-second throttle. When set, overrides `load_profile` + /// to a constant rate derived from the average block size. + #[serde(default)] + blocks_per_second: Option, } /// Profile for load in this filesystem. @@ -154,6 +158,26 @@ impl Server { // divvy this up in the future. total_bytes.get() as usize, )?; + let average_block_size = { + let len = block_cache.len() as u64; + if len == 0 { + 1 + } else { + block_cache.total_size().saturating_div(len).max(1) + } + }; + let load_profile = if let Some(blocks_per_second) = config.blocks_per_second { + let bytes = average_block_size.saturating_mul(u64::from(blocks_per_second.get())); + info!( + blocks_per_second = blocks_per_second.get(), + average_block_size, + derived_bytes_per_second = bytes, + "logrotate_fs using block-based throttle derived from average block size" + ); + LoadProfile::Constant(byte_unit::Byte::from_u64(bytes)) + } else { + config.load_profile + }; let start_time = Instant::now(); let start_time_system = SystemTime::now(); @@ -166,7 +190,7 @@ impl Server { block_cache, config.max_depth, config.concurrent_logs, - config.load_profile.to_model(), + load_profile.to_model(), ); info!( diff --git a/lading/src/generator/file_gen/traditional.rs b/lading/src/generator/file_gen/traditional.rs index 1d6157484..fc3972792 100644 --- a/lading/src/generator/file_gen/traditional.rs +++ b/lading/src/generator/file_gen/traditional.rs @@ -38,7 +38,8 @@ use lading_payload::{self, block}; use super::General; use crate::generator::common::{ - BytesThrottleConfig, MetricsBuilder, ThrottleConversionError, create_throttle, + BytesThrottleConfig, MetricsBuilder, ThrottleConversionError, ThrottleMode, + create_block_or_byte_throttle, }; #[derive(thiserror::Error, Debug)] @@ -121,6 +122,9 @@ pub struct Config { rotate: bool, /// The load throttle configuration pub throttle: Option, + /// Optional blocks-per-second throttle. Conflicts with byte-based throttles. + #[serde(default)] + pub blocks_per_second: Option, /// Optional fixed interval between blocks. When set, the generator waits /// this duration before emitting the next block, regardless of byte size. pub block_interval_millis: Option, @@ -179,8 +183,11 @@ impl Server { let file_index = Arc::new(AtomicU32::new(0)); for _ in 0..config.duplicates { - let throttle = - create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; + let throughput_throttle = create_block_or_byte_throttle( + config.throttle.as_ref(), + config.bytes_per_second.as_ref(), + config.blocks_per_second, + )?; let block_cache = match config.block_cache_method { block::CacheMethod::Fixed => block::Cache::fixed_with_max_overhead( @@ -203,7 +210,8 @@ impl Server { let child = Child { path_template: config.path_template.clone(), maximum_bytes_per_file, - throttle, + throttle: throughput_throttle.throttle, + throttle_mode: throughput_throttle.mode, block_cache: Arc::new(block_cache), file_index: Arc::clone(&file_index), rotate: config.rotate, @@ -281,6 +289,7 @@ struct Child { path_template: String, maximum_bytes_per_file: NonZeroU32, throttle: lading_throttle::Throttle, + throttle_mode: ThrottleMode, block_cache: Arc, rotate: bool, file_index: Arc, @@ -292,32 +301,12 @@ struct Child { impl Child { pub(crate) async fn spin(mut self) -> Result<(), Error> { - let buffer_capacity = self.throttle.maximum_capacity() as usize; let mut total_bytes_written: u64 = 0; let maximum_bytes_per_file: u64 = u64::from(self.maximum_bytes_per_file.get()); let mut file_index = self.file_index.fetch_add(1, Ordering::Relaxed); let mut path = path_from_template(&self.path_template, file_index); - let mut fp = BufWriter::with_capacity( - buffer_capacity, - fs::OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(&path) - .await - .map_err(|source| { - error!( - "Failed to open file {path:?}: {source}. Ensure parent directory exists." - ); - Error::FileOpen { - path: path.clone(), - source, - } - })?, - ); - let mut handle = self.block_cache.handle(); if self.start_line_index > 0 { let mut remaining = self.start_line_index; @@ -336,6 +325,28 @@ impl Child { let _ = self.block_cache.advance(&mut handle); } } + let buffer_capacity = match self.throttle_mode { + ThrottleMode::Bytes => self.throttle.maximum_capacity() as usize, + ThrottleMode::Blocks => self.block_cache.peek_next_size(&handle).get() as usize, + }; + let mut fp = BufWriter::with_capacity( + buffer_capacity, + fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&path) + .await + .map_err(|source| { + error!( + "Failed to open file {path:?}: {source}. Ensure parent directory exists." + ); + Error::FileOpen { + path: path.clone(), + source, + } + })?, + ); let shutdown_wait = self.shutdown.recv(); tokio::pin!(shutdown_wait); let mut next_tick = self @@ -344,9 +355,10 @@ impl Child { .map(|dur| Instant::now() + *dur); loop { let total_bytes = self.block_cache.peek_next_size(&handle); + let tokens = self.throttle_mode.tokens_for_block(total_bytes); tokio::select! { - result = self.throttle.wait_for(total_bytes) => { + result = self.throttle.wait_for(tokens) => { match result { Ok(()) => { if let Some(dur) = self.block_interval { @@ -409,7 +421,9 @@ impl Child { } } Err(err) => { - error!("Throttle request of {total_bytes} is larger than throttle capacity. Block will be discarded. Error: {err}"); + error!( + "Throttle request of {tokens} token(s) for block size {total_bytes} is larger than throttle capacity. Block will be discarded. Error: {err}" + ); } } } diff --git a/lading_payload/src/block.rs b/lading_payload/src/block.rs index ef1e2e398..4a9b3fd4c 100644 --- a/lading_payload/src/block.rs +++ b/lading_payload/src/block.rs @@ -393,7 +393,7 @@ impl Cache { static_path, ×tamp_format, *emit_placeholder, - start_line_index.unwrap_or(0), + *start_line_index, )?; construct_block_cache_inner( &mut rng, diff --git a/lading_payload/src/statik_second.rs b/lading_payload/src/statik_second.rs index c23ffb0f4..eb263cb27 100644 --- a/lading_payload/src/statik_second.rs +++ b/lading_payload/src/statik_second.rs @@ -47,12 +47,14 @@ impl StaticSecond { /// timestamp is parsed from the start of the line up to the first /// whitespace, using `timestamp_format` (chrono strftime syntax). The /// parsed timestamp is removed from the emitted line, leaving only the - /// remainder of the message. + /// remainder of the message. `start_line_index`, when provided, skips that + /// many lines (modulo the total number of available lines) before + /// returning payloads. pub fn new( path: &Path, timestamp_format: &str, emit_placeholder: bool, - start_line_index: u64, + start_line_index: Option, ) -> Result { let file = File::open(path)?; let reader = BufReader::new(file); @@ -128,6 +130,7 @@ impl StaticSecond { return Err(Error::NoLines); } + let start_line_index = start_line_index.unwrap_or(0); // Apply starting line offset by trimming leading lines across buckets. let total_lines: u64 = blocks.iter().map(|b| b.lines.len() as u64).sum(); let mut start_idx = 0usize; @@ -239,6 +242,7 @@ mod tests { &path, "%Y-%m-%dT%H:%M:%S", /* emit_placeholder */ false, + None, ) .unwrap(); let mut rng = StdRng::seed_from_u64(7); @@ -251,4 +255,58 @@ mod tests { serializer.to_bytes(&mut rng, 1024, &mut buf).unwrap(); assert_eq!(buf, b"third\n"); } + + #[test] + fn emits_placeholders_for_missing_seconds() { + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("placeholder_test.log"); + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "2024-01-01T00:00:00 first").unwrap(); + // Intentionally skip 00:00:01 + writeln!(f, "2024-01-01T00:00:02 third").unwrap(); + } + + let mut serializer = StaticSecond::new(&path, "%Y-%m-%dT%H:%M:%S", true, None).unwrap(); + let mut rng = StdRng::seed_from_u64(7); + + let mut buf = Vec::new(); + serializer.to_bytes(&mut rng, 1024, &mut buf).unwrap(); + assert_eq!(buf, b"first\n"); + + buf.clear(); + serializer.to_bytes(&mut rng, 1024, &mut buf).unwrap(); + // Placeholder newline for the missing second + assert_eq!(buf, b"\n"); + + buf.clear(); + serializer.to_bytes(&mut rng, 1024, &mut buf).unwrap(); + assert_eq!(buf, b"third\n"); + } + + #[test] + fn honors_start_line_index_with_wraparound() { + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("start_index_test.log"); + { + let mut f = File::create(&path).unwrap(); + // Two lines in the first second, one in the second second. + writeln!(f, "2024-01-01T00:00:00 first").unwrap(); + writeln!(f, "2024-01-01T00:00:00 second").unwrap(); + writeln!(f, "2024-01-01T00:00:01 third").unwrap(); + } + + // Skip the first two lines; the stream should begin with "third". + let mut serializer = StaticSecond::new(&path, "%Y-%m-%dT%H:%M:%S", false, Some(2)).unwrap(); + let mut rng = StdRng::seed_from_u64(7); + + let mut buf = Vec::new(); + serializer.to_bytes(&mut rng, 1024, &mut buf).unwrap(); + assert_eq!(buf, b"third\n"); + + buf.clear(); + serializer.to_bytes(&mut rng, 1024, &mut buf).unwrap(); + // After wrapping, we return to the beginning of the stream. + assert_eq!(buf, b"first\nsecond\n"); + } } From 1147f701949d996a63980fc1198dc7291d7f0d7c Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Fri, 19 Dec 2025 12:53:52 -0500 Subject: [PATCH 06/21] loadprofile --- lading/src/generator/common.rs | 83 +++++++++++++------ lading/src/generator/file_gen/logrotate.rs | 27 +++--- lading/src/generator/file_gen/logrotate_fs.rs | 15 +++- lading/src/generator/file_gen/traditional.rs | 28 ++++--- 4 files changed, 102 insertions(+), 51 deletions(-) diff --git a/lading/src/generator/common.rs b/lading/src/generator/common.rs index 0c5d0c0e2..e7194efcd 100644 --- a/lading/src/generator/common.rs +++ b/lading/src/generator/common.rs @@ -76,6 +76,36 @@ pub(super) struct ThroughputThrottle { pub mode: ThrottleMode, } +#[derive(Debug, Deserialize, Serialize, PartialEq, Clone)] +#[serde(rename_all = "snake_case", tag = "mode")] +#[serde(deny_unknown_fields)] +/// Unified throughput profile for file generators. +pub enum ThroughputProfile { + /// Byte-oriented throttling (default). + Bytes { + /// Standard throttle configuration. + #[serde(default)] + throttle: Option, + /// Legacy shorthand for a stable bytes-per-second throttle. + #[serde(default)] + bytes_per_second: Option, + }, + /// Block-oriented throttling; each emitted block costs one token. + Blocks { + /// Blocks allowed per second. + blocks_per_second: NonZeroU32, + }, +} + +impl Default for ThroughputProfile { + fn default() -> Self { + Self::Bytes { + throttle: None, + bytes_per_second: None, + } + } +} + /// Create a throttle from optional config and `bytes_per_second` fallback /// /// This function implements the standard throttle creation logic for @@ -123,36 +153,35 @@ pub(super) fn create_throttle( Ok(lading_throttle::Throttle::new_with_config(throttle_config)) } -/// Create a throttle that can be interpreted either as bytes-per-second -/// or blocks-per-second. -/// -/// Blocks-based throttling conflicts with byte-based throttling; providing both -/// will result in [`ThrottleConversionError::ConflictingConfig`]. -pub(super) fn create_block_or_byte_throttle( - config: Option<&BytesThrottleConfig>, - bytes_per_second: Option<&byte_unit::Byte>, - blocks_per_second: Option, +/// Create a throttle from a unified throughput profile. +pub(super) fn create_throughput_throttle( + profile: Option<&ThroughputProfile>, ) -> Result { - if let Some(blocks) = blocks_per_second { - if config.is_some() || bytes_per_second.is_some() { - return Err(ThrottleConversionError::ConflictingConfig); - } - let throttle = - lading_throttle::Throttle::new_with_config(lading_throttle::Config::Stable { - maximum_capacity: blocks, - timeout_micros: 0, - }); - return Ok(ThroughputThrottle { + match profile.unwrap_or(&ThroughputProfile::default()) { + ThroughputProfile::Bytes { throttle, - mode: ThrottleMode::Blocks, - }); + bytes_per_second, + } => { + let throttle = create_throttle(throttle.as_ref(), bytes_per_second.as_ref())?; + Ok(ThroughputThrottle { + throttle, + mode: ThrottleMode::Bytes, + }) + } + ThroughputProfile::Blocks { + blocks_per_second, + } => { + let throttle = + lading_throttle::Throttle::new_with_config(lading_throttle::Config::Stable { + maximum_capacity: *blocks_per_second, + timeout_micros: 0, + }); + Ok(ThroughputThrottle { + throttle, + mode: ThrottleMode::Blocks, + }) + } } - - let throttle = create_throttle(config, bytes_per_second)?; - Ok(ThroughputThrottle { - throttle, - mode: ThrottleMode::Bytes, - }) } impl TryFrom<&BytesThrottleConfig> for lading_throttle::Config { diff --git a/lading/src/generator/file_gen/logrotate.rs b/lading/src/generator/file_gen/logrotate.rs index 5212c5e09..15fc4b1a6 100644 --- a/lading/src/generator/file_gen/logrotate.rs +++ b/lading/src/generator/file_gen/logrotate.rs @@ -36,8 +36,8 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - BytesThrottleConfig, MetricsBuilder, ThrottleConversionError, ThrottleMode, - create_block_or_byte_throttle, + MetricsBuilder, ThrottleConversionError, ThrottleMode, ThroughputProfile, + create_throughput_throttle, }; /// An enum to allow us to determine what operation caused an IO errror as the @@ -138,6 +138,7 @@ pub struct Config { /// Sets the [`crate::payload::Config`] of this template. pub variant: lading_payload::Config, /// Defines the number of bytes that written in each log file. + #[deprecated(note = "Use load_profile.bytes.bytes_per_second instead")] bytes_per_second: Option, /// Defines the maximum internal cache of this log target. `file_gen` will /// pre-build its outputs up to the byte capacity specified here. @@ -148,11 +149,9 @@ pub struct Config { /// Whether to use a fixed or streaming block cache #[serde(default = "lading_payload::block::default_cache_method")] block_cache_method: block::CacheMethod, - /// The load throttle configuration - pub throttle: Option, - /// Optional blocks-per-second throttle. Conflicts with byte-based throttles. + /// Throughput profile controlling emission rate (bytes or blocks). #[serde(default)] - pub blocks_per_second: Option, + pub load_profile: Option, } #[derive(Debug)] @@ -218,11 +217,17 @@ impl Server { let mut handles = Vec::new(); for idx in 0..config.concurrent_logs { - let throughput_throttle = create_block_or_byte_throttle( - config.throttle.as_ref(), - config.bytes_per_second.as_ref(), - config.blocks_per_second, - )?; + let throughput_throttle = + create_throughput_throttle(config.load_profile.as_ref()).or_else(|e| { + if config.bytes_per_second.is_some() { + create_throughput_throttle(Some(&ThroughputProfile::Bytes { + throttle: None, + bytes_per_second: config.bytes_per_second, + })) + } else { + Err(e) + } + })?; let mut dir_path = config.root.clone(); let depth = rng.random_range(0..config.max_depth); diff --git a/lading/src/generator/file_gen/logrotate_fs.rs b/lading/src/generator/file_gen/logrotate_fs.rs index 4a9bbe34c..070d19f15 100644 --- a/lading/src/generator/file_gen/logrotate_fs.rs +++ b/lading/src/generator/file_gen/logrotate_fs.rs @@ -76,10 +76,15 @@ pub enum LoadProfile { /// Amount to increase per second rate: byte_unit::Byte, }, + /// Constant blocks per second (derived to bytes via average block size). + Blocks { + /// Blocks per second + blocks_per_second: NonZeroU32, + }, } impl LoadProfile { - fn to_model(self) -> model::LoadProfile { + fn to_model(self, average_block_size: u64) -> model::LoadProfile { // For now, one tick is one second. match self { LoadProfile::Constant(bpt) => model::LoadProfile::Constant(bpt.as_u128() as u64), @@ -90,6 +95,12 @@ impl LoadProfile { start: initial_bytes_per_second.as_u128() as u64, rate: rate.as_u128() as u64, }, + LoadProfile::Blocks { blocks_per_second } => { + let bytes = average_block_size + .saturating_mul(u64::from(blocks_per_second.get())) + .max(1); + model::LoadProfile::Constant(bytes) + } } } } @@ -190,7 +201,7 @@ impl Server { block_cache, config.max_depth, config.concurrent_logs, - load_profile.to_model(), + load_profile.to_model(average_block_size), ); info!( diff --git a/lading/src/generator/file_gen/traditional.rs b/lading/src/generator/file_gen/traditional.rs index fc3972792..ad15dc565 100644 --- a/lading/src/generator/file_gen/traditional.rs +++ b/lading/src/generator/file_gen/traditional.rs @@ -38,8 +38,8 @@ use lading_payload::{self, block}; use super::General; use crate::generator::common::{ - BytesThrottleConfig, MetricsBuilder, ThrottleConversionError, ThrottleMode, - create_block_or_byte_throttle, + MetricsBuilder, ThrottleConversionError, ThrottleMode, ThroughputProfile, + create_throughput_throttle, }; #[derive(thiserror::Error, Debug)] @@ -105,6 +105,7 @@ pub struct Config { /// written _continuously_ per second from this target. Higher bursts are /// possible as the internal governor accumulates, up to /// `maximum_bytes_burst`. + #[deprecated(note = "Use load_profile.bytes.bytes_per_second instead")] bytes_per_second: Option, /// Defines the maximum internal cache of this log target. `file_gen` will /// pre-build its outputs up to the byte capacity specified here. @@ -120,11 +121,9 @@ pub struct Config { /// tailing software to remove old files. #[serde(default = "default_rotation")] rotate: bool, - /// The load throttle configuration - pub throttle: Option, - /// Optional blocks-per-second throttle. Conflicts with byte-based throttles. + /// Throughput profile controlling emission rate (bytes or blocks). #[serde(default)] - pub blocks_per_second: Option, + pub load_profile: Option, /// Optional fixed interval between blocks. When set, the generator waits /// this duration before emitting the next block, regardless of byte size. pub block_interval_millis: Option, @@ -183,11 +182,18 @@ impl Server { let file_index = Arc::new(AtomicU32::new(0)); for _ in 0..config.duplicates { - let throughput_throttle = create_block_or_byte_throttle( - config.throttle.as_ref(), - config.bytes_per_second.as_ref(), - config.blocks_per_second, - )?; + let throughput_throttle = + create_throughput_throttle(config.load_profile.as_ref()).or_else(|e| { + // Backwards compatibility: fall back to deprecated fields. + if config.bytes_per_second.is_some() { + create_throughput_throttle(Some(&ThroughputProfile::Bytes { + throttle: None, + bytes_per_second: config.bytes_per_second, + })) + } else { + Err(e) + } + })?; let block_cache = match config.block_cache_method { block::CacheMethod::Fixed => block::Cache::fixed_with_max_overhead( From c144dc0c4ad8d90bd88b83dd7523a0ef20137795 Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Fri, 19 Dec 2025 14:24:02 -0500 Subject: [PATCH 07/21] more changes --- lading/src/generator/common.rs | 315 +++++++++++------- lading/src/generator/file_gen/logrotate.rs | 46 ++- lading/src/generator/file_gen/logrotate_fs.rs | 43 ++- lading/src/generator/file_gen/traditional.rs | 43 +-- lading/src/generator/grpc.rs | 4 +- lading/src/generator/http.rs | 5 +- lading/src/generator/passthru_file.rs | 4 +- lading/src/generator/splunk_hec.rs | 4 +- lading/src/generator/tcp.rs | 5 +- lading/src/generator/trace_agent.rs | 5 +- lading/src/generator/udp.rs | 5 +- lading/src/generator/unix_datagram.rs | 4 +- lading/src/generator/unix_stream.rs | 4 +- 13 files changed, 293 insertions(+), 194 deletions(-) diff --git a/lading/src/generator/common.rs b/lading/src/generator/common.rs index e7194efcd..aef4481ac 100644 --- a/lading/src/generator/common.rs +++ b/lading/src/generator/common.rs @@ -1,33 +1,73 @@ //! Common types for generators use byte_unit::Byte; +use lading_payload::block::Block; use serde::{Deserialize, Serialize}; use std::num::{NonZeroU16, NonZeroU32}; -/// Generator-specific throttle configuration with field names that are specific -/// to byte-oriented generators. +/// Unified rate specification; defaults to bytes when `mode` is unset. +#[derive(Debug, Deserialize, Serialize, PartialEq, Clone, Copy, Default)] +#[serde(deny_unknown_fields)] +pub struct RateSpec { + /// Throttle mode; defaults to bytes when absent. + #[serde(default)] + pub mode: Option, + /// Bytes per second (bytes mode only). + #[serde(default)] + pub bytes_per_second: Option, + /// Blocks per second (blocks mode only). + #[serde(default)] + pub blocks_per_second: Option, +} + +impl RateSpec { + fn resolve(&self) -> Result<(ThrottleMode, NonZeroU32), ThrottleConversionError> { + let mode = self.mode.unwrap_or(ThrottleMode::Bytes); + match mode { + ThrottleMode::Bytes => { + let bps = self + .bytes_per_second + .ok_or(ThrottleConversionError::MissingRate)?; + let val = bps.as_u128(); + if val > u128::from(u32::MAX) { + return Err(ThrottleConversionError::ValueTooLarge(bps)); + } + NonZeroU32::new(val as u32) + .map(|n| (ThrottleMode::Bytes, n)) + .ok_or(ThrottleConversionError::Zero) + } + ThrottleMode::Blocks => self + .blocks_per_second + .map(|n| (ThrottleMode::Blocks, n)) + .ok_or(ThrottleConversionError::MissingRate), + } + } +} + +/// Generator-specific throttle configuration unified for bytes or blocks. #[derive(Debug, Deserialize, Serialize, PartialEq, Clone, Copy)] #[serde(rename_all = "snake_case")] #[serde(deny_unknown_fields)] -pub enum BytesThrottleConfig { +pub enum ThrottleConfig { /// A throttle that allows the generator to produce as fast as possible AllOut, /// A throttle that attempts stable load Stable { - /// The bytes per second rate limit (e.g., "1MB", "512KiB") - bytes_per_second: Byte, + /// Rate specification (bytes or blocks). Defaults to bytes when mode is unset. + #[serde(default)] + rate: RateSpec, /// The timeout in milliseconds for IO operations. Default is 0. #[serde(default)] timeout_millis: u64, }, /// A throttle that linearly increases load over time Linear { - /// The initial bytes per second (e.g., "100KB") - initial_bytes_per_second: Byte, - /// The maximum bytes per second (e.g., "10MB") - maximum_bytes_per_second: Byte, - /// The rate of change in bytes per second per second - rate_of_change: Byte, + /// The initial rate (bytes or blocks per second) + initial: RateSpec, + /// The maximum rate (bytes or blocks per second) + maximum: RateSpec, + /// The rate of change per second (bytes or blocks per second) + rate_of_change: RateSpec, }, } @@ -43,66 +83,45 @@ pub enum ThrottleConversionError { /// Conflicting configuration provided #[error("Cannot specify both throttle config and bytes_per_second")] ConflictingConfig, + /// Missing rate specification + #[error("Rate must be specified for the selected throttle mode")] + MissingRate, + /// Mixed throttle modes in a linear profile + #[error("All rate specs in a linear throttle must use the same mode")] + MixedModes, } /// Indicates how a throttle should interpret its token units. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum ThrottleMode { +#[derive(Debug, Deserialize, Serialize, PartialEq, Clone, Copy)] +#[serde(rename_all = "snake_case")] +#[serde(deny_unknown_fields)] +pub enum ThrottleMode { /// Throttle tokens represent bytes. Bytes, /// Throttle tokens represent block counts. Blocks, } -impl ThrottleMode { - /// Return the number of tokens required for a block of the given byte size. - /// - /// Bytes mode consumes the actual block size. Blocks mode consumes a single - /// token per block regardless of its byte length. - pub(super) fn tokens_for_block(&self, block_size: NonZeroU32) -> NonZeroU32 { - match self { - ThrottleMode::Bytes => block_size, - ThrottleMode::Blocks => NonZeroU32::new(1).expect("non-zero"), - } - } -} - /// Wrapper around a throttle and how its tokens should be interpreted. #[derive(Debug)] -pub(super) struct ThroughputThrottle { +pub(super) struct BlockThrottle { /// Underlying throttle instance. pub throttle: lading_throttle::Throttle, /// Token interpretation mode. pub mode: ThrottleMode, } -#[derive(Debug, Deserialize, Serialize, PartialEq, Clone)] -#[serde(rename_all = "snake_case", tag = "mode")] -#[serde(deny_unknown_fields)] -/// Unified throughput profile for file generators. -pub enum ThroughputProfile { - /// Byte-oriented throttling (default). - Bytes { - /// Standard throttle configuration. - #[serde(default)] - throttle: Option, - /// Legacy shorthand for a stable bytes-per-second throttle. - #[serde(default)] - bytes_per_second: Option, - }, - /// Block-oriented throttling; each emitted block costs one token. - Blocks { - /// Blocks allowed per second. - blocks_per_second: NonZeroU32, - }, -} - -impl Default for ThroughputProfile { - fn default() -> Self { - Self::Bytes { - throttle: None, - bytes_per_second: None, - } +impl BlockThrottle { + /// Wait for capacity for a block, interpreting tokens according to `mode`. + pub(super) async fn wait_for_block( + &mut self, + block: &Block, + ) -> Result<(), lading_throttle::Error> { + let tokens: NonZeroU32 = match self.mode { + ThrottleMode::Bytes => block.total_bytes, + ThrottleMode::Blocks => NonZeroU32::new(1).expect("non-zero"), + }; + self.throttle.wait_for(tokens).await } } @@ -128,9 +147,34 @@ impl Default for ThroughputProfile { /// - The `bytes_per_second` value exceeds `u32::MAX` /// - The `bytes_per_second` value is zero pub(super) fn create_throttle( - config: Option<&BytesThrottleConfig>, + config: Option<&ThrottleConfig>, bytes_per_second: Option<&byte_unit::Byte>, ) -> Result { + // Bytes-only helper for legacy callers. Reject block-mode usage here. + if let Some(ThrottleConfig::Stable { rate, .. }) = config { + if matches!( + rate.mode.unwrap_or(ThrottleMode::Bytes), + ThrottleMode::Blocks + ) { + return Err(ThrottleConversionError::ConflictingConfig); + } + } + if let Some(ThrottleConfig::Linear { + initial, + maximum, + rate_of_change, + }) = config + { + let modes = [ + initial.mode.unwrap_or(ThrottleMode::Bytes), + maximum.mode.unwrap_or(ThrottleMode::Bytes), + rate_of_change.mode.unwrap_or(ThrottleMode::Bytes), + ]; + if modes.iter().any(|m| *m == ThrottleMode::Blocks) { + return Err(ThrottleConversionError::ConflictingConfig); + } + } + let throttle_config = match (config, bytes_per_second) { (Some(_), Some(_)) => { return Err(ThrottleConversionError::ConflictingConfig); @@ -153,92 +197,127 @@ pub(super) fn create_throttle( Ok(lading_throttle::Throttle::new_with_config(throttle_config)) } -/// Create a throttle from a unified throughput profile. +/// Create a throttle from a unified config plus optional legacy fallbacks. pub(super) fn create_throughput_throttle( - profile: Option<&ThroughputProfile>, -) -> Result { - match profile.unwrap_or(&ThroughputProfile::default()) { - ThroughputProfile::Bytes { - throttle, - bytes_per_second, + profile: Option<&ThrottleConfig>, + legacy_bytes_per_second: Option<&byte_unit::Byte>, + legacy_blocks_per_second: Option, +) -> Result { + let fallback = if let Some(bps) = legacy_bytes_per_second { + Some(ThrottleConfig::Stable { + rate: RateSpec { + mode: Some(ThrottleMode::Bytes), + bytes_per_second: Some(*bps), + blocks_per_second: None, + }, + timeout_millis: 0, + }) + } else if let Some(bps) = legacy_blocks_per_second { + Some(ThrottleConfig::Stable { + rate: RateSpec { + mode: Some(ThrottleMode::Blocks), + bytes_per_second: None, + blocks_per_second: Some(bps), + }, + timeout_millis: 0, + }) + } else { + None + }; + + let cfg = profile.copied().or(fallback); + let throttle_cfg = cfg.ok_or(ThrottleConversionError::MissingRate)?; + + let throttle = match throttle_cfg { + ThrottleConfig::AllOut => { + lading_throttle::Throttle::new_with_config(lading_throttle::Config::AllOut) + } + ThrottleConfig::Stable { + rate, + timeout_millis, } => { - let throttle = create_throttle(throttle.as_ref(), bytes_per_second.as_ref())?; - Ok(ThroughputThrottle { - throttle, - mode: ThrottleMode::Bytes, + let (_mode, cap) = rate.resolve()?; + lading_throttle::Throttle::new_with_config(lading_throttle::Config::Stable { + maximum_capacity: cap, + timeout_micros: timeout_millis.saturating_mul(1000), }) } - ThroughputProfile::Blocks { - blocks_per_second, + ThrottleConfig::Linear { + initial, + maximum, + rate_of_change, } => { - let throttle = - lading_throttle::Throttle::new_with_config(lading_throttle::Config::Stable { - maximum_capacity: *blocks_per_second, - timeout_micros: 0, - }); - Ok(ThroughputThrottle { - throttle, - mode: ThrottleMode::Blocks, + let (m1, init) = initial.resolve()?; + let (m2, max) = maximum.resolve()?; + let (m3, rate) = rate_of_change.resolve()?; + if m1 != m2 || m1 != m3 { + return Err(ThrottleConversionError::MixedModes); + } + lading_throttle::Throttle::new_with_config(lading_throttle::Config::Linear { + initial_capacity: init.get(), + maximum_capacity: max, + rate_of_change: rate.get(), }) } - } + }; + + // Mode from the first rate in the config (AllOut => default to bytes mode) + let mode = match throttle_cfg { + ThrottleConfig::AllOut => ThrottleMode::Bytes, + ThrottleConfig::Stable { rate, .. } => rate.resolve()?.0, + ThrottleConfig::Linear { + initial, + maximum, + rate_of_change, + } => { + let (m1, _) = initial.resolve()?; + let (m2, _) = maximum.resolve()?; + let (m3, _) = rate_of_change.resolve()?; + if m1 != m2 || m1 != m3 { + return Err(ThrottleConversionError::MixedModes); + } + m1 + } + }; + + Ok(BlockThrottle { throttle, mode }) } -impl TryFrom<&BytesThrottleConfig> for lading_throttle::Config { +impl TryFrom<&ThrottleConfig> for lading_throttle::Config { type Error = ThrottleConversionError; #[allow(clippy::cast_possible_truncation)] - fn try_from(config: &BytesThrottleConfig) -> Result { + fn try_from(config: &ThrottleConfig) -> Result { match config { - BytesThrottleConfig::AllOut => Ok(lading_throttle::Config::AllOut), - BytesThrottleConfig::Stable { - bytes_per_second, + ThrottleConfig::AllOut => Ok(lading_throttle::Config::AllOut), + ThrottleConfig::Stable { + rate, timeout_millis, } => { - let value = bytes_per_second.as_u128(); - if value > u128::from(u32::MAX) { - return Err(ThrottleConversionError::ValueTooLarge(*bytes_per_second)); + let (mode, cap) = rate.resolve()?; + if mode != ThrottleMode::Bytes { + return Err(ThrottleConversionError::MixedModes); } - let value = value as u32; - let value = NonZeroU32::new(value).ok_or(ThrottleConversionError::Zero)?; Ok(lading_throttle::Config::Stable { - maximum_capacity: value, + maximum_capacity: cap, timeout_micros: timeout_millis.saturating_mul(1000), }) } - BytesThrottleConfig::Linear { - initial_bytes_per_second, - maximum_bytes_per_second, + ThrottleConfig::Linear { + initial, + maximum, rate_of_change, } => { - let initial = initial_bytes_per_second.as_u128(); - let maximum = maximum_bytes_per_second.as_u128(); - let rate = rate_of_change.as_u128(); - - if initial > u128::from(u32::MAX) { - return Err(ThrottleConversionError::ValueTooLarge( - *initial_bytes_per_second, - )); - } - if maximum > u128::from(u32::MAX) { - return Err(ThrottleConversionError::ValueTooLarge( - *maximum_bytes_per_second, - )); + let (m1, init) = initial.resolve()?; + let (m2, max) = maximum.resolve()?; + let (m3, rate) = rate_of_change.resolve()?; + if m1 != m2 || m1 != m3 || m1 != ThrottleMode::Bytes { + return Err(ThrottleConversionError::MixedModes); } - if rate > u128::from(u32::MAX) { - return Err(ThrottleConversionError::ValueTooLarge(*rate_of_change)); - } - - let initial = initial as u32; - let maximum = maximum as u32; - let rate = rate as u32; - - let maximum = NonZeroU32::new(maximum).ok_or(ThrottleConversionError::Zero)?; - Ok(lading_throttle::Config::Linear { - initial_capacity: initial, - maximum_capacity: maximum, - rate_of_change: rate, + initial_capacity: init.get(), + maximum_capacity: max, + rate_of_change: rate.get(), }) } } @@ -253,7 +332,7 @@ impl TryFrom<&BytesThrottleConfig> for lading_throttle::Config { /// - Pooled: Multiple concurrent requests with semaphore limiting (HTTP/Splunk /// HEC pattern) /// - Workers: Multiple persistent worker tasks (TCP/UDP/Unix pattern) -#[derive(Debug)] +#[derive(Debug, Clone, Copy)] pub(super) enum ConcurrencyStrategy { /// Pool of connections with semaphore limiting concurrent requests Pooled { diff --git a/lading/src/generator/file_gen/logrotate.rs b/lading/src/generator/file_gen/logrotate.rs index 15fc4b1a6..aa89878b0 100644 --- a/lading/src/generator/file_gen/logrotate.rs +++ b/lading/src/generator/file_gen/logrotate.rs @@ -36,7 +36,7 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - MetricsBuilder, ThrottleConversionError, ThrottleMode, ThroughputProfile, + BlockThrottle, MetricsBuilder, ThrottleConfig, ThrottleConversionError, ThrottleMode, create_throughput_throttle, }; @@ -138,7 +138,7 @@ pub struct Config { /// Sets the [`crate::payload::Config`] of this template. pub variant: lading_payload::Config, /// Defines the number of bytes that written in each log file. - #[deprecated(note = "Use load_profile.bytes.bytes_per_second instead")] + #[deprecated(note = "Use load_profile bytes-per-second instead")] bytes_per_second: Option, /// Defines the maximum internal cache of this log target. `file_gen` will /// pre-build its outputs up to the byte capacity specified here. @@ -151,7 +151,7 @@ pub struct Config { block_cache_method: block::CacheMethod, /// Throughput profile controlling emission rate (bytes or blocks). #[serde(default)] - pub load_profile: Option, + pub load_profile: Option, } #[derive(Debug)] @@ -217,17 +217,14 @@ impl Server { let mut handles = Vec::new(); for idx in 0..config.concurrent_logs { + let legacy_bps = { + #[allow(deprecated)] + { + config.bytes_per_second.as_ref() + } + }; let throughput_throttle = - create_throughput_throttle(config.load_profile.as_ref()).or_else(|e| { - if config.bytes_per_second.is_some() { - create_throughput_throttle(Some(&ThroughputProfile::Bytes { - throttle: None, - bytes_per_second: config.bytes_per_second, - })) - } else { - Err(e) - } - })?; + create_throughput_throttle(config.load_profile.as_ref(), legacy_bps, None)?; let mut dir_path = config.root.clone(); let depth = rng.random_range(0..config.max_depth); @@ -247,8 +244,7 @@ impl Server { config.total_rotations, maximum_bytes_per_log, Arc::clone(&block_cache), - throughput_throttle.throttle, - throughput_throttle.mode, + throughput_throttle, shutdown.clone(), child_labels, ); @@ -297,8 +293,7 @@ struct Child { // The soft limit bytes per file that will trigger a rotation. maximum_bytes_per_log: NonZeroU32, block_cache: Arc, - throttle: lading_throttle::Throttle, - throttle_mode: ThrottleMode, + throttle: BlockThrottle, shutdown: lading_signal::Watcher, labels: Vec<(String, String)>, } @@ -310,8 +305,7 @@ impl Child { total_rotations: u8, maximum_bytes_per_log: NonZeroU32, block_cache: Arc, - throttle: lading_throttle::Throttle, - throttle_mode: ThrottleMode, + throttle: BlockThrottle, shutdown: lading_signal::Watcher, labels: Vec<(String, String)>, ) -> Self { @@ -333,7 +327,6 @@ impl Child { maximum_bytes_per_log, block_cache, throttle, - throttle_mode, shutdown, labels, } @@ -341,8 +334,8 @@ impl Child { async fn spin(mut self) -> Result<(), Error> { let mut handle = self.block_cache.handle(); - let buffer_capacity = match self.throttle_mode { - ThrottleMode::Bytes => self.throttle.maximum_capacity() as usize, + let buffer_capacity = match self.throttle.mode { + ThrottleMode::Bytes => self.throttle.throttle.maximum_capacity() as usize, ThrottleMode::Blocks => self.block_cache.peek_next_size(&handle).get() as usize, }; let mut total_bytes_written: u64 = 0; @@ -382,14 +375,13 @@ impl Child { loop { // SAFETY: By construction the block cache will never be empty // except in the event of a catastrophic failure. - let total_bytes = self.block_cache.peek_next_size(&handle); - let tokens = self.throttle_mode.tokens_for_block(total_bytes); + let block = self.block_cache.advance(&mut handle); + let total_bytes = u64::from(block.total_bytes.get()); tokio::select! { - result = self.throttle.wait_for(tokens) => { + result = self.throttle.wait_for_block(block) => { match result { Ok(()) => { - let block = self.block_cache.advance(&mut handle); write_bytes(block, &mut fp, &mut total_bytes_written, @@ -402,7 +394,7 @@ impl Child { } Err(err) => { error!( - "Throttle request of {tokens} token(s) for block size {total_bytes} is larger than throttle capacity. Block will be discarded. Error: {err}" + "Throttle request for block size {total_bytes} failed. Block will be discarded. Error: {err}" ); } } diff --git a/lading/src/generator/file_gen/logrotate_fs.rs b/lading/src/generator/file_gen/logrotate_fs.rs index 070d19f15..ac63a6c04 100644 --- a/lading/src/generator/file_gen/logrotate_fs.rs +++ b/lading/src/generator/file_gen/logrotate_fs.rs @@ -5,6 +5,9 @@ #![allow(clippy::cast_possible_wrap)] use crate::generator; +use crate::generator::common::{ + BytesThrottleConfig, RateSpec, ThrottleConversionError, ThrottleMode, +}; use fuser::{ BackgroundSession, FileAttr, Filesystem, MountOption, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry, Request, spawn_mount2, @@ -61,6 +64,10 @@ pub struct Config { /// to a constant rate derived from the average block size. #[serde(default)] blocks_per_second: Option, + /// Unified throttle profile (bytes or blocks). If set to blocks, overrides + /// `load_profile` / `blocks_per_second` and derives bytes via average block size. + #[serde(default)] + pub load_throttle: Option, } /// Profile for load in this filesystem. @@ -114,6 +121,9 @@ pub enum Error { /// Creation of payload blocks failed. #[error("Block creation error: {0}")] Block(#[from] block::Error), + /// Throttle conversion error + #[error("Throttle configuration error: {0}")] + ThrottleConversion(#[from] ThrottleConversionError), /// Failed to convert, value is 0 #[error("Value provided must not be zero")] Zero, @@ -177,7 +187,38 @@ impl Server { block_cache.total_size().saturating_div(len).max(1) } }; - let load_profile = if let Some(blocks_per_second) = config.blocks_per_second { + let load_profile = if let Some(throttle) = &config.load_throttle { + match throttle { + BytesThrottleConfig::Stable { rate, .. } => { + let (mode, cap) = rate.resolve()?; + match mode { + ThrottleMode::Bytes => { + LoadProfile::Constant(byte_unit::Byte::from_u64(cap.get().into())) + } + ThrottleMode::Blocks => { + let bytes = average_block_size + .saturating_mul(u64::from(cap.get())) + .max(1); + info!( + blocks_per_second = cap.get(), + average_block_size, + derived_bytes_per_second = bytes, + "logrotate_fs using block-based throttle derived from average block size" + ); + LoadProfile::Constant(byte_unit::Byte::from_u64(bytes)) + } + } + } + BytesThrottleConfig::AllOut => LoadProfile::Blocks { + blocks_per_second: NonZeroU32::new(1).unwrap(), + }, + BytesThrottleConfig::Linear { .. } => { + return Err(Error::ThrottleConversion( + ThrottleConversionError::MixedModes, + )); + } + } + } else if let Some(blocks_per_second) = config.blocks_per_second { let bytes = average_block_size.saturating_mul(u64::from(blocks_per_second.get())); info!( blocks_per_second = blocks_per_second.get(), diff --git a/lading/src/generator/file_gen/traditional.rs b/lading/src/generator/file_gen/traditional.rs index ad15dc565..67699ab71 100644 --- a/lading/src/generator/file_gen/traditional.rs +++ b/lading/src/generator/file_gen/traditional.rs @@ -38,7 +38,7 @@ use lading_payload::{self, block}; use super::General; use crate::generator::common::{ - MetricsBuilder, ThrottleConversionError, ThrottleMode, ThroughputProfile, + BlockThrottle, MetricsBuilder, ThrottleConfig, ThrottleConversionError, ThrottleMode, create_throughput_throttle, }; @@ -123,7 +123,7 @@ pub struct Config { rotate: bool, /// Throughput profile controlling emission rate (bytes or blocks). #[serde(default)] - pub load_profile: Option, + pub load_profile: Option, /// Optional fixed interval between blocks. When set, the generator waits /// this duration before emitting the next block, regardless of byte size. pub block_interval_millis: Option, @@ -182,18 +182,14 @@ impl Server { let file_index = Arc::new(AtomicU32::new(0)); for _ in 0..config.duplicates { + let legacy_bps = { + #[allow(deprecated)] + { + config.bytes_per_second.as_ref() + } + }; let throughput_throttle = - create_throughput_throttle(config.load_profile.as_ref()).or_else(|e| { - // Backwards compatibility: fall back to deprecated fields. - if config.bytes_per_second.is_some() { - create_throughput_throttle(Some(&ThroughputProfile::Bytes { - throttle: None, - bytes_per_second: config.bytes_per_second, - })) - } else { - Err(e) - } - })?; + create_throughput_throttle(config.load_profile.as_ref(), legacy_bps, None)?; let block_cache = match config.block_cache_method { block::CacheMethod::Fixed => block::Cache::fixed_with_max_overhead( @@ -216,8 +212,7 @@ impl Server { let child = Child { path_template: config.path_template.clone(), maximum_bytes_per_file, - throttle: throughput_throttle.throttle, - throttle_mode: throughput_throttle.mode, + throttle: throughput_throttle, block_cache: Arc::new(block_cache), file_index: Arc::clone(&file_index), rotate: config.rotate, @@ -294,8 +289,7 @@ impl Server { struct Child { path_template: String, maximum_bytes_per_file: NonZeroU32, - throttle: lading_throttle::Throttle, - throttle_mode: ThrottleMode, + throttle: BlockThrottle, block_cache: Arc, rotate: bool, file_index: Arc, @@ -331,8 +325,8 @@ impl Child { let _ = self.block_cache.advance(&mut handle); } } - let buffer_capacity = match self.throttle_mode { - ThrottleMode::Bytes => self.throttle.maximum_capacity() as usize, + let buffer_capacity = match self.throttle.mode { + ThrottleMode::Bytes => self.throttle.throttle.maximum_capacity() as usize, ThrottleMode::Blocks => self.block_cache.peek_next_size(&handle).get() as usize, }; let mut fp = BufWriter::with_capacity( @@ -360,11 +354,11 @@ impl Child { .as_ref() .map(|dur| Instant::now() + *dur); loop { - let total_bytes = self.block_cache.peek_next_size(&handle); - let tokens = self.throttle_mode.tokens_for_block(total_bytes); + let block = self.block_cache.advance(&mut handle); + let total_bytes = u64::from(block.total_bytes.get()); tokio::select! { - result = self.throttle.wait_for(tokens) => { + result = self.throttle.wait_for_block(block) => { match result { Ok(()) => { if let Some(dur) = self.block_interval { @@ -383,9 +377,6 @@ impl Child { } } - let block = self.block_cache.advance(&mut handle); - let total_bytes = u64::from(total_bytes.get()); - { fp.write_all(&block.bytes).await?; counter!("bytes_written").increment(total_bytes); @@ -428,7 +419,7 @@ impl Child { } Err(err) => { error!( - "Throttle request of {tokens} token(s) for block size {total_bytes} is larger than throttle capacity. Block will be discarded. Error: {err}" + "Throttle request for block size {total_bytes} failed. Block will be discarded. Error: {err}" ); } } diff --git a/lading/src/generator/grpc.rs b/lading/src/generator/grpc.rs index 8cda3eab1..632e21df4 100644 --- a/lading/src/generator/grpc.rs +++ b/lading/src/generator/grpc.rs @@ -36,7 +36,7 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - BytesThrottleConfig, MetricsBuilder, ThrottleConversionError, create_throttle, + MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, }; /// Errors produced by [`Grpc`] @@ -115,7 +115,7 @@ pub struct Config { /// The total number of parallel connections to maintain pub parallel_connections: u16, /// The load throttle configuration - pub throttle: Option, + pub throttle: Option, } /// No-op tonic codec. Sends raw bytes and returns the number of bytes received. diff --git a/lading/src/generator/http.rs b/lading/src/generator/http.rs index 3ba49a64d..a4f08c0b1 100644 --- a/lading/src/generator/http.rs +++ b/lading/src/generator/http.rs @@ -30,8 +30,7 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - BytesThrottleConfig, ConcurrencyStrategy, MetricsBuilder, ThrottleConversionError, - create_throttle, + ConcurrencyStrategy, MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, }; static CONNECTION_SEMAPHORE: OnceCell = OnceCell::new(); @@ -75,7 +74,7 @@ pub struct Config { /// The total number of parallel connections to maintain pub parallel_connections: u16, /// The load throttle configuration - pub throttle: Option, + pub throttle: Option, } #[derive(thiserror::Error, Debug)] diff --git a/lading/src/generator/passthru_file.rs b/lading/src/generator/passthru_file.rs index 5baddd096..2d5fa3671 100644 --- a/lading/src/generator/passthru_file.rs +++ b/lading/src/generator/passthru_file.rs @@ -24,7 +24,7 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - BytesThrottleConfig, MetricsBuilder, ThrottleConversionError, create_throttle, + MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, }; #[derive(Debug, Deserialize, Serialize, PartialEq, Clone)] @@ -45,7 +45,7 @@ pub struct Config { /// The maximum size in bytes of the cache of prebuilt messages pub maximum_prebuild_cache_size_bytes: Byte, /// The load throttle configuration - pub throttle: Option, + pub throttle: Option, } /// Errors produced by [`PassthruFile`]. diff --git a/lading/src/generator/splunk_hec.rs b/lading/src/generator/splunk_hec.rs index 9cc03e7a3..7be189974 100644 --- a/lading/src/generator/splunk_hec.rs +++ b/lading/src/generator/splunk_hec.rs @@ -45,7 +45,7 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - BytesThrottleConfig, MetricsBuilder, ThrottleConversionError, create_throttle, + MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, }; static CONNECTION_SEMAPHORE: OnceCell = OnceCell::new(); @@ -93,7 +93,7 @@ pub struct Config { /// The total number of parallel connections to maintain pub parallel_connections: u16, /// The load throttle configuration - pub throttle: Option, + pub throttle: Option, } #[derive(thiserror::Error, Debug)] diff --git a/lading/src/generator/tcp.rs b/lading/src/generator/tcp.rs index 8bcdb3efe..3e862db6a 100644 --- a/lading/src/generator/tcp.rs +++ b/lading/src/generator/tcp.rs @@ -32,8 +32,7 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - BytesThrottleConfig, ConcurrencyStrategy, MetricsBuilder, ThrottleConversionError, - create_throttle, + ConcurrencyStrategy, MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, }; fn default_parallel_connections() -> u16 { @@ -61,7 +60,7 @@ pub struct Config { #[serde(default = "default_parallel_connections")] pub parallel_connections: u16, /// The load throttle configuration - pub throttle: Option, + pub throttle: Option, } #[derive(thiserror::Error, Debug)] diff --git a/lading/src/generator/trace_agent.rs b/lading/src/generator/trace_agent.rs index c31e8ef73..f611530c1 100644 --- a/lading/src/generator/trace_agent.rs +++ b/lading/src/generator/trace_agent.rs @@ -17,8 +17,7 @@ //! Additional metrics may be emitted by this generator's [throttle]. use super::General; use crate::generator::common::{ - BytesThrottleConfig, ConcurrencyStrategy, MetricsBuilder, ThrottleConversionError, - create_throttle, + ConcurrencyStrategy, MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, }; use bytes::Bytes; use http_body_util::combinators::BoxBody; @@ -175,7 +174,7 @@ pub struct Config { /// The total number of parallel connections to maintain pub parallel_connections: u16, /// The load throttle configuration - pub throttle: Option, + pub throttle: Option, } #[derive(thiserror::Error, Debug)] diff --git a/lading/src/generator/udp.rs b/lading/src/generator/udp.rs index 3062ae8e5..517f3a6d3 100644 --- a/lading/src/generator/udp.rs +++ b/lading/src/generator/udp.rs @@ -32,8 +32,7 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - BytesThrottleConfig, ConcurrencyStrategy, MetricsBuilder, ThrottleConversionError, - create_throttle, + ConcurrencyStrategy, MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, }; fn default_parallel_connections() -> u16 { @@ -66,7 +65,7 @@ pub struct Config { #[serde(default = "default_parallel_connections")] pub parallel_connections: u16, /// The load throttle configuration - pub throttle: Option, + pub throttle: Option, } /// Errors produced by [`Udp`]. diff --git a/lading/src/generator/unix_datagram.rs b/lading/src/generator/unix_datagram.rs index 006e0cc78..ba9f69136 100644 --- a/lading/src/generator/unix_datagram.rs +++ b/lading/src/generator/unix_datagram.rs @@ -29,7 +29,7 @@ use tracing::{debug, error, info}; use super::General; use crate::generator::common::{ - BytesThrottleConfig, MetricsBuilder, ThrottleConversionError, create_throttle, + MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, }; fn default_parallel_connections() -> u16 { @@ -68,7 +68,7 @@ pub struct Config { #[serde(default = "default_parallel_connections")] pub parallel_connections: u16, /// The load throttle configuration - pub throttle: Option, + pub throttle: Option, } /// Errors produced by [`UnixDatagram`]. diff --git a/lading/src/generator/unix_stream.rs b/lading/src/generator/unix_stream.rs index 366f20662..fde1e162d 100644 --- a/lading/src/generator/unix_stream.rs +++ b/lading/src/generator/unix_stream.rs @@ -27,7 +27,7 @@ use tracing::{debug, error, info, warn}; use super::General; use crate::generator::common::{ - BytesThrottleConfig, MetricsBuilder, ThrottleConversionError, create_throttle, + MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, }; fn default_parallel_connections() -> u16 { @@ -59,7 +59,7 @@ pub struct Config { #[serde(default = "default_parallel_connections")] pub parallel_connections: u16, /// The load throttle configuration - pub throttle: Option, + pub throttle: Option, } /// Errors produced by [`UnixStream`]. From 8cd5308b0b415cc22589d9bada8466382f6ef385 Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Fri, 19 Dec 2025 15:02:35 -0500 Subject: [PATCH 08/21] more changes --- lading/src/generator/common.rs | 139 +++++-------------- lading/src/generator/file_gen/logrotate.rs | 7 +- lading/src/generator/file_gen/traditional.rs | 7 +- lading/src/generator/grpc.rs | 3 +- lading/src/generator/http.rs | 3 +- lading/src/generator/passthru_file.rs | 3 +- lading/src/generator/splunk_hec.rs | 3 +- lading/src/generator/tcp.rs | 2 +- lading/src/generator/trace_agent.rs | 3 +- lading/src/generator/udp.rs | 2 +- lading/src/generator/unix_datagram.rs | 3 +- lading/src/generator/unix_stream.rs | 2 +- 12 files changed, 55 insertions(+), 122 deletions(-) diff --git a/lading/src/generator/common.rs b/lading/src/generator/common.rs index aef4481ac..cb80ae193 100644 --- a/lading/src/generator/common.rs +++ b/lading/src/generator/common.rs @@ -71,7 +71,7 @@ pub enum ThrottleConfig { }, } -/// Error converting `BytesThrottleConfig` to internal throttle config +/// Error converting `ThrottleConfig` to internal throttle config #[derive(Debug, thiserror::Error, Clone, Copy)] pub enum ThrottleConversionError { /// Value exceeds u32 capacity @@ -106,7 +106,7 @@ pub enum ThrottleMode { #[derive(Debug)] pub(super) struct BlockThrottle { /// Underlying throttle instance. - pub throttle: lading_throttle::Throttle, + pub inner: lading_throttle::Throttle, /// Token interpretation mode. pub mode: ThrottleMode, } @@ -121,114 +121,41 @@ impl BlockThrottle { ThrottleMode::Bytes => block.total_bytes, ThrottleMode::Blocks => NonZeroU32::new(1).expect("non-zero"), }; - self.throttle.wait_for(tokens).await + self.inner.wait_for(tokens).await + } + + /// Divide the underlying throttle capacity by `n`, preserving mode. + pub(super) fn divide(self, n: NonZeroU32) -> Result { + let throttle = self.inner.divide(n)?; + Ok(Self { + inner: throttle, + mode: self.mode, + }) } } -/// Create a throttle from optional config and `bytes_per_second` fallback -/// -/// This function implements the standard throttle creation logic for -/// byte-oriented generators. It handles the interaction between the new -/// `BytesThrottleConfig` and the legacy `bytes_per_second` field. -/// -/// # Decision Logic -/// -/// | `BytesThrottleConfig` | `bytes_per_second` | Result | -/// |---------------------|------------------|--------| -/// | Some(config) | Some(bps) | Error - Conflicting configuration | -/// | Some(config) | None | Use `BytesThrottleConfig` | -/// | None | Some(bps) | Create Stable throttle with `timeout_micros`: 0 | -/// | None | None | `AllOut` throttle (no rate limiting) | -/// -/// # Errors +/// Create a throttle from config plus optional legacy bytes-per-second fallback. /// -/// Returns an error if: -/// - Both config and `bytes_per_second` are provided (conflicting configuration) -/// - The `bytes_per_second` value exceeds `u32::MAX` -/// - The `bytes_per_second` value is zero +/// Returns a [`BlockThrottle`] that carries both the throttle and its mode +/// (bytes vs blocks). pub(super) fn create_throttle( config: Option<&ThrottleConfig>, - bytes_per_second: Option<&byte_unit::Byte>, -) -> Result { - // Bytes-only helper for legacy callers. Reject block-mode usage here. - if let Some(ThrottleConfig::Stable { rate, .. }) = config { - if matches!( - rate.mode.unwrap_or(ThrottleMode::Bytes), - ThrottleMode::Blocks - ) { - return Err(ThrottleConversionError::ConflictingConfig); - } - } - if let Some(ThrottleConfig::Linear { - initial, - maximum, - rate_of_change, - }) = config - { - let modes = [ - initial.mode.unwrap_or(ThrottleMode::Bytes), - maximum.mode.unwrap_or(ThrottleMode::Bytes), - rate_of_change.mode.unwrap_or(ThrottleMode::Bytes), - ]; - if modes.iter().any(|m| *m == ThrottleMode::Blocks) { - return Err(ThrottleConversionError::ConflictingConfig); - } - } - - let throttle_config = match (config, bytes_per_second) { - (Some(_), Some(_)) => { - return Err(ThrottleConversionError::ConflictingConfig); - } - (Some(tc), None) => tc.try_into()?, - (None, Some(bps)) => { - let bps_value = bps.as_u128(); - if bps_value > u128::from(u32::MAX) { - return Err(ThrottleConversionError::ValueTooLarge(*bps)); - } - #[allow(clippy::cast_possible_truncation)] - let bps_u32 = NonZeroU32::new(bps_value as u32).ok_or(ThrottleConversionError::Zero)?; - lading_throttle::Config::Stable { - maximum_capacity: bps_u32, - timeout_micros: 0, - } - } - (None, None) => lading_throttle::Config::AllOut, - }; - Ok(lading_throttle::Throttle::new_with_config(throttle_config)) -} - -/// Create a throttle from a unified config plus optional legacy fallbacks. -pub(super) fn create_throughput_throttle( - profile: Option<&ThrottleConfig>, legacy_bytes_per_second: Option<&byte_unit::Byte>, - legacy_blocks_per_second: Option, ) -> Result { - let fallback = if let Some(bps) = legacy_bytes_per_second { - Some(ThrottleConfig::Stable { - rate: RateSpec { - mode: Some(ThrottleMode::Bytes), - bytes_per_second: Some(*bps), - blocks_per_second: None, - }, - timeout_millis: 0, - }) - } else if let Some(bps) = legacy_blocks_per_second { - Some(ThrottleConfig::Stable { - rate: RateSpec { - mode: Some(ThrottleMode::Blocks), - bytes_per_second: None, - blocks_per_second: Some(bps), - }, - timeout_millis: 0, - }) - } else { - None - }; - - let cfg = profile.copied().or(fallback); - let throttle_cfg = cfg.ok_or(ThrottleConversionError::MissingRate)?; - - let throttle = match throttle_cfg { + let fallback = legacy_bytes_per_second.map(|bps| ThrottleConfig::Stable { + rate: RateSpec { + mode: Some(ThrottleMode::Bytes), + bytes_per_second: Some(*bps), + blocks_per_second: None, + }, + timeout_millis: 0, + }); + + let cfg = config + .copied() + .or(fallback) + .unwrap_or(ThrottleConfig::AllOut); + let throttle = match cfg { ThrottleConfig::AllOut => { lading_throttle::Throttle::new_with_config(lading_throttle::Config::AllOut) } @@ -261,8 +188,7 @@ pub(super) fn create_throughput_throttle( } }; - // Mode from the first rate in the config (AllOut => default to bytes mode) - let mode = match throttle_cfg { + let mode = match cfg { ThrottleConfig::AllOut => ThrottleMode::Bytes, ThrottleConfig::Stable { rate, .. } => rate.resolve()?.0, ThrottleConfig::Linear { @@ -280,7 +206,10 @@ pub(super) fn create_throughput_throttle( } }; - Ok(BlockThrottle { throttle, mode }) + Ok(BlockThrottle { + inner: throttle, + mode, + }) } impl TryFrom<&ThrottleConfig> for lading_throttle::Config { diff --git a/lading/src/generator/file_gen/logrotate.rs b/lading/src/generator/file_gen/logrotate.rs index aa89878b0..dca045d3e 100644 --- a/lading/src/generator/file_gen/logrotate.rs +++ b/lading/src/generator/file_gen/logrotate.rs @@ -37,7 +37,7 @@ use lading_payload::block; use super::General; use crate::generator::common::{ BlockThrottle, MetricsBuilder, ThrottleConfig, ThrottleConversionError, ThrottleMode, - create_throughput_throttle, + create_throttle, }; /// An enum to allow us to determine what operation caused an IO errror as the @@ -223,8 +223,7 @@ impl Server { config.bytes_per_second.as_ref() } }; - let throughput_throttle = - create_throughput_throttle(config.load_profile.as_ref(), legacy_bps, None)?; + let throughput_throttle = create_throttle(config.load_profile.as_ref(), legacy_bps)?; let mut dir_path = config.root.clone(); let depth = rng.random_range(0..config.max_depth); @@ -335,7 +334,7 @@ impl Child { async fn spin(mut self) -> Result<(), Error> { let mut handle = self.block_cache.handle(); let buffer_capacity = match self.throttle.mode { - ThrottleMode::Bytes => self.throttle.throttle.maximum_capacity() as usize, + ThrottleMode::Bytes => self.throttle.inner.maximum_capacity() as usize, ThrottleMode::Blocks => self.block_cache.peek_next_size(&handle).get() as usize, }; let mut total_bytes_written: u64 = 0; diff --git a/lading/src/generator/file_gen/traditional.rs b/lading/src/generator/file_gen/traditional.rs index 67699ab71..e19462a48 100644 --- a/lading/src/generator/file_gen/traditional.rs +++ b/lading/src/generator/file_gen/traditional.rs @@ -39,7 +39,7 @@ use lading_payload::{self, block}; use super::General; use crate::generator::common::{ BlockThrottle, MetricsBuilder, ThrottleConfig, ThrottleConversionError, ThrottleMode, - create_throughput_throttle, + create_throttle, }; #[derive(thiserror::Error, Debug)] @@ -188,8 +188,7 @@ impl Server { config.bytes_per_second.as_ref() } }; - let throughput_throttle = - create_throughput_throttle(config.load_profile.as_ref(), legacy_bps, None)?; + let throughput_throttle = create_throttle(config.load_profile.as_ref(), legacy_bps)?; let block_cache = match config.block_cache_method { block::CacheMethod::Fixed => block::Cache::fixed_with_max_overhead( @@ -326,7 +325,7 @@ impl Child { } } let buffer_capacity = match self.throttle.mode { - ThrottleMode::Bytes => self.throttle.throttle.maximum_capacity() as usize, + ThrottleMode::Bytes => self.throttle.inner.maximum_capacity() as usize, ThrottleMode::Blocks => self.block_cache.peek_next_size(&handle).get() as usize, }; let mut fp = BufWriter::with_capacity( diff --git a/lading/src/generator/grpc.rs b/lading/src/generator/grpc.rs index 632e21df4..d5681cd48 100644 --- a/lading/src/generator/grpc.rs +++ b/lading/src/generator/grpc.rs @@ -200,7 +200,8 @@ impl Grpc { let mut rng = StdRng::from_seed(config.seed); let labels = MetricsBuilder::new("grpc").with_id(general.id).build(); - let throttle = create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; + let throttle = + create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?.inner; let maximum_prebuild_cache_size_bytes = NonZeroU32::new(config.maximum_prebuild_cache_size_bytes.as_u128() as u32) diff --git a/lading/src/generator/http.rs b/lading/src/generator/http.rs index a4f08c0b1..19a4943ad 100644 --- a/lading/src/generator/http.rs +++ b/lading/src/generator/http.rs @@ -149,7 +149,8 @@ impl Http { let labels = MetricsBuilder::new("http").with_id(general.id).build(); - let throttle = create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; + let throttle = + create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?.inner; match config.method { Method::Post { diff --git a/lading/src/generator/passthru_file.rs b/lading/src/generator/passthru_file.rs index 2d5fa3671..11d7b1c5f 100644 --- a/lading/src/generator/passthru_file.rs +++ b/lading/src/generator/passthru_file.rs @@ -102,7 +102,8 @@ impl PassthruFile { .with_id(general.id) .build(); - let throttle = create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; + let throttle = + create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?.inner; if let Some(bytes_per_second) = config.bytes_per_second { gauge!("bytes_per_second", &labels).set(bytes_per_second.as_u128() as f64 / 1000.0); diff --git a/lading/src/generator/splunk_hec.rs b/lading/src/generator/splunk_hec.rs index 7be189974..7f90f6f90 100644 --- a/lading/src/generator/splunk_hec.rs +++ b/lading/src/generator/splunk_hec.rs @@ -202,7 +202,8 @@ impl SplunkHec { .with_id(general.id) .build(); - let throttle = create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; + let throttle = + create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?.inner; let uri = get_uri_by_format(&config.target_uri, config.format)?; diff --git a/lading/src/generator/tcp.rs b/lading/src/generator/tcp.rs index 3e862db6a..2ec049f01 100644 --- a/lading/src/generator/tcp.rs +++ b/lading/src/generator/tcp.rs @@ -178,7 +178,7 @@ impl Tcp { let worker = TcpWorker { addr, - throttle, + throttle: throttle.inner, block_cache: Arc::clone(&block_cache), metric_labels: worker_labels, shutdown: shutdown.clone(), diff --git a/lading/src/generator/trace_agent.rs b/lading/src/generator/trace_agent.rs index f611530c1..7db59213f 100644 --- a/lading/src/generator/trace_agent.rs +++ b/lading/src/generator/trace_agent.rs @@ -275,7 +275,8 @@ impl TraceAgent { .with_id(general.id) .build(); - let throttle = create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; + let throttle = + create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?.inner; let maximum_prebuild_cache_size_bytes = validate_cache_size(config.maximum_prebuild_cache_size_bytes)?; diff --git a/lading/src/generator/udp.rs b/lading/src/generator/udp.rs index 517f3a6d3..25a4aa027 100644 --- a/lading/src/generator/udp.rs +++ b/lading/src/generator/udp.rs @@ -181,7 +181,7 @@ impl Udp { let worker = UdpWorker { addr, - throttle, + throttle: throttle.inner, block_cache: Arc::clone(&block_cache), metric_labels: worker_labels, shutdown: shutdown.clone(), diff --git a/lading/src/generator/unix_datagram.rs b/lading/src/generator/unix_datagram.rs index ba9f69136..3a9daba42 100644 --- a/lading/src/generator/unix_datagram.rs +++ b/lading/src/generator/unix_datagram.rs @@ -186,7 +186,8 @@ impl UnixDatagram { let mut handles = Vec::new(); for _ in 0..config.parallel_connections { let throttle = - create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; + create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())? + .inner; let child = Child { path: config.path.clone(), diff --git a/lading/src/generator/unix_stream.rs b/lading/src/generator/unix_stream.rs index fde1e162d..5a91290b2 100644 --- a/lading/src/generator/unix_stream.rs +++ b/lading/src/generator/unix_stream.rs @@ -161,7 +161,7 @@ impl UnixStream { let mut handles = JoinSet::new(); for _ in 0..config.parallel_connections { let throttle = - create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; + create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?.inner; let total_bytes = NonZeroU32::new(config.maximum_prebuild_cache_size_bytes.as_u128() as u32) From 72899075fd92323c62217efd9c287a890250b0b1 Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Fri, 19 Dec 2025 16:27:49 -0500 Subject: [PATCH 09/21] more changes --- lading/src/generator/common.rs | 13 +- lading/src/generator/file_gen/logrotate.rs | 8 +- lading/src/generator/file_gen/logrotate_fs.rs | 135 +++++++++++------- lading/src/generator/file_gen/traditional.rs | 13 +- lading/src/generator/grpc.rs | 11 +- lading/src/generator/http.rs | 12 +- lading/src/generator/passthru_file.rs | 11 +- lading/src/generator/splunk_hec.rs | 11 +- lading/src/generator/tcp.rs | 11 +- lading/src/generator/trace_agent.rs | 12 +- lading/src/generator/udp.rs | 12 +- lading/src/generator/unix_datagram.rs | 13 +- lading/src/generator/unix_stream.rs | 16 +-- 13 files changed, 158 insertions(+), 120 deletions(-) diff --git a/lading/src/generator/common.rs b/lading/src/generator/common.rs index cb80ae193..925f5def2 100644 --- a/lading/src/generator/common.rs +++ b/lading/src/generator/common.rs @@ -1,7 +1,6 @@ //! Common types for generators use byte_unit::Byte; -use lading_payload::block::Block; use serde::{Deserialize, Serialize}; use std::num::{NonZeroU16, NonZeroU32}; @@ -106,7 +105,7 @@ pub enum ThrottleMode { #[derive(Debug)] pub(super) struct BlockThrottle { /// Underlying throttle instance. - pub inner: lading_throttle::Throttle, + inner: lading_throttle::Throttle, /// Token interpretation mode. pub mode: ThrottleMode, } @@ -115,10 +114,11 @@ impl BlockThrottle { /// Wait for capacity for a block, interpreting tokens according to `mode`. pub(super) async fn wait_for_block( &mut self, - block: &Block, + block_cache: &lading_payload::block::Cache, + handle: &lading_payload::block::Handle, ) -> Result<(), lading_throttle::Error> { let tokens: NonZeroU32 = match self.mode { - ThrottleMode::Bytes => block.total_bytes, + ThrottleMode::Bytes => block_cache.peek_next_size(handle), ThrottleMode::Blocks => NonZeroU32::new(1).expect("non-zero"), }; self.inner.wait_for(tokens).await @@ -132,6 +132,11 @@ impl BlockThrottle { mode: self.mode, }) } + + /// Get the maximum capacity of the underlying throttle + pub(super) fn maximum_capacity(&self) -> u32 { + self.inner.maximum_capacity() + } } /// Create a throttle from config plus optional legacy bytes-per-second fallback. diff --git a/lading/src/generator/file_gen/logrotate.rs b/lading/src/generator/file_gen/logrotate.rs index dca045d3e..8e3f3a981 100644 --- a/lading/src/generator/file_gen/logrotate.rs +++ b/lading/src/generator/file_gen/logrotate.rs @@ -374,14 +374,11 @@ impl Child { loop { // SAFETY: By construction the block cache will never be empty // except in the event of a catastrophic failure. - let block = self.block_cache.advance(&mut handle); - let total_bytes = u64::from(block.total_bytes.get()); - tokio::select! { - result = self.throttle.wait_for_block(block) => { + result = self.throttle.wait_for_block(&self.block_cache, &handle) => { match result { Ok(()) => { - write_bytes(block, + write_bytes(self.block_cache.advance(&mut handle), &mut fp, &mut total_bytes_written, buffer_capacity, @@ -392,6 +389,7 @@ impl Child { &self.labels).await?; } Err(err) => { + let total_bytes = self.block_cache.peek_next_size(&handle); error!( "Throttle request for block size {total_bytes} failed. Block will be discarded. Error: {err}" ); diff --git a/lading/src/generator/file_gen/logrotate_fs.rs b/lading/src/generator/file_gen/logrotate_fs.rs index ac63a6c04..eeea6950b 100644 --- a/lading/src/generator/file_gen/logrotate_fs.rs +++ b/lading/src/generator/file_gen/logrotate_fs.rs @@ -6,7 +6,7 @@ use crate::generator; use crate::generator::common::{ - BytesThrottleConfig, RateSpec, ThrottleConversionError, ThrottleMode, + RateSpec, ThrottleConfig, ThrottleConversionError, ThrottleMode, }; use fuser::{ BackgroundSession, FileAttr, Filesystem, MountOption, ReplyAttr, ReplyData, ReplyDirectory, @@ -60,14 +60,10 @@ pub struct Config { mount_point: PathBuf, /// The load profile, controlling bytes per second as a function of time. load_profile: LoadProfile, - /// Optional blocks-per-second throttle. When set, overrides `load_profile` - /// to a constant rate derived from the average block size. + /// Optional throttle profile (bytes or blocks). When set, overrides + /// `load_profile` and derives bytes via average block size. #[serde(default)] - blocks_per_second: Option, - /// Unified throttle profile (bytes or blocks). If set to blocks, overrides - /// `load_profile` / `blocks_per_second` and derives bytes via average block size. - #[serde(default)] - pub load_throttle: Option, + pub throttle: Option, } /// Profile for load in this filesystem. @@ -112,6 +108,87 @@ impl LoadProfile { } } +fn resolve_rate(rate: &RateSpec) -> Result<(ThrottleMode, NonZeroU32), ThrottleConversionError> { + let mode = rate.mode.unwrap_or(ThrottleMode::Bytes); + match mode { + ThrottleMode::Bytes => { + let bps = rate + .bytes_per_second + .ok_or(ThrottleConversionError::MissingRate)?; + let val = bps.as_u128(); + if val > u128::from(u32::MAX) { + return Err(ThrottleConversionError::ValueTooLarge(bps)); + } + NonZeroU32::new(val as u32) + .map(|n| (ThrottleMode::Bytes, n)) + .ok_or(ThrottleConversionError::Zero) + } + ThrottleMode::Blocks => rate + .blocks_per_second + .map(|n| (ThrottleMode::Blocks, n)) + .ok_or(ThrottleConversionError::MissingRate), + } +} + +fn rate_to_bytes( + mode: ThrottleMode, + cap: NonZeroU32, + average_block_size: u64, +) -> u64 { + match mode { + ThrottleMode::Bytes => u64::from(cap.get()), + ThrottleMode::Blocks => average_block_size + .saturating_mul(u64::from(cap.get())) + .max(1), + } +} + +fn load_profile_from_throttle( + throttle: &ThrottleConfig, + average_block_size: u64, +) -> Result { + match throttle { + ThrottleConfig::AllOut => Ok(LoadProfile::Blocks { + blocks_per_second: NonZeroU32::new(1).unwrap(), + }), + ThrottleConfig::Stable { rate, .. } => { + let (mode, cap) = resolve_rate(rate)?; + if mode == ThrottleMode::Blocks { + let bytes = rate_to_bytes(mode, cap, average_block_size); + info!( + blocks_per_second = cap.get(), + average_block_size, + derived_bytes_per_second = bytes, + "logrotate_fs using block-based throttle derived from average block size" + ); + Ok(LoadProfile::Constant(byte_unit::Byte::from_u64(bytes))) + } else { + Ok(LoadProfile::Constant(byte_unit::Byte::from_u64( + rate_to_bytes(mode, cap, average_block_size), + ))) + } + } + ThrottleConfig::Linear { + initial, + maximum, + rate_of_change, + } => { + let (m1, init) = resolve_rate(initial)?; + let (m2, _max) = resolve_rate(maximum)?; + let (m3, rate) = resolve_rate(rate_of_change)?; + if m1 != m2 || m1 != m3 { + return Err(ThrottleConversionError::MixedModes); + } + let init_bytes = rate_to_bytes(m1, init, average_block_size); + let rate_bytes = rate_to_bytes(m1, rate, average_block_size); + Ok(LoadProfile::Linear { + initial_bytes_per_second: byte_unit::Byte::from_u64(init_bytes), + rate: byte_unit::Byte::from_u64(rate_bytes), + }) + } + } +} + #[derive(thiserror::Error, Debug)] /// Error for `LogrotateFs` pub enum Error { @@ -187,46 +264,8 @@ impl Server { block_cache.total_size().saturating_div(len).max(1) } }; - let load_profile = if let Some(throttle) = &config.load_throttle { - match throttle { - BytesThrottleConfig::Stable { rate, .. } => { - let (mode, cap) = rate.resolve()?; - match mode { - ThrottleMode::Bytes => { - LoadProfile::Constant(byte_unit::Byte::from_u64(cap.get().into())) - } - ThrottleMode::Blocks => { - let bytes = average_block_size - .saturating_mul(u64::from(cap.get())) - .max(1); - info!( - blocks_per_second = cap.get(), - average_block_size, - derived_bytes_per_second = bytes, - "logrotate_fs using block-based throttle derived from average block size" - ); - LoadProfile::Constant(byte_unit::Byte::from_u64(bytes)) - } - } - } - BytesThrottleConfig::AllOut => LoadProfile::Blocks { - blocks_per_second: NonZeroU32::new(1).unwrap(), - }, - BytesThrottleConfig::Linear { .. } => { - return Err(Error::ThrottleConversion( - ThrottleConversionError::MixedModes, - )); - } - } - } else if let Some(blocks_per_second) = config.blocks_per_second { - let bytes = average_block_size.saturating_mul(u64::from(blocks_per_second.get())); - info!( - blocks_per_second = blocks_per_second.get(), - average_block_size, - derived_bytes_per_second = bytes, - "logrotate_fs using block-based throttle derived from average block size" - ); - LoadProfile::Constant(byte_unit::Byte::from_u64(bytes)) + let load_profile = if let Some(throttle) = &config.throttle { + load_profile_from_throttle(throttle, average_block_size)? } else { config.load_profile }; diff --git a/lading/src/generator/file_gen/traditional.rs b/lading/src/generator/file_gen/traditional.rs index e19462a48..03692c74f 100644 --- a/lading/src/generator/file_gen/traditional.rs +++ b/lading/src/generator/file_gen/traditional.rs @@ -290,6 +290,7 @@ struct Child { maximum_bytes_per_file: NonZeroU32, throttle: BlockThrottle, block_cache: Arc, + maximum_block_size: u64, rotate: bool, file_index: Arc, shutdown: lading_signal::Watcher, @@ -325,8 +326,8 @@ impl Child { } } let buffer_capacity = match self.throttle.mode { - ThrottleMode::Bytes => self.throttle.inner.maximum_capacity() as usize, - ThrottleMode::Blocks => self.block_cache.peek_next_size(&handle).get() as usize, + ThrottleMode::Bytes => self.throttle.maximum_capacity() as usize, + ThrottleMode::Blocks => self.maximum_block_size as usize, }; let mut fp = BufWriter::with_capacity( buffer_capacity, @@ -353,13 +354,12 @@ impl Child { .as_ref() .map(|dur| Instant::now() + *dur); loop { - let block = self.block_cache.advance(&mut handle); - let total_bytes = u64::from(block.total_bytes.get()); - tokio::select! { - result = self.throttle.wait_for_block(block) => { + result = self.throttle.wait_for_block(&self.block_cache, &handle) => { match result { Ok(()) => { + let block = self.block_cache.advance(&mut handle); + let total_bytes = u64::from(block.total_bytes.get()); if let Some(dur) = self.block_interval { if let Some(deadline) = next_tick { tokio::select! { @@ -417,6 +417,7 @@ impl Child { } } Err(err) => { + let total_bytes = self.block_cache.peek_next_size(&handle); error!( "Throttle request for block size {total_bytes} failed. Block will be discarded. Error: {err}" ); diff --git a/lading/src/generator/grpc.rs b/lading/src/generator/grpc.rs index d5681cd48..ac02d0556 100644 --- a/lading/src/generator/grpc.rs +++ b/lading/src/generator/grpc.rs @@ -36,7 +36,7 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, + BlockThrottle, MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, }; /// Errors produced by [`Grpc`] @@ -175,7 +175,7 @@ pub struct Grpc { target_uri: Uri, rpc_path: PathAndQuery, shutdown: lading_signal::Watcher, - throttle: lading_throttle::Throttle, + throttle: BlockThrottle, block_cache: block::Cache, metric_labels: Vec<(String, String)>, } @@ -201,7 +201,7 @@ impl Grpc { let labels = MetricsBuilder::new("grpc").with_id(general.id).build(); let throttle = - create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?.inner; + create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; let maximum_prebuild_cache_size_bytes = NonZeroU32::new(config.maximum_prebuild_cache_size_bytes.as_u128() as u32) @@ -305,10 +305,9 @@ impl Grpc { let shutdown_wait = self.shutdown.recv(); tokio::pin!(shutdown_wait); loop { - let total_bytes = self.block_cache.peek_next_size(&handle); - tokio::select! { - _ = self.throttle.wait_for(total_bytes) => { + result = self.throttle.wait_for_block(&self.block_cache, &handle) => { + let _ = result; let block = self.block_cache.advance(&mut handle); let block_length = block.bytes.len(); counter!("requests_sent", &self.metric_labels).increment(1); diff --git a/lading/src/generator/http.rs b/lading/src/generator/http.rs index 19a4943ad..889f13361 100644 --- a/lading/src/generator/http.rs +++ b/lading/src/generator/http.rs @@ -30,7 +30,8 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - ConcurrencyStrategy, MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, + BlockThrottle, ConcurrencyStrategy, MetricsBuilder, ThrottleConfig, ThrottleConversionError, + create_throttle, }; static CONNECTION_SEMAPHORE: OnceCell = OnceCell::new(); @@ -122,7 +123,7 @@ pub struct Http { method: hyper::Method, headers: hyper::HeaderMap, concurrency: ConcurrencyStrategy, - throttle: lading_throttle::Throttle, + throttle: BlockThrottle, block_cache: Arc, metric_labels: Vec<(String, String)>, shutdown: lading_signal::Watcher, @@ -149,8 +150,7 @@ impl Http { let labels = MetricsBuilder::new("http").with_id(general.id).build(); - let throttle = - create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?.inner; + let throttle = create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; match config.method { Method::Post { @@ -225,9 +225,8 @@ impl Http { let shutdown_wait = self.shutdown.recv(); tokio::pin!(shutdown_wait); loop { - let total_bytes = self.block_cache.peek_next_size(&handle); tokio::select! { - result = self.throttle.wait_for(total_bytes) => { + result = self.throttle.wait_for_block(&self.block_cache, &handle) => { match result { Ok(()) => { let block = self.block_cache.advance(&mut handle); @@ -285,6 +284,7 @@ impl Http { } Err(err) => { + let total_bytes: std::num::NonZero = self.block_cache.peek_next_size(&handle); error!("Throttle request of {total_bytes} is larger than throttle capacity. Block will be discarded. Error: {err}"); } } diff --git a/lading/src/generator/passthru_file.rs b/lading/src/generator/passthru_file.rs index 11d7b1c5f..c5512157e 100644 --- a/lading/src/generator/passthru_file.rs +++ b/lading/src/generator/passthru_file.rs @@ -14,7 +14,6 @@ use std::{num::NonZeroU32, path::PathBuf, time::Duration}; use tokio::{fs, io::AsyncWriteExt}; use byte_unit::Byte; -use lading_throttle::Throttle; use metrics::{counter, gauge}; use rand::{SeedableRng, rngs::StdRng}; use serde::{Deserialize, Serialize}; @@ -24,7 +23,7 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, + BlockThrottle, MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, }; #[derive(Debug, Deserialize, Serialize, PartialEq, Clone)] @@ -74,7 +73,7 @@ pub enum Error { /// This generator is responsible for sending data to a file on disk. pub struct PassthruFile { path: PathBuf, - throttle: Throttle, + throttle: BlockThrottle, block_cache: block::Cache, metric_labels: Vec<(String, String)>, shutdown: lading_signal::Watcher, @@ -103,7 +102,7 @@ impl PassthruFile { .build(); let throttle = - create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?.inner; + create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; if let Some(bytes_per_second) = config.bytes_per_second { gauge!("bytes_per_second", &labels).set(bytes_per_second.as_u128() as f64 / 1000.0); @@ -184,9 +183,9 @@ impl PassthruFile { continue; }; - let total_bytes = self.block_cache.peek_next_size(&handle); tokio::select! { - _ = self.throttle.wait_for(total_bytes) => { + result = self.throttle.wait_for_block(&self.block_cache, &handle) => { + let _ = result; let block = self.block_cache.advance(&mut handle); match current_file.write_all(&block.bytes).await { Ok(()) => { diff --git a/lading/src/generator/splunk_hec.rs b/lading/src/generator/splunk_hec.rs index 7f90f6f90..2e8c8d2ba 100644 --- a/lading/src/generator/splunk_hec.rs +++ b/lading/src/generator/splunk_hec.rs @@ -45,7 +45,7 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, + BlockThrottle, MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, }; static CONNECTION_SEMAPHORE: OnceCell = OnceCell::new(); @@ -152,7 +152,7 @@ pub struct SplunkHec { uri: Uri, token: String, parallel_connections: u16, - throttle: lading_throttle::Throttle, + throttle: BlockThrottle, block_cache: Arc, metric_labels: Vec<(String, String)>, channels: Channels, @@ -202,8 +202,7 @@ impl SplunkHec { .with_id(general.id) .build(); - let throttle = - create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?.inner; + let throttle = create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; let uri = get_uri_by_format(&config.target_uri, config.format)?; @@ -289,10 +288,9 @@ impl SplunkHec { .next() .expect("channel should never be empty") .clone(); - let total_bytes = self.block_cache.peek_next_size(&handle); tokio::select! { - result = self.throttle.wait_for(total_bytes) => { + result = self.throttle.wait_for_block(&self.block_cache, &handle) => { match result { Ok(()) => { let client = client.clone(); @@ -320,6 +318,7 @@ impl SplunkHec { tokio::spawn(send_hec_request(permit, block_length, labels, channel, client, request, request_shutdown.clone(), uri_clone)); } Err(err) => { + let total_bytes = self.block_cache.peek_next_size(&handle); error!("Throttle request of {total_bytes} is larger than throttle capacity. Block will be discarded. Error: {err}"); } } diff --git a/lading/src/generator/tcp.rs b/lading/src/generator/tcp.rs index 2ec049f01..432e08193 100644 --- a/lading/src/generator/tcp.rs +++ b/lading/src/generator/tcp.rs @@ -32,7 +32,8 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - ConcurrencyStrategy, MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, + BlockThrottle, ConcurrencyStrategy, MetricsBuilder, ThrottleConfig, ThrottleConversionError, + create_throttle, }; fn default_parallel_connections() -> u16 { @@ -178,7 +179,7 @@ impl Tcp { let worker = TcpWorker { addr, - throttle: throttle.inner, + throttle, block_cache: Arc::clone(&block_cache), metric_labels: worker_labels, shutdown: shutdown.clone(), @@ -215,7 +216,7 @@ impl Tcp { struct TcpWorker { addr: SocketAddr, - throttle: lading_throttle::Throttle, + throttle: BlockThrottle, block_cache: Arc, metric_labels: Vec<(String, String)>, shutdown: lading_signal::Watcher, @@ -249,9 +250,8 @@ impl TcpWorker { continue; }; - let total_bytes = self.block_cache.peek_next_size(&handle); tokio::select! { - result = self.throttle.wait_for(total_bytes) => { + result = self.throttle.wait_for_block(&self.block_cache, &handle) => { match result { Ok(()) => { let block = self.block_cache.advance(&mut handle); @@ -271,6 +271,7 @@ impl TcpWorker { } } Err(err) => { + let total_bytes: std::num::NonZero = self.block_cache.peek_next_size(&handle); error!("Throttle request of {total_bytes} is larger than throttle capacity. Block will be discarded. Error: {err}"); } } diff --git a/lading/src/generator/trace_agent.rs b/lading/src/generator/trace_agent.rs index 7db59213f..178b7617b 100644 --- a/lading/src/generator/trace_agent.rs +++ b/lading/src/generator/trace_agent.rs @@ -17,7 +17,8 @@ //! Additional metrics may be emitted by this generator's [throttle]. use super::General; use crate::generator::common::{ - ConcurrencyStrategy, MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, + BlockThrottle, ConcurrencyStrategy, MetricsBuilder, ThrottleConfig, ThrottleConversionError, + create_throttle, }; use bytes::Bytes; use http_body_util::combinators::BoxBody; @@ -245,7 +246,7 @@ pub struct TraceAgent { trace_endpoint: Uri, backoff_behavior: BackoffBehavior, concurrency: ConcurrencyStrategy, - throttle: lading_throttle::Throttle, + throttle: BlockThrottle, block_cache: Arc, metric_labels: Vec<(String, String)>, shutdown: lading_signal::Watcher, @@ -275,8 +276,7 @@ impl TraceAgent { .with_id(general.id) .build(); - let throttle = - create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?.inner; + let throttle = create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; let maximum_prebuild_cache_size_bytes = validate_cache_size(config.maximum_prebuild_cache_size_bytes)?; @@ -353,9 +353,8 @@ impl TraceAgent { let shutdown_wait = self.shutdown.recv(); tokio::pin!(shutdown_wait); loop { - let total_bytes = self.block_cache.peek_next_size(&handle); tokio::select! { - result = self.throttle.wait_for(total_bytes) => { + result = self.throttle.wait_for_block(&self.block_cache, &handle) => { match result { Ok(()) => { let block = self.block_cache.advance(&mut handle); @@ -378,6 +377,7 @@ impl TraceAgent { }); } Err(err) => { + let total_bytes = self.block_cache.peek_next_size(&handle); error!("Throttle request of {total_bytes} is larger than throttle capacity. Block will be discarded. Error: {err}", total_bytes = total_bytes); } } diff --git a/lading/src/generator/udp.rs b/lading/src/generator/udp.rs index 25a4aa027..b34113a90 100644 --- a/lading/src/generator/udp.rs +++ b/lading/src/generator/udp.rs @@ -32,7 +32,8 @@ use lading_payload::block; use super::General; use crate::generator::common::{ - ConcurrencyStrategy, MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, + BlockThrottle, ConcurrencyStrategy, MetricsBuilder, ThrottleConfig, ThrottleConversionError, + create_throttle, }; fn default_parallel_connections() -> u16 { @@ -181,7 +182,7 @@ impl Udp { let worker = UdpWorker { addr, - throttle: throttle.inner, + throttle, block_cache: Arc::clone(&block_cache), metric_labels: worker_labels, shutdown: shutdown.clone(), @@ -218,7 +219,7 @@ impl Udp { struct UdpWorker { addr: SocketAddr, - throttle: lading_throttle::Throttle, + throttle: BlockThrottle, block_cache: Arc, metric_labels: Vec<(String, String)>, shutdown: lading_signal::Watcher, @@ -233,8 +234,6 @@ impl UdpWorker { let shutdown_wait = self.shutdown.recv(); tokio::pin!(shutdown_wait); loop { - let total_bytes = self.block_cache.peek_next_size(&handle); - tokio::select! { conn = UdpSocket::bind("127.0.0.1:0"), if connection.is_none() => { match conn { @@ -252,7 +251,7 @@ impl UdpWorker { } } } - result = self.throttle.wait_for(total_bytes), if connection.is_some() => { + result = self.throttle.wait_for_block(&self.block_cache, &handle), if connection.is_some() => { match result { Ok(()) => { let sock = connection.expect("connection failed"); @@ -274,6 +273,7 @@ impl UdpWorker { } } Err(err) => { + let total_bytes = self.block_cache.peek_next_size(&handle); error!("Throttle request of {total_bytes} is larger than throttle capacity. Block will be discarded. Error: {err}"); } } diff --git a/lading/src/generator/unix_datagram.rs b/lading/src/generator/unix_datagram.rs index 3a9daba42..1d410b102 100644 --- a/lading/src/generator/unix_datagram.rs +++ b/lading/src/generator/unix_datagram.rs @@ -14,7 +14,6 @@ use byte_unit::{Byte, Unit}; use futures::future::join_all; use lading_payload::block; -use lading_throttle::Throttle; use metrics::counter; use rand::{SeedableRng, rngs::StdRng}; use serde::{Deserialize, Serialize}; @@ -29,7 +28,7 @@ use tracing::{debug, error, info}; use super::General; use crate::generator::common::{ - MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, + BlockThrottle, MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, }; fn default_parallel_connections() -> u16 { @@ -186,8 +185,7 @@ impl UnixDatagram { let mut handles = Vec::new(); for _ in 0..config.parallel_connections { let throttle = - create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())? - .inner; + create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; let child = Child { path: config.path.clone(), @@ -234,7 +232,7 @@ impl UnixDatagram { #[derive(Debug)] struct Child { path: PathBuf, - throttle: Throttle, + throttle: BlockThrottle, block_cache: Arc, metric_labels: Vec<(String, String)>, shutdown: lading_signal::Watcher, @@ -270,10 +268,8 @@ impl Child { let shutdown_wait = self.shutdown.recv(); tokio::pin!(shutdown_wait); loop { - let total_bytes = self.block_cache.peek_next_size(&handle); - tokio::select! { - result = self.throttle.wait_for(total_bytes) => { + result = self.throttle.wait_for_block(&self.block_cache, &handle) => { match result { Ok(()) => { // NOTE When we write into a unix socket it may be that only @@ -299,6 +295,7 @@ impl Child { } } Err(err) => { + let total_bytes = self.block_cache.peek_next_size(&handle); error!("Throttle request of {total_bytes} is larger than throttle capacity. Block will be discarded. Error: {err}"); } } diff --git a/lading/src/generator/unix_stream.rs b/lading/src/generator/unix_stream.rs index 5a91290b2..34f7622a2 100644 --- a/lading/src/generator/unix_stream.rs +++ b/lading/src/generator/unix_stream.rs @@ -12,7 +12,6 @@ //! use lading_payload::block; -use lading_throttle::Throttle; use metrics::counter; use rand::{SeedableRng, rngs::StdRng}; use serde::{Deserialize, Serialize}; @@ -27,7 +26,7 @@ use tracing::{debug, error, info, warn}; use super::General; use crate::generator::common::{ - MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, + BlockThrottle, MetricsBuilder, ThrottleConfig, ThrottleConversionError, create_throttle, }; fn default_parallel_connections() -> u16 { @@ -161,7 +160,7 @@ impl UnixStream { let mut handles = JoinSet::new(); for _ in 0..config.parallel_connections { let throttle = - create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?.inner; + create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; let total_bytes = NonZeroU32::new(config.maximum_prebuild_cache_size_bytes.as_u128() as u32) @@ -229,7 +228,7 @@ impl UnixStream { #[derive(Debug)] struct Child { path: PathBuf, - throttle: Throttle, + throttle: BlockThrottle, block_cache: block::Cache, metric_labels: Vec<(String, String)>, shutdown: lading_signal::Watcher, @@ -268,10 +267,8 @@ impl Child { continue; }; - let total_bytes = self.block_cache.peek_next_size(&handle); - tokio::select! { - result = self.throttle.wait_for(total_bytes) => { + result = self.throttle.wait_for_block(&self.block_cache, &handle) => { match result { Ok(()) => { // NOTE When we write into a unix stream it may be that only @@ -321,7 +318,10 @@ impl Child { } } Err(err) => { - error!("Throttle request of {total_bytes} is larger than throttle capacity. Block will be discarded. Error: {err}"); + let total_bytes = self.block_cache.peek_next_size(&handle); + error!( + "Throttle request of {total_bytes} is larger than throttle capacity. Block will be discarded. Error: {err}" + ); } } } From c146cb0006615e11e17f77ec5a9cdf182471fa8b Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Fri, 19 Dec 2025 17:53:13 -0500 Subject: [PATCH 10/21] more changes --- lading/src/generator/file_gen/logrotate.rs | 8 ++++++-- lading/src/generator/file_gen/traditional.rs | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lading/src/generator/file_gen/logrotate.rs b/lading/src/generator/file_gen/logrotate.rs index 8e3f3a981..56b63224a 100644 --- a/lading/src/generator/file_gen/logrotate.rs +++ b/lading/src/generator/file_gen/logrotate.rs @@ -242,6 +242,7 @@ impl Server { &basename, config.total_rotations, maximum_bytes_per_log, + maximum_block_size.get(), Arc::clone(&block_cache), throughput_throttle, shutdown.clone(), @@ -291,6 +292,7 @@ struct Child { names: Vec, // The soft limit bytes per file that will trigger a rotation. maximum_bytes_per_log: NonZeroU32, + maximum_block_size: u32, block_cache: Arc, throttle: BlockThrottle, shutdown: lading_signal::Watcher, @@ -303,6 +305,7 @@ impl Child { basename: &Path, total_rotations: u8, maximum_bytes_per_log: NonZeroU32, + maximum_block_size: u32, block_cache: Arc, throttle: BlockThrottle, shutdown: lading_signal::Watcher, @@ -324,6 +327,7 @@ impl Child { Self { names, maximum_bytes_per_log, + maximum_block_size, block_cache, throttle, shutdown, @@ -334,8 +338,8 @@ impl Child { async fn spin(mut self) -> Result<(), Error> { let mut handle = self.block_cache.handle(); let buffer_capacity = match self.throttle.mode { - ThrottleMode::Bytes => self.throttle.inner.maximum_capacity() as usize, - ThrottleMode::Blocks => self.block_cache.peek_next_size(&handle).get() as usize, + ThrottleMode::Bytes => self.throttle.maximum_capacity() as usize, + ThrottleMode::Blocks => self.maximum_block_size as usize, }; let mut total_bytes_written: u64 = 0; let maximum_bytes_per_log: u64 = u64::from(self.maximum_bytes_per_log.get()); diff --git a/lading/src/generator/file_gen/traditional.rs b/lading/src/generator/file_gen/traditional.rs index 03692c74f..64fd09b03 100644 --- a/lading/src/generator/file_gen/traditional.rs +++ b/lading/src/generator/file_gen/traditional.rs @@ -213,6 +213,7 @@ impl Server { maximum_bytes_per_file, throttle: throughput_throttle, block_cache: Arc::new(block_cache), + maximum_block_size: maximum_block_size as u64, file_index: Arc::clone(&file_index), rotate: config.rotate, shutdown: shutdown.clone(), From 208f1443644c115c0d8ddc1b5a3693995d84dc0a Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Mon, 22 Dec 2025 10:33:41 -0500 Subject: [PATCH 11/21] logrotatefs1 --- examples/lading-logrotatefs.yaml | 4 +- lading/src/generator/file_gen/logrotate_fs.rs | 223 ++++++++++++------ .../generator/file_gen/logrotate_fs/model.rs | 126 ++++++++-- lading_payload/src/block.rs | 2 +- 4 files changed, 252 insertions(+), 103 deletions(-) diff --git a/examples/lading-logrotatefs.yaml b/examples/lading-logrotatefs.yaml index 08a1da16a..adafa2130 100644 --- a/examples/lading-logrotatefs.yaml +++ b/examples/lading-logrotatefs.yaml @@ -9,7 +9,9 @@ generator: max_depth: 0 variant: "ascii" load_profile: - constant: 1.3MiB + constant: + rate: + bytes_per_second: 1.3MiB maximum_prebuild_cache_size_bytes: 1GiB mount_point: /tmp/logrotate diff --git a/lading/src/generator/file_gen/logrotate_fs.rs b/lading/src/generator/file_gen/logrotate_fs.rs index eeea6950b..957e1c0a2 100644 --- a/lading/src/generator/file_gen/logrotate_fs.rs +++ b/lading/src/generator/file_gen/logrotate_fs.rs @@ -5,9 +5,7 @@ #![allow(clippy::cast_possible_wrap)] use crate::generator; -use crate::generator::common::{ - RateSpec, ThrottleConfig, ThrottleConversionError, ThrottleMode, -}; +use crate::generator::common::{RateSpec, ThrottleConfig, ThrottleConversionError, ThrottleMode}; use fuser::{ BackgroundSession, FileAttr, Filesystem, MountOption, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry, Request, spawn_mount2, @@ -16,7 +14,7 @@ use lading_payload::block; use metrics::counter; use nix::libc::{self, ENOENT}; use rand::{SeedableRng, rngs::SmallRng}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use std::{ collections::HashMap, ffi::OsStr, @@ -58,51 +56,150 @@ pub struct Config { maximum_block_size: byte_unit::Byte, /// The mount-point for this filesystem mount_point: PathBuf, - /// The load profile, controlling bytes per second as a function of time. + /// The load profile, controlling bytes or blocks per second as a function of time. load_profile: LoadProfile, /// Optional throttle profile (bytes or blocks). When set, overrides - /// `load_profile` and derives bytes via average block size. + /// `load_profile`. #[serde(default)] pub throttle: Option, } /// Profile for load in this filesystem. -#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq)] +#[derive(Debug, Serialize, Clone, Copy, PartialEq)] #[serde(rename_all = "snake_case")] pub enum LoadProfile { - /// Constant bytes per second + /// Constant rate (bytes or blocks per second). + Constant { + /// Rate specification (bytes or blocks). + rate: RateSpec, + }, + /// Linear growth of rate (bytes or blocks per second). + Linear { + /// Starting point for the rate. + initial: RateSpec, + /// Amount to increase per second. + rate_of_change: RateSpec, + }, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "snake_case")] +enum LoadProfileWire { + Constant { rate: RateSpec }, + Linear { + initial: RateSpec, + rate_of_change: RateSpec, + }, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "snake_case")] +enum LegacyLoadProfile { Constant(byte_unit::Byte), - /// Linear growth of bytes per second Linear { - /// Starting point for bytes per second initial_bytes_per_second: byte_unit::Byte, - /// Amount to increase per second rate: byte_unit::Byte, }, - /// Constant blocks per second (derived to bytes via average block size). - Blocks { - /// Blocks per second - blocks_per_second: NonZeroU32, - }, + Blocks { blocks_per_second: NonZeroU32 }, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum LoadProfileCompat { + New(LoadProfileWire), + Legacy(LegacyLoadProfile), } impl LoadProfile { - fn to_model(self, average_block_size: u64) -> model::LoadProfile { - // For now, one tick is one second. - match self { - LoadProfile::Constant(bpt) => model::LoadProfile::Constant(bpt.as_u128() as u64), - LoadProfile::Linear { + fn from_legacy(profile: LegacyLoadProfile) -> Self { + match profile { + LegacyLoadProfile::Constant(bps) => LoadProfile::Constant { + rate: RateSpec { + mode: None, + bytes_per_second: Some(bps), + blocks_per_second: None, + }, + }, + LegacyLoadProfile::Linear { initial_bytes_per_second, rate, - } => model::LoadProfile::Linear { - start: initial_bytes_per_second.as_u128() as u64, - rate: rate.as_u128() as u64, + } => LoadProfile::Linear { + initial: RateSpec { + mode: None, + bytes_per_second: Some(initial_bytes_per_second), + blocks_per_second: None, + }, + rate_of_change: RateSpec { + mode: None, + bytes_per_second: Some(rate), + blocks_per_second: None, + }, + }, + LegacyLoadProfile::Blocks { blocks_per_second } => LoadProfile::Constant { + rate: RateSpec { + mode: Some(ThrottleMode::Blocks), + bytes_per_second: None, + blocks_per_second: Some(blocks_per_second), + }, + }, + } + } +} + +impl<'de> Deserialize<'de> for LoadProfile { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let compat = LoadProfileCompat::deserialize(deserializer)?; + Ok(match compat { + LoadProfileCompat::New(profile) => match profile { + LoadProfileWire::Constant { rate } => LoadProfile::Constant { rate }, + LoadProfileWire::Linear { + initial, + rate_of_change, + } => LoadProfile::Linear { + initial, + rate_of_change, + }, }, - LoadProfile::Blocks { blocks_per_second } => { - let bytes = average_block_size - .saturating_mul(u64::from(blocks_per_second.get())) - .max(1); - model::LoadProfile::Constant(bytes) + LoadProfileCompat::Legacy(profile) => LoadProfile::from_legacy(profile), + }) + } +} + +impl LoadProfile { + fn to_model(self) -> Result { + // For now, one tick is one second. + match self { + LoadProfile::Constant { rate } => { + let (mode, cap) = resolve_rate(&rate)?; + match mode { + ThrottleMode::Bytes => Ok(model::LoadProfile::Constant(u64::from(cap.get()))), + ThrottleMode::Blocks => Ok(model::LoadProfile::Blocks { + blocks_per_tick: u64::from(cap.get()), + }), + } + } + LoadProfile::Linear { + initial, + rate_of_change, + } => { + let (m1, init) = resolve_rate(&initial)?; + let (m2, rate) = resolve_rate(&rate_of_change)?; + if m1 != m2 { + return Err(ThrottleConversionError::MixedModes); + } + match m1 { + ThrottleMode::Bytes => Ok(model::LoadProfile::Linear { + start: u64::from(init.get()), + rate: u64::from(rate.get()), + }), + ThrottleMode::Blocks => Ok(model::LoadProfile::BlocksLinear { + start: u64::from(init.get()), + rate: u64::from(rate.get()), + }), + } } } } @@ -130,42 +227,18 @@ fn resolve_rate(rate: &RateSpec) -> Result<(ThrottleMode, NonZeroU32), ThrottleC } } -fn rate_to_bytes( - mode: ThrottleMode, - cap: NonZeroU32, - average_block_size: u64, -) -> u64 { - match mode { - ThrottleMode::Bytes => u64::from(cap.get()), - ThrottleMode::Blocks => average_block_size - .saturating_mul(u64::from(cap.get())) - .max(1), - } -} - fn load_profile_from_throttle( throttle: &ThrottleConfig, - average_block_size: u64, -) -> Result { +) -> Result { match throttle { - ThrottleConfig::AllOut => Ok(LoadProfile::Blocks { - blocks_per_second: NonZeroU32::new(1).unwrap(), - }), + ThrottleConfig::AllOut => Ok(model::LoadProfile::Blocks { blocks_per_tick: 1 }), ThrottleConfig::Stable { rate, .. } => { let (mode, cap) = resolve_rate(rate)?; - if mode == ThrottleMode::Blocks { - let bytes = rate_to_bytes(mode, cap, average_block_size); - info!( - blocks_per_second = cap.get(), - average_block_size, - derived_bytes_per_second = bytes, - "logrotate_fs using block-based throttle derived from average block size" - ); - Ok(LoadProfile::Constant(byte_unit::Byte::from_u64(bytes))) - } else { - Ok(LoadProfile::Constant(byte_unit::Byte::from_u64( - rate_to_bytes(mode, cap, average_block_size), - ))) + match mode { + ThrottleMode::Bytes => Ok(model::LoadProfile::Constant(u64::from(cap.get()))), + ThrottleMode::Blocks => Ok(model::LoadProfile::Blocks { + blocks_per_tick: u64::from(cap.get()), + }), } } ThrottleConfig::Linear { @@ -179,12 +252,16 @@ fn load_profile_from_throttle( if m1 != m2 || m1 != m3 { return Err(ThrottleConversionError::MixedModes); } - let init_bytes = rate_to_bytes(m1, init, average_block_size); - let rate_bytes = rate_to_bytes(m1, rate, average_block_size); - Ok(LoadProfile::Linear { - initial_bytes_per_second: byte_unit::Byte::from_u64(init_bytes), - rate: byte_unit::Byte::from_u64(rate_bytes), - }) + match m1 { + ThrottleMode::Bytes => Ok(model::LoadProfile::Linear { + start: u64::from(init.get()), + rate: u64::from(rate.get()), + }), + ThrottleMode::Blocks => Ok(model::LoadProfile::BlocksLinear { + start: u64::from(init.get()), + rate: u64::from(rate.get()), + }), + } } } } @@ -256,18 +333,10 @@ impl Server { // divvy this up in the future. total_bytes.get() as usize, )?; - let average_block_size = { - let len = block_cache.len() as u64; - if len == 0 { - 1 - } else { - block_cache.total_size().saturating_div(len).max(1) - } - }; let load_profile = if let Some(throttle) = &config.throttle { - load_profile_from_throttle(throttle, average_block_size)? + load_profile_from_throttle(throttle)? } else { - config.load_profile + config.load_profile.to_model()? }; let start_time = Instant::now(); @@ -281,7 +350,7 @@ impl Server { block_cache, config.max_depth, config.concurrent_logs, - load_profile.to_model(average_block_size), + load_profile, ); info!( diff --git a/lading/src/generator/file_gen/logrotate_fs/model.rs b/lading/src/generator/file_gen/logrotate_fs/model.rs index bc5c7d9d9..6b7be2b06 100644 --- a/lading/src/generator/file_gen/logrotate_fs/model.rs +++ b/lading/src/generator/file_gen/logrotate_fs/model.rs @@ -82,6 +82,9 @@ pub(crate) struct File { /// starting positions in the cache. cache_offset: u64, + /// Handle for iterating block sizes when simulating block-based writes. + block_handle: block::Handle, + /// The random number generator used to generate the cache offset. rng: SmallRng, } @@ -112,6 +115,21 @@ pub(crate) fn generate_cache_offset(rng: &mut SmallRng, total_cache_size: u64) - rng.random_range(0..total_cache_size) } +fn generate_block_handle(rng: &mut R, block_cache: &block::Cache) -> block::Handle +where + R: Rng + ?Sized, +{ + let mut handle = block_cache.handle(); + let len = block_cache.len(); + if len == 0 { + return handle; + } + let offset = rng.random_range(0..len); + for _ in 0..offset { + let _ = block_cache.advance(&mut handle); + } + handle +} impl File { /// Create a new instance of `File` pub(crate) fn new( @@ -122,6 +140,7 @@ impl File { now: Tick, peer: Option, total_cache_size: u64, + block_handle: block::Handle, ) -> Self { let cache_offset = generate_cache_offset(&mut rng, total_cache_size); Self { @@ -142,6 +161,7 @@ impl File { unlinked: false, max_offset_observed: 0, cache_offset, + block_handle, rng, } } @@ -295,6 +315,18 @@ pub(crate) enum LoadProfile { /// Amount to increase per tick rate: u64, }, + /// Constant blocks per tick + Blocks { + /// Blocks per tick + blocks_per_tick: u64, + }, + /// Linear growth of blocks per tick + BlocksLinear { + /// Starting point for blocks per tick + start: u64, + /// Amount to increase per tick + rate: u64, + }, } /// The state of the filesystem @@ -505,7 +537,8 @@ impl State { // Generate a new SmallRng instance from the states rng to be used in deterministic offset generation let child_seed: [u8; 32] = rng.random(); - let child_rng = SmallRng::from_seed(child_seed); + let mut child_rng = SmallRng::from_seed(child_seed); + let block_handle = generate_block_handle(&mut child_rng, &state.block_cache); let file = File::new( child_rng, @@ -515,6 +548,7 @@ impl State { state.now, None, state.total_cache_size, + block_handle, ); state.nodes.insert(file_inode, Node::File { file }); @@ -581,19 +615,32 @@ impl State { } } + fn blocks_to_bytes(&self, handle: &mut block::Handle, blocks_per_tick: u64) -> u64 { + if blocks_per_tick == 0 { + return 0; + } + let blocks_len = self.block_cache.len() as u64; + if blocks_len == 0 { + return 0; + } + let cycles = blocks_per_tick / blocks_len; + let remainder = blocks_per_tick % blocks_len; + let mut bytes = self.block_cache.total_size().saturating_mul(cycles); + for _ in 0..remainder { + let size = self.block_cache.peek_next_size(handle); + bytes = bytes.saturating_add(u64::from(size.get())); + let _ = self.block_cache.advance(handle); + } + bytes + } + #[inline] #[allow(clippy::too_many_lines)] fn advance_time_inner(&mut self, now: Tick) { assert!(now >= self.now); - // Compute new global bytes_per_tick, at now - 1. + // Compute new global throughput, at now - 1. let elapsed_ticks = now.saturating_sub(self.initial_tick).saturating_sub(1); - let bytes_per_tick = match &self.load_profile { - LoadProfile::Constant(bytes) => *bytes, - LoadProfile::Linear { start, rate } => { - start.saturating_add(rate.saturating_mul(elapsed_ticks)) - } - }; // Update each File's bytes_per_tick but do not advance time, as that is // done later. @@ -602,7 +649,20 @@ impl State { && !file.read_only && !file.unlinked { - file.bytes_per_tick = bytes_per_tick; + file.bytes_per_tick = match &self.load_profile { + LoadProfile::Constant(bytes) => *bytes, + LoadProfile::Linear { start, rate } => { + start.saturating_add(rate.saturating_mul(elapsed_ticks)) + } + LoadProfile::Blocks { blocks_per_tick } => { + self.blocks_to_bytes(&mut file.block_handle, *blocks_per_tick) + } + LoadProfile::BlocksLinear { start, rate } => { + let blocks_per_tick = + start.saturating_add(rate.saturating_mul(elapsed_ticks)); + self.blocks_to_bytes(&mut file.block_handle, blocks_per_tick) + } + }; } } @@ -611,7 +671,15 @@ impl State { } for inode in self.inode_scratch.drain(..) { - let (rotated_inode, parent_inode, group_id, ordinal, file_rng, cache_offset) = { + let ( + rotated_inode, + parent_inode, + group_id, + ordinal, + file_rng, + cache_offset, + bytes_per_tick, + ) = { // If the node pointed to by inode doesn't exist, that's a // catastrophic programming error. We just copied all inode to node // pairs. @@ -656,6 +724,7 @@ impl State { file.ordinal, file.rng.clone(), file.cache_offset, + file.bytes_per_tick, ) }; @@ -666,6 +735,8 @@ impl State { // Set bytes_per_tick to current and now to now-1 else we'll never // ramp properly. let new_file_inode = self.next_inode; + let mut file_rng = file_rng; + let block_handle = generate_block_handle(&mut file_rng, &self.block_cache); let mut new_file = File::new( file_rng, parent_inode, @@ -674,6 +745,7 @@ impl State { self.now.saturating_sub(1), Some(rotated_inode), self.total_cache_size, + block_handle, ); let new_file_cache_offset = new_file.cache_offset; @@ -1277,20 +1349,25 @@ mod test { } // Property 7: bytes_written are tick accurate - for (&inode, node) in &state.nodes { - if let Node::File { file } = node { - let end_tick = file.read_only_since.unwrap_or(state.now); - let expected_bytes = compute_expected_bytes_written( - &state.load_profile, - state.initial_tick, - file.created_tick, - end_tick, - ); - assert_eq!( - file.bytes_written, expected_bytes, - "bytes_written ({}) does not match expected_bytes_written ({expected_bytes}) for file with inode {inode}", - file.bytes_written, - ); + if !matches!( + state.load_profile, + LoadProfile::Blocks { .. } | LoadProfile::BlocksLinear { .. } + ) { + for (&inode, node) in &state.nodes { + if let Node::File { file } = node { + let end_tick = file.read_only_since.unwrap_or(state.now); + let expected_bytes = compute_expected_bytes_written( + &state.load_profile, + state.initial_tick, + file.created_tick, + end_tick, + ); + assert_eq!( + file.bytes_written, expected_bytes, + "bytes_written ({}) does not match expected_bytes_written ({expected_bytes}) for file with inode {inode}", + file.bytes_written, + ); + } } } @@ -1392,6 +1469,7 @@ mod test { .saturating_add(rate.saturating_mul(sum_of_terms)); total_bytes } + LoadProfile::Blocks { .. } | LoadProfile::BlocksLinear { .. } => 0, } } diff --git a/lading_payload/src/block.rs b/lading_payload/src/block.rs index 4a9b3fd4c..5c4d20382 100644 --- a/lading_payload/src/block.rs +++ b/lading_payload/src/block.rs @@ -188,7 +188,7 @@ pub enum Cache { /// Each independent consumer should create its own Handle by calling /// `Cache::handle()`. Handles maintain their own position in the cache /// and advance independently. -#[derive(Debug)] +#[derive(Debug, Clone)] #[allow(missing_copy_implementations)] // intentionally not Copy to force callers to call `handle`. pub struct Handle { idx: usize, From bab0b24881b8fde173f84b7179f8493b23e9b2fd Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Mon, 22 Dec 2025 10:54:27 -0500 Subject: [PATCH 12/21] cargo clippy/fmt --- lading/src/common.rs | 9 ++--- lading/src/generator/common.rs | 9 +++-- lading/src/generator/file_gen/logrotate_fs.rs | 8 +++-- .../generator/file_gen/logrotate_fs/model.rs | 36 ++++++++++++++----- lading/src/generator/file_gen/traditional.rs | 7 ++-- lading/src/generator/grpc.rs | 3 +- lading/src/generator/passthru_file.rs | 3 +- lading_payload/src/block.rs | 8 ++++- lading_payload/src/splunk_hec.rs | 9 ++--- lading_payload/src/statik_line_rate.rs | 2 +- lading_payload/src/statik_second.rs | 8 +++-- 11 files changed, 62 insertions(+), 40 deletions(-) diff --git a/lading/src/common.rs b/lading/src/common.rs index 0782fb2f5..c7d3d66e3 100644 --- a/lading/src/common.rs +++ b/lading/src/common.rs @@ -14,23 +14,18 @@ pub struct Output { pub stdout: Behavior, } -#[derive(Debug, Deserialize, Clone, PartialEq, Eq)] +#[derive(Debug, Deserialize, Clone, PartialEq, Eq, Default)] #[serde(deny_unknown_fields)] #[serde(untagged)] /// Defines the [`Output`] behavior for stderr and stdout. pub enum Behavior { /// Redirect stdout, stderr to /dev/null + #[default] Quiet, /// Write to a location on-disk. Log(PathBuf), } -impl Default for Behavior { - fn default() -> Self { - Self::Quiet - } -} - impl fmt::Display for Behavior { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { match self { diff --git a/lading/src/generator/common.rs b/lading/src/generator/common.rs index 925f5def2..8da9604fc 100644 --- a/lading/src/generator/common.rs +++ b/lading/src/generator/common.rs @@ -28,10 +28,9 @@ impl RateSpec { .bytes_per_second .ok_or(ThrottleConversionError::MissingRate)?; let val = bps.as_u128(); - if val > u128::from(u32::MAX) { - return Err(ThrottleConversionError::ValueTooLarge(bps)); - } - NonZeroU32::new(val as u32) + let val = + u32::try_from(val).map_err(|_| ThrottleConversionError::ValueTooLarge(bps))?; + NonZeroU32::new(val) .map(|n| (ThrottleMode::Bytes, n)) .ok_or(ThrottleConversionError::Zero) } @@ -298,7 +297,7 @@ impl ConcurrencyStrategy { } /// Get the number of parallel connections for this strategy - pub(super) fn connection_count(&self) -> u16 { + pub(super) fn connection_count(self) -> u16 { match self { Self::Pooled { max_connections } => max_connections.get(), Self::Workers { count } => count.get(), diff --git a/lading/src/generator/file_gen/logrotate_fs.rs b/lading/src/generator/file_gen/logrotate_fs.rs index 957e1c0a2..5c832eeb8 100644 --- a/lading/src/generator/file_gen/logrotate_fs.rs +++ b/lading/src/generator/file_gen/logrotate_fs.rs @@ -85,7 +85,9 @@ pub enum LoadProfile { #[derive(Debug, Deserialize)] #[serde(rename_all = "snake_case")] enum LoadProfileWire { - Constant { rate: RateSpec }, + Constant { + rate: RateSpec, + }, Linear { initial: RateSpec, rate_of_change: RateSpec, @@ -100,7 +102,9 @@ enum LegacyLoadProfile { initial_bytes_per_second: byte_unit::Byte, rate: byte_unit::Byte, }, - Blocks { blocks_per_second: NonZeroU32 }, + Blocks { + blocks_per_second: NonZeroU32, + }, } #[derive(Debug, Deserialize)] diff --git a/lading/src/generator/file_gen/logrotate_fs/model.rs b/lading/src/generator/file_gen/logrotate_fs/model.rs index 6b7be2b06..d4fa5cb30 100644 --- a/lading/src/generator/file_gen/logrotate_fs/model.rs +++ b/lading/src/generator/file_gen/logrotate_fs/model.rs @@ -615,21 +615,26 @@ impl State { } } - fn blocks_to_bytes(&self, handle: &mut block::Handle, blocks_per_tick: u64) -> u64 { + fn blocks_to_bytes( + block_cache: &block::Cache, + blocks_len: u64, + total_cache_size: u64, + handle: &mut block::Handle, + blocks_per_tick: u64, + ) -> u64 { if blocks_per_tick == 0 { return 0; } - let blocks_len = self.block_cache.len() as u64; if blocks_len == 0 { return 0; } let cycles = blocks_per_tick / blocks_len; let remainder = blocks_per_tick % blocks_len; - let mut bytes = self.block_cache.total_size().saturating_mul(cycles); + let mut bytes = total_cache_size.saturating_mul(cycles); for _ in 0..remainder { - let size = self.block_cache.peek_next_size(handle); + let size = block_cache.peek_next_size(handle); bytes = bytes.saturating_add(u64::from(size.get())); - let _ = self.block_cache.advance(handle); + let _ = block_cache.advance(handle); } bytes } @@ -642,6 +647,9 @@ impl State { // Compute new global throughput, at now - 1. let elapsed_ticks = now.saturating_sub(self.initial_tick).saturating_sub(1); + let block_cache = &self.block_cache; + let blocks_len = block_cache.len() as u64; + let total_cache_size = block_cache.total_size(); // Update each File's bytes_per_tick but do not advance time, as that is // done later. for node in self.nodes.values_mut() { @@ -654,13 +662,23 @@ impl State { LoadProfile::Linear { start, rate } => { start.saturating_add(rate.saturating_mul(elapsed_ticks)) } - LoadProfile::Blocks { blocks_per_tick } => { - self.blocks_to_bytes(&mut file.block_handle, *blocks_per_tick) - } + LoadProfile::Blocks { blocks_per_tick } => Self::blocks_to_bytes( + block_cache, + blocks_len, + total_cache_size, + &mut file.block_handle, + *blocks_per_tick, + ), LoadProfile::BlocksLinear { start, rate } => { let blocks_per_tick = start.saturating_add(rate.saturating_mul(elapsed_ticks)); - self.blocks_to_bytes(&mut file.block_handle, blocks_per_tick) + Self::blocks_to_bytes( + block_cache, + blocks_len, + total_cache_size, + &mut file.block_handle, + blocks_per_tick, + ) } }; } diff --git a/lading/src/generator/file_gen/traditional.rs b/lading/src/generator/file_gen/traditional.rs index 64fd09b03..b096ec645 100644 --- a/lading/src/generator/file_gen/traditional.rs +++ b/lading/src/generator/file_gen/traditional.rs @@ -301,6 +301,7 @@ struct Child { } impl Child { + #[allow(clippy::too_many_lines)] pub(crate) async fn spin(mut self) -> Result<(), Error> { let mut total_bytes_written: u64 = 0; let maximum_bytes_per_file: u64 = u64::from(self.maximum_bytes_per_file.get()); @@ -328,7 +329,9 @@ impl Child { } let buffer_capacity = match self.throttle.mode { ThrottleMode::Bytes => self.throttle.maximum_capacity() as usize, - ThrottleMode::Blocks => self.maximum_block_size as usize, + ThrottleMode::Blocks => { + usize::try_from(self.maximum_block_size).unwrap_or(usize::MAX) + } }; let mut fp = BufWriter::with_capacity( buffer_capacity, @@ -364,7 +367,7 @@ impl Child { if let Some(dur) = self.block_interval { if let Some(deadline) = next_tick { tokio::select! { - _ = tokio::time::sleep_until(deadline) => {}, + () = tokio::time::sleep_until(deadline) => {}, () = &mut shutdown_wait => { fp.flush().await?; info!("shutdown signal received"); diff --git a/lading/src/generator/grpc.rs b/lading/src/generator/grpc.rs index ac02d0556..12b449074 100644 --- a/lading/src/generator/grpc.rs +++ b/lading/src/generator/grpc.rs @@ -200,8 +200,7 @@ impl Grpc { let mut rng = StdRng::from_seed(config.seed); let labels = MetricsBuilder::new("grpc").with_id(general.id).build(); - let throttle = - create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; + let throttle = create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; let maximum_prebuild_cache_size_bytes = NonZeroU32::new(config.maximum_prebuild_cache_size_bytes.as_u128() as u32) diff --git a/lading/src/generator/passthru_file.rs b/lading/src/generator/passthru_file.rs index c5512157e..f46ecf88d 100644 --- a/lading/src/generator/passthru_file.rs +++ b/lading/src/generator/passthru_file.rs @@ -101,8 +101,7 @@ impl PassthruFile { .with_id(general.id) .build(); - let throttle = - create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; + let throttle = create_throttle(config.throttle.as_ref(), config.bytes_per_second.as_ref())?; if let Some(bytes_per_second) = config.bytes_per_second { gauge!("bytes_per_second", &labels).set(bytes_per_second.as_u128() as f64 / 1000.0); diff --git a/lading_payload/src/block.rs b/lading_payload/src/block.rs index 5c4d20382..6882397a8 100644 --- a/lading_payload/src/block.rs +++ b/lading_payload/src/block.rs @@ -391,7 +391,7 @@ impl Cache { let _guard = span.enter(); let mut serializer = crate::StaticSecond::new( static_path, - ×tamp_format, + timestamp_format, *emit_placeholder, *start_line_index, )?; @@ -482,6 +482,12 @@ impl Cache { } } + /// Returns true if the cache has no blocks. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + /// Get metadata of the next block without advancing. #[must_use] pub fn peek_next_metadata(&self, handle: &Handle) -> BlockMetadata { diff --git a/lading_payload/src/splunk_hec.rs b/lading_payload/src/splunk_hec.rs index d2d82e7d3..5b14811c8 100644 --- a/lading_payload/src/splunk_hec.rs +++ b/lading_payload/src/splunk_hec.rs @@ -118,7 +118,7 @@ impl Distribution for StandardUniform { } /// Encoding to be used -#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq, Default)] #[serde(deny_unknown_fields)] #[serde(rename_all = "snake_case")] #[cfg_attr(test, derive(proptest_derive::Arbitrary))] @@ -127,15 +127,10 @@ pub enum Encoding { /// Use text-encoded log messages Text, /// Use JSON-encoded log messages + #[default] Json, } -impl Default for Encoding { - fn default() -> Self { - Self::Json - } -} - #[derive(Debug, Default, Clone, Copy)] #[cfg_attr(test, derive(proptest_derive::Arbitrary))] /// Splunk's HEC diff --git a/lading_payload/src/statik_line_rate.rs b/lading_payload/src/statik_line_rate.rs index 92ad4ed4e..2240b6f55 100644 --- a/lading_payload/src/statik_line_rate.rs +++ b/lading_payload/src/statik_line_rate.rs @@ -34,7 +34,7 @@ pub enum Error { /// No lines were discovered in the provided path #[error("No lines found in static path")] NoLines, - /// The provided lines_per_second value was zero + /// The provided `lines_per_second` value was zero #[error("lines_per_second must be greater than zero")] ZeroLinesPerSecond, } diff --git a/lading_payload/src/statik_second.rs b/lading_payload/src/statik_second.rs index eb263cb27..667eddab3 100644 --- a/lading_payload/src/statik_second.rs +++ b/lading_payload/src/statik_second.rs @@ -50,6 +50,11 @@ impl StaticSecond { /// remainder of the message. `start_line_index`, when provided, skips that /// many lines (modulo the total number of available lines) before /// returning payloads. + /// + /// # Errors + /// + /// Returns an error if the file cannot be read, contains no lines, or a + /// timestamp fails to parse. pub fn new( path: &Path, timestamp_format: &str, @@ -144,9 +149,8 @@ impl StaticSecond { } if remaining >= len { remaining -= len; - continue; } else { - let cut = remaining as usize; + let cut = usize::try_from(remaining).unwrap_or(block.lines.len()); block.lines.drain(0..cut); start_idx = idx; break; From c20392506b1edba486e2bee28c5fd1e85c725e33 Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Mon, 22 Dec 2025 11:40:22 -0500 Subject: [PATCH 13/21] dockerfile changes --- Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8091d9695..5da2e6146 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,9 @@ -FROM docker.io/rust:1.90.0-bookworm AS chef -RUN cargo install cargo-chef -WORKDIR /app +# Update the rust version in-sync with the version in rust-toolchain.toml -FROM chef AS planner +# Stage 0: Planner - Extract dependency metadata +FROM docker.io/rust:1.90.0-slim-bookworm AS planner +WORKDIR /app +RUN cargo install cargo-chef --version 0.1.73 COPY . . RUN cargo chef prepare --recipe-path recipe.json From 7ce9a98b29eb63a6d9e066bc56d5636b63d9b013 Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Mon, 22 Dec 2025 13:13:07 -0500 Subject: [PATCH 14/21] dockerfile changes --- Dockerfile | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5da2e6146..9c26b7f8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,15 +7,6 @@ RUN cargo install cargo-chef --version 0.1.73 COPY . . RUN cargo chef prepare --recipe-path recipe.json -# Update the rust version in-sync with the version in rust-toolchain.toml - -# Stage 0: Planner - Extract dependency metadata -FROM docker.io/rust:1.90.0-slim-bookworm AS planner -WORKDIR /app -RUN cargo install cargo-chef --version 0.1.73 -COPY . . -RUN cargo chef prepare --recipe-path recipe.json - # Stage 1: Cacher - Build dependencies only FROM docker.io/rust:1.90.0-slim-bookworm AS cacher ARG SCCACHE_BUCKET From 398c0a6a1546b8ff572979592bf3e56d91269dd8 Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Mon, 22 Dec 2025 13:13:55 -0500 Subject: [PATCH 15/21] dockerfile changes --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9c26b7f8f..54893cefb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,9 +26,9 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* # Download pre-built sccache binary RUN case "$(uname -m)" in \ - x86_64) ARCH=x86_64-unknown-linux-musl ;; \ - aarch64) ARCH=aarch64-unknown-linux-musl ;; \ - *) echo "Unsupported architecture" && exit 1 ;; \ + x86_64) ARCH=x86_64-unknown-linux-musl ;; \ + aarch64) ARCH=aarch64-unknown-linux-musl ;; \ + *) echo "Unsupported architecture" && exit 1 ;; \ esac && \ curl -L https://github.com/mozilla/sccache/releases/download/v0.8.2/sccache-v0.8.2-${ARCH}.tar.gz | tar xz && \ mv sccache-v0.8.2-${ARCH}/sccache /usr/local/cargo/bin/ && \ From f769ab97072764d7bdfaf70e3b57b5bedbf52562 Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Mon, 22 Dec 2025 13:15:07 -0500 Subject: [PATCH 16/21] toml fix --- Cargo.lock | 1 - lading_payload/Cargo.toml | 5 ----- 2 files changed, 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f57e4737a..d65c6066b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1966,7 +1966,6 @@ dependencies = [ "serde", "serde_json", "serde_tuple", - "tempfile", "thiserror 2.0.17", "time", "tokio", diff --git a/lading_payload/Cargo.toml b/lading_payload/Cargo.toml index 601c5e8da..a761f2369 100644 --- a/lading_payload/Cargo.toml +++ b/lading_payload/Cargo.toml @@ -31,12 +31,7 @@ arbitrary = { version = "1", optional = true, features = ["derive"] } [dev-dependencies] proptest = { workspace = true } proptest-derive = { workspace = true } -<<<<<<< HEAD criterion = { version = "0.8", features = ["html_reports"] } -======= -criterion = { version = "0.7", features = ["html_reports"] } -tempfile = { workspace = true } ->>>>>>> ac30b30 (omit timestamp) [features] default = [] From d57cf00b9bc28911f94fdc2a41eec44a0f1dba87 Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Mon, 22 Dec 2025 13:16:41 -0500 Subject: [PATCH 17/21] add chrono --- Cargo.lock | 3 +++ lading_payload/Cargo.toml | 1 + 2 files changed, 4 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index d65c6066b..b9b09e638 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -624,8 +624,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-link 0.2.1", ] @@ -1956,6 +1958,7 @@ dependencies = [ "arbitrary", "byte-unit", "bytes", + "chrono", "criterion", "opentelemetry-proto", "proptest", diff --git a/lading_payload/Cargo.toml b/lading_payload/Cargo.toml index a761f2369..29e62a92f 100644 --- a/lading_payload/Cargo.toml +++ b/lading_payload/Cargo.toml @@ -27,6 +27,7 @@ time = { version = "0.3", features = ["formatting"] } tracing = { workspace = true } tokio = { workspace = true } arbitrary = { version = "1", optional = true, features = ["derive"] } +chrono = "0.4.42" [dev-dependencies] proptest = { workspace = true } From 83a67fa59f7561e0101994e45ec6b8ca16576f9a Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Mon, 22 Dec 2025 13:26:07 -0500 Subject: [PATCH 18/21] fix dep --- Cargo.lock | 1 + lading_payload/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index b9b09e638..070191f56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1969,6 +1969,7 @@ dependencies = [ "serde", "serde_json", "serde_tuple", + "tempfile", "thiserror 2.0.17", "time", "tokio", diff --git a/lading_payload/Cargo.toml b/lading_payload/Cargo.toml index 29e62a92f..cd7096626 100644 --- a/lading_payload/Cargo.toml +++ b/lading_payload/Cargo.toml @@ -33,6 +33,7 @@ chrono = "0.4.42" proptest = { workspace = true } proptest-derive = { workspace = true } criterion = { version = "0.8", features = ["html_reports"] } +tempfile = { workspace = true } [features] default = [] From 365e2019e9df8ef08c882c261ad36cc50cf0e45c Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Mon, 22 Dec 2025 13:37:17 -0500 Subject: [PATCH 19/21] fix ci --- lading/src/generator/file_gen/logrotate_fs.rs | 12 +++--- .../generator/file_gen/logrotate_fs/model.rs | 39 +++++++++++++------ lading_payload/src/statik_line_rate.rs | 2 +- lading_payload/src/statik_second.rs | 2 +- 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/lading/src/generator/file_gen/logrotate_fs.rs b/lading/src/generator/file_gen/logrotate_fs.rs index 5c832eeb8..8129a5d00 100644 --- a/lading/src/generator/file_gen/logrotate_fs.rs +++ b/lading/src/generator/file_gen/logrotate_fs.rs @@ -115,12 +115,12 @@ enum LoadProfileCompat { } impl LoadProfile { - fn from_legacy(profile: LegacyLoadProfile) -> Self { + fn from_legacy(profile: &LegacyLoadProfile) -> Self { match profile { LegacyLoadProfile::Constant(bps) => LoadProfile::Constant { rate: RateSpec { mode: None, - bytes_per_second: Some(bps), + bytes_per_second: Some(*bps), blocks_per_second: None, }, }, @@ -130,12 +130,12 @@ impl LoadProfile { } => LoadProfile::Linear { initial: RateSpec { mode: None, - bytes_per_second: Some(initial_bytes_per_second), + bytes_per_second: Some(*initial_bytes_per_second), blocks_per_second: None, }, rate_of_change: RateSpec { mode: None, - bytes_per_second: Some(rate), + bytes_per_second: Some(*rate), blocks_per_second: None, }, }, @@ -143,7 +143,7 @@ impl LoadProfile { rate: RateSpec { mode: Some(ThrottleMode::Blocks), bytes_per_second: None, - blocks_per_second: Some(blocks_per_second), + blocks_per_second: Some(*blocks_per_second), }, }, } @@ -167,7 +167,7 @@ impl<'de> Deserialize<'de> for LoadProfile { rate_of_change, }, }, - LoadProfileCompat::Legacy(profile) => LoadProfile::from_legacy(profile), + LoadProfileCompat::Legacy(profile) => LoadProfile::from_legacy(&profile), }) } } diff --git a/lading/src/generator/file_gen/logrotate_fs/model.rs b/lading/src/generator/file_gen/logrotate_fs/model.rs index d4fa5cb30..0829b75fe 100644 --- a/lading/src/generator/file_gen/logrotate_fs/model.rs +++ b/lading/src/generator/file_gen/logrotate_fs/model.rs @@ -13,6 +13,17 @@ pub(crate) type Tick = u64; /// The identification node number pub(crate) type Inode = usize; +/// Parameters describing a file's position in the rotation hierarchy +#[derive(Debug, Clone, Copy)] +pub(crate) struct FileHierarchy { + /// The parent node of this file + pub(crate) parent: Inode, + /// The peer of this file (next in rotation sequence) + pub(crate) peer: Option, + /// The group ID shared by all files in the same rotation group + pub(crate) group_id: u16, +} + /// Model representation of a `File`. Does not actually contain any bytes but /// stores sufficient metadata to determine access patterns over time. #[derive(Debug, Clone)] @@ -134,17 +145,15 @@ impl File { /// Create a new instance of `File` pub(crate) fn new( mut rng: SmallRng, - parent: Inode, - group_id: u16, + hierarchy: FileHierarchy, bytes_per_tick: u64, now: Tick, - peer: Option, total_cache_size: u64, block_handle: block::Handle, ) -> Self { let cache_offset = generate_cache_offset(&mut rng, total_cache_size); Self { - parent, + parent: hierarchy.parent, bytes_written: 0, bytes_read: 0, access_tick: now, @@ -155,8 +164,8 @@ impl File { read_only: false, read_only_since: None, ordinal: 0, - peer, - group_id, + peer: hierarchy.peer, + group_id: hierarchy.group_id, open_handles: 0, unlinked: false, max_offset_observed: 0, @@ -540,13 +549,16 @@ impl State { let mut child_rng = SmallRng::from_seed(child_seed); let block_handle = generate_block_handle(&mut child_rng, &state.block_cache); + let hierarchy = FileHierarchy { + parent: current_inode, + peer: None, + group_id, + }; let file = File::new( child_rng, - current_inode, - group_id, + hierarchy, 0, state.now, - None, state.total_cache_size, block_handle, ); @@ -755,13 +767,16 @@ impl State { let new_file_inode = self.next_inode; let mut file_rng = file_rng; let block_handle = generate_block_handle(&mut file_rng, &self.block_cache); + let hierarchy = FileHierarchy { + parent: parent_inode, + peer: Some(rotated_inode), + group_id, + }; let mut new_file = File::new( file_rng, - parent_inode, - group_id, + hierarchy, bytes_per_tick, self.now.saturating_sub(1), - Some(rotated_inode), self.total_cache_size, block_handle, ); diff --git a/lading_payload/src/statik_line_rate.rs b/lading_payload/src/statik_line_rate.rs index 2240b6f55..d5ddc4c10 100644 --- a/lading_payload/src/statik_line_rate.rs +++ b/lading_payload/src/statik_line_rate.rs @@ -153,7 +153,7 @@ mod tests { use super::*; use crate::Serialize; use rand::{SeedableRng, rngs::StdRng}; - use std::{env, fs::File, io::Write as IoWrite}; + use std::{env, fs::File, io::Write}; #[test] fn writes_requested_number_of_lines() { diff --git a/lading_payload/src/statik_second.rs b/lading_payload/src/statik_second.rs index 667eddab3..1fe0b21c1 100644 --- a/lading_payload/src/statik_second.rs +++ b/lading_payload/src/statik_second.rs @@ -228,7 +228,7 @@ mod tests { use super::*; use crate::Serialize; use rand::{SeedableRng, rngs::StdRng}; - use std::{fs::File, io::Write as IoWrite}; + use std::{fs::File, io::Write}; use tempfile::tempdir; #[test] From 8646aaae70a63843f7346a921edd193d1c28beb9 Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Mon, 22 Dec 2025 13:39:10 -0500 Subject: [PATCH 20/21] fix fmt2 --- lading/src/generator/file_gen/traditional.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lading/src/generator/file_gen/traditional.rs b/lading/src/generator/file_gen/traditional.rs index b096ec645..4589a0d71 100644 --- a/lading/src/generator/file_gen/traditional.rs +++ b/lading/src/generator/file_gen/traditional.rs @@ -329,9 +329,7 @@ impl Child { } let buffer_capacity = match self.throttle.mode { ThrottleMode::Bytes => self.throttle.maximum_capacity() as usize, - ThrottleMode::Blocks => { - usize::try_from(self.maximum_block_size).unwrap_or(usize::MAX) - } + ThrottleMode::Blocks => usize::try_from(self.maximum_block_size).unwrap_or(usize::MAX), }; let mut fp = BufWriter::with_capacity( buffer_capacity, From ea093dcd9a5074d0aa3818ea5d99a7aabee81604 Mon Sep 17 00:00:00 2001 From: Jake Saferstein Date: Mon, 22 Dec 2025 16:04:12 -0500 Subject: [PATCH 21/21] add debugging for memory --- lading/src/bin/lading.rs | 6 ++++++ lading/src/generator/file_gen/logrotate_fs.rs | 7 +++++++ lading_payload/src/statik_second.rs | 12 ++++++++---- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/lading/src/bin/lading.rs b/lading/src/bin/lading.rs index 27440964f..76f65f93e 100644 --- a/lading/src/bin/lading.rs +++ b/lading/src/bin/lading.rs @@ -716,6 +716,12 @@ fn main() -> Result<(), Error> { "Lading running with {limit} amount of memory.", limit = memory_limit.to_string() ); + if let Ok(limit_v1) = std::fs::read_to_string("/sys/fs/cgroup/memory/memory.limit_in_bytes") { + info!("cgroup v1 memory.limit_in_bytes: {}", limit_v1.trim()); + } + if let Ok(max_v2) = std::fs::read_to_string("/sys/fs/cgroup/memory.max") { + info!("cgroup v2 memory.max: {}", max_v2.trim()); + } // Two-parser fallback logic until CliFlatLegacy is removed let args = match CliWithSubcommands::try_parse() { diff --git a/lading/src/generator/file_gen/logrotate_fs.rs b/lading/src/generator/file_gen/logrotate_fs.rs index 8129a5d00..75de02918 100644 --- a/lading/src/generator/file_gen/logrotate_fs.rs +++ b/lading/src/generator/file_gen/logrotate_fs.rs @@ -346,6 +346,13 @@ impl Server { let start_time = Instant::now(); let start_time_system = SystemTime::now(); + let block_cache_size = block_cache.total_size(); + info!( + "LogrotateFS block cache initialized: requested={}, actual={} bytes, blocks={}", + config.maximum_prebuild_cache_size_bytes, + block_cache_size, + block_cache.len() + ); let state = model::State::new( &mut rng, start_time.elapsed().as_secs(), diff --git a/lading_payload/src/statik_second.rs b/lading_payload/src/statik_second.rs index 1fe0b21c1..36617ec49 100644 --- a/lading_payload/src/statik_second.rs +++ b/lading_payload/src/statik_second.rs @@ -10,7 +10,7 @@ use std::{ use chrono::{NaiveDateTime, TimeZone, Utc}; use rand::Rng; -use tracing::debug; +use tracing::{debug, info}; #[derive(Debug)] struct BlockLines { @@ -62,6 +62,7 @@ impl StaticSecond { start_line_index: Option, ) -> Result { let file = File::open(path)?; + let file_size_bytes = file.metadata().map(|m| m.len()).unwrap_or(0); let reader = BufReader::new(file); let mut blocks: Vec = Vec::new(); @@ -159,10 +160,13 @@ impl StaticSecond { } } - debug!( - "StaticSecond loaded {} second-buckets from {}", + info!( + "StaticSecond loaded {} second-buckets ({} total lines) from {} ({} bytes, emit_placeholder={})", blocks.len(), - path.display() + total_lines, + path.display(), + file_size_bytes, + emit_placeholder ); Ok(Self {