From 622cc37aa747e71c67cd5205d686a76815cdda9c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 Mar 2026 17:08:47 +0100 Subject: [PATCH 01/12] threading --- Cargo.lock | 1 + bin/node/src/commands/bundled.rs | 9 +- bin/node/src/commands/store.rs | 9 +- bin/stress-test/src/seeding/mod.rs | 3 +- bin/stress-test/src/store/mod.rs | 2 +- crates/store/Cargo.toml | 2 +- crates/store/src/server/mod.rs | 5 +- crates/store/src/state/loader.rs | 29 ++++-- crates/store/src/state/mod.rs | 14 ++- crates/utils/Cargo.toml | 4 + crates/utils/src/clap.rs | 139 +++++++++++++++++++++++++++++ 11 files changed, 202 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a6b076df8..610d8fd0b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3233,6 +3233,7 @@ dependencies = [ "humantime", "itertools 0.14.0", "lru", + "miden-large-smt-backend-rocksdb", "miden-protocol", "opentelemetry", "opentelemetry-otlp", diff --git a/bin/node/src/commands/bundled.rs b/bin/node/src/commands/bundled.rs index 4f20d812bf..697e7f7c41 100644 --- a/bin/node/src/commands/bundled.rs +++ b/bin/node/src/commands/bundled.rs @@ -5,7 +5,7 @@ use anyhow::Context; use miden_node_block_producer::BlockProducer; use miden_node_rpc::Rpc; use miden_node_store::Store; -use miden_node_utils::clap::GrpcOptionsExternal; +use miden_node_utils::clap::{GrpcOptionsExternal, StorageOptions}; use miden_node_utils::grpc::UrlExt; use miden_node_validator::{Validator, ValidatorSigner}; use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SecretKey; @@ -84,6 +84,9 @@ pub enum BundledCommand { #[command(flatten)] grpc_options: GrpcOptionsExternal, + + #[command(flatten)] + storage_options: StorageOptions, }, } @@ -116,6 +119,7 @@ impl BundledCommand { validator, enable_otel: _, grpc_options, + storage_options, } => { Self::start( rpc_url, @@ -125,6 +129,7 @@ impl BundledCommand { ntx_builder, validator, grpc_options, + storage_options, ) .await }, @@ -140,6 +145,7 @@ impl BundledCommand { ntx_builder: NtxBuilderConfig, validator: BundledValidatorConfig, grpc_options: GrpcOptionsExternal, + storage_options: StorageOptions, ) -> anyhow::Result<()> { // Start listening on all gRPC urls so that inter-component connections can be created // before each component is fully started up. @@ -197,6 +203,7 @@ impl BundledCommand { data_directory: data_directory_clone, block_prover_url, grpc_options: grpc_options.into(), + storage_options, } .serve() .await diff --git a/bin/node/src/commands/store.rs b/bin/node/src/commands/store.rs index d3105cc5aa..e7730124e8 100644 --- a/bin/node/src/commands/store.rs +++ b/bin/node/src/commands/store.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use anyhow::Context; use miden_node_store::Store; use miden_node_store::genesis::config::{AccountFileWithName, GenesisConfig}; -use miden_node_utils::clap::GrpcOptionsInternal; +use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions}; use miden_node_utils::grpc::UrlExt; use miden_node_utils::signer::BlockSigner; use miden_node_validator::ValidatorSigner; @@ -80,6 +80,9 @@ pub enum StoreCommand { #[command(flatten)] grpc_options: GrpcOptionsInternal, + + #[command(flatten)] + storage_options: StorageOptions, }, } @@ -109,6 +112,7 @@ impl StoreCommand { data_directory, enable_otel: _, grpc_options, + storage_options, } => { Self::start( rpc_url, @@ -117,6 +121,7 @@ impl StoreCommand { block_prover_url, data_directory, grpc_options, + storage_options, ) .await }, @@ -138,6 +143,7 @@ impl StoreCommand { block_prover_url: Option, data_directory: PathBuf, grpc_options: GrpcOptionsInternal, + storage_options: StorageOptions, ) -> anyhow::Result<()> { let rpc_listener = rpc_url .to_socket() @@ -167,6 +173,7 @@ impl StoreCommand { block_producer_listener, data_directory, grpc_options, + storage_options, } .serve() .await diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index 6621bf227d..d210818051 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -9,7 +9,7 @@ use miden_node_block_producer::store::StoreClient; use miden_node_proto::domain::batch::BatchInputs; use miden_node_proto::generated::store::rpc_client::RpcClient; use miden_node_store::{DataDirectory, GenesisState, Store}; -use miden_node_utils::clap::GrpcOptionsInternal; +use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions}; use miden_node_utils::tracing::grpc::OtelInterceptor; use miden_protocol::account::auth::AuthScheme; use miden_protocol::account::delta::AccountUpdateDetails; @@ -552,6 +552,7 @@ pub async fn start_store( block_producer_listener, data_directory: dir, grpc_options: GrpcOptionsInternal::bench(), + storage_options: StorageOptions::bench(), } .serve() .await diff --git a/bin/stress-test/src/store/mod.rs b/bin/stress-test/src/store/mod.rs index 314a5e95d0..d973ca1f0e 100644 --- a/bin/stress-test/src/store/mod.rs +++ b/bin/stress-test/src/store/mod.rs @@ -490,7 +490,7 @@ struct SyncChainMmrRun { pub async fn load_state(data_directory: &Path) { let start = Instant::now(); let (termination_ask, _) = tokio::sync::mpsc::channel(1); - let _state = State::load(data_directory, termination_ask).await.unwrap(); + let _state = State::load(data_directory, Default::default(), termination_ask).await.unwrap(); let elapsed = start.elapsed(); // Get database path and run SQL commands to count records diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index f22555e10a..7c8135d97a 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -72,7 +72,7 @@ termtree = "1.0" [features] default = ["rocksdb"] -rocksdb = ["dep:miden-large-smt-backend-rocksdb"] +rocksdb = ["dep:miden-large-smt-backend-rocksdb", "miden-node-utils/rocksdb"] [[bench]] harness = false diff --git a/crates/store/src/server/mod.rs b/crates/store/src/server/mod.rs index a35f25e8b9..3d167e6bef 100644 --- a/crates/store/src/server/mod.rs +++ b/crates/store/src/server/mod.rs @@ -10,7 +10,7 @@ use miden_node_proto_build::{ store_ntx_builder_api_descriptor, store_rpc_api_descriptor, }; -use miden_node_utils::clap::GrpcOptionsInternal; +use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions}; use miden_node_utils::panic::{CatchPanicLayer, catch_panic_layer_fn}; use miden_node_utils::signer::BlockSigner; use miden_node_utils::tracing::grpc::grpc_trace_fn; @@ -41,6 +41,7 @@ pub struct Store { /// URL for the Block Prover client. Uses local prover if `None`. pub block_prover_url: Option, pub data_directory: PathBuf, + pub storage_options: StorageOptions, pub grpc_options: GrpcOptionsInternal, } @@ -98,7 +99,7 @@ impl Store { let (termination_ask, mut termination_signal) = tokio::sync::mpsc::channel::(1); let state = Arc::new( - State::load(&self.data_directory, termination_ask) + State::load(&self.data_directory, self.storage_options, termination_ask) .await .context("failed to load state")?, ); diff --git a/crates/store/src/state/loader.rs b/crates/store/src/state/loader.rs index d632a1c0c4..20ef9cdc04 100644 --- a/crates/store/src/state/loader.rs +++ b/crates/store/src/state/loader.rs @@ -14,7 +14,8 @@ use std::path::Path; use miden_crypto::merkle::mmr::Mmr; #[cfg(feature = "rocksdb")] -use miden_large_smt_backend_rocksdb::{RocksDbConfig, RocksDbStorage}; +use miden_large_smt_backend_rocksdb::RocksDbStorage; +use miden_node_utils::clap::RocksDbOptions; use miden_protocol::block::account_tree::{AccountTree, account_id_to_smt_key}; use miden_protocol::block::nullifier_tree::NullifierTree; use miden_protocol::block::{BlockNumber, Blockchain}; @@ -98,8 +99,14 @@ fn block_num_to_nullifier_leaf(block_num: BlockNumber) -> Word { /// which detects divergence between persistent storage and the database. If divergence is detected, /// the user should manually delete the tree storage directories and restart the node. pub trait StorageLoader: SmtStorage + Sized { + /// A configuration type for the implementation. + type Config: std::fmt::Debug + std::default::Default; /// Creates a storage backend for the given domain. - fn create(data_dir: &Path, domain: &'static str) -> Result; + fn create( + data_dir: &Path, + storage_options: &Self::Config, + domain: &'static str, + ) -> Result; /// Loads an account tree, either from persistent storage or by rebuilding from DB. fn load_account_tree( @@ -119,7 +126,12 @@ pub trait StorageLoader: SmtStorage + Sized { #[cfg(not(feature = "rocksdb"))] impl StorageLoader for MemoryStorage { - fn create(_data_dir: &Path, _domain: &'static str) -> Result { + type Config = (); + fn create( + _data_dir: &Path, + _storage_options: &Self::Config, + _domain: &'static str, + ) -> Result { Ok(MemoryStorage::default()) } @@ -207,12 +219,17 @@ impl StorageLoader for MemoryStorage { #[cfg(feature = "rocksdb")] impl StorageLoader for RocksDbStorage { - fn create(data_dir: &Path, domain: &'static str) -> Result { + type Config = RocksDbOptions; + fn create( + data_dir: &Path, + storage_options: &Self::Config, + domain: &'static str, + ) -> Result { let storage_path = data_dir.join(domain); - + let config = storage_options.with_path(&storage_path); fs_err::create_dir_all(&storage_path) .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string()))?; - RocksDbStorage::open(RocksDbConfig::new(storage_path)) + RocksDbStorage::open(config) .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string())) } diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index a996e1956a..a511eee95d 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -21,6 +21,7 @@ use miden_node_proto::domain::account::{ StorageMapRequest, }; use miden_node_proto::domain::batch::BatchInputs; +use miden_node_utils::clap::StorageOptions; use miden_node_utils::formatting::format_array; use miden_node_utils::limiter::{QueryParamLimiter, QueryParamStorageMapKeyTotalLimit}; use miden_protocol::Word; @@ -134,6 +135,7 @@ impl State { #[instrument(target = COMPONENT, skip_all)] pub async fn load( data_path: &Path, + storage_options: StorageOptions, termination_ask: tokio::sync::mpsc::Sender, ) -> Result { let data_directory = DataDirectory::load(data_path.to_path_buf()) @@ -152,10 +154,18 @@ impl State { let blockchain = load_mmr(&mut db).await?; let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS); - let account_storage = TreeStorage::create(data_path, ACCOUNT_TREE_STORAGE_DIR)?; + let account_storage = TreeStorage::create( + data_path, + &storage_options.account_tree.into(), + ACCOUNT_TREE_STORAGE_DIR, + )?; let account_tree = account_storage.load_account_tree(&mut db).await?; - let nullifier_storage = TreeStorage::create(data_path, NULLIFIER_TREE_STORAGE_DIR)?; + let nullifier_storage = TreeStorage::create( + data_path, + &storage_options.nullifier_tree.into(), + NULLIFIER_TREE_STORAGE_DIR, + )?; let nullifier_tree = nullifier_storage.load_nullifier_tree(&mut db).await?; // Verify that tree roots match the expected roots from the database. diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 4a02ea107e..92aa378188 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -17,6 +17,7 @@ workspace = true [features] # Enables utility functions for testing traces created by some other crate's stack. testing = ["miden-protocol/testing"] +rocksdb = ["dep:miden-large-smt-backend-rocksdb"] [dependencies] anyhow = { workspace = true } @@ -45,5 +46,8 @@ tracing-opentelemetry = { version = "0.32" } tracing-subscriber = { workspace = true } url = { workspace = true } +# RocksDbConfig is needed due to orphan rules +miden-large-smt-backend-rocksdb = { workspace = true, optional = true } + [dev-dependencies] thiserror = { workspace = true } diff --git a/crates/utils/src/clap.rs b/crates/utils/src/clap.rs index 0d85dce5ad..5384993277 100644 --- a/crates/utils/src/clap.rs +++ b/crates/utils/src/clap.rs @@ -1,8 +1,13 @@ //! Public module for share clap pieces to reduce duplication use std::num::{NonZeroU32, NonZeroU64}; +#[cfg(feature = "rocksdb")] +use std::path::Path; use std::time::Duration; +#[cfg(feature = "rocksdb")] +use miden_large_smt_backend_rocksdb::RocksDbConfig; + const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); const TEST_REQUEST_TIMEOUT: Duration = Duration::from_secs(5); const DEFAULT_MAX_CONNECTION_AGE: Duration = Duration::from_mins(30); @@ -132,3 +137,137 @@ impl GrpcOptionsExternal { } } } + +/// Collection of per usage storage backend configurations. +/// +/// Note: Currently only contains `rocksdb` related configuration. +#[derive(clap::Args, Clone, Debug, PartialEq, Eq)] +pub struct StorageOptions { + #[cfg(feature = "rocksdb")] + #[clap(flatten)] + pub account_tree: AccountTreeRocksDbOptions, + #[cfg(feature = "rocksdb")] + #[clap(flatten)] + pub nullifier_tree: NullifierTreeRocksDbOptions, +} + +impl Default for StorageOptions { + fn default() -> Self { + Self { + account_tree: RocksDbOptions::default().into(), + nullifier_tree: RocksDbOptions::default().into(), + } + } +} + +impl StorageOptions { + /// Benchmark setup. + /// + /// These values were determined during development of `LargeSmt` + pub fn bench() -> Self { + let account_tree = AccountTreeRocksDbOptions { + max_open_fds: 512, + cache_size_in_bytes: 2 << 30, + }; + let nullifier_tree = NullifierTreeRocksDbOptions { + max_open_fds: 512, + cache_size_in_bytes: 2 << 30, + }; + Self { account_tree, nullifier_tree } + } +} + +/// Per usage options for rocksdb configuration +#[cfg(feature = "rocksdb")] +#[derive(clap::Args, Clone, Debug, PartialEq, Eq)] +pub struct NullifierTreeRocksDbOptions { + #[arg( + long = "nullifier_tree.rocksdb.max_open_fds", + default_value_t = 64, + value_name = "NULLIFIER_TREE__ROCKSDB__MAX_OPEN_FDS" + )] + pub max_open_fds: i32, + #[arg( + long = "nullifier_tree.rocksdb.max_cache_size", + default_value_t = 2 * 1024 * 1024 * 1024, + value_name = "NULLIFIER_TREE__ROCKSDB__CACHE_SIZE" + )] + pub cache_size_in_bytes: usize, +} + +/// Per usage options for rocksdb configuration +#[cfg(feature = "rocksdb")] +#[derive(clap::Args, Clone, Debug, PartialEq, Eq)] +pub struct AccountTreeRocksDbOptions { + #[arg( + long = "account_tree.rocksdb.max_open_fds", + default_value_t = 64, + value_name = "ACCOUNT_TREE__ROCKSDB__MAX_OPEN_FDS" + )] + pub max_open_fds: i32, + #[arg( + long = "account_tree.rocksdb.max_cache_size", + default_value_t = 2<<30, + value_name = "ACCOUNT_TREE__ROCKSDB__CACHE_SIZE" + )] + pub cache_size_in_bytes: usize, +} + +/// General confiration options for rocksdb. +#[cfg(feature = "rocksdb")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RocksDbOptions { + pub max_open_fds: i32, + pub cache_size_in_bytes: usize, +} + +#[cfg(feature = "rocksdb")] +impl Default for RocksDbOptions { + fn default() -> Self { + Self { + max_open_fds: 512, + cache_size_in_bytes: 2 << 30, + } + } +} + +#[cfg(feature = "rocksdb")] +impl From for RocksDbOptions { + fn from(value: AccountTreeRocksDbOptions) -> Self { + let AccountTreeRocksDbOptions { max_open_fds, cache_size_in_bytes } = value; + Self { max_open_fds, cache_size_in_bytes } + } +} + +#[cfg(feature = "rocksdb")] +impl From for RocksDbOptions { + fn from(value: NullifierTreeRocksDbOptions) -> Self { + let NullifierTreeRocksDbOptions { max_open_fds, cache_size_in_bytes } = value; + Self { max_open_fds, cache_size_in_bytes } + } +} + +#[cfg(feature = "rocksdb")] +impl From for AccountTreeRocksDbOptions { + fn from(value: RocksDbOptions) -> Self { + let RocksDbOptions { max_open_fds, cache_size_in_bytes } = value; + Self { max_open_fds, cache_size_in_bytes } + } +} + +#[cfg(feature = "rocksdb")] +impl From for NullifierTreeRocksDbOptions { + fn from(value: RocksDbOptions) -> Self { + let RocksDbOptions { max_open_fds, cache_size_in_bytes } = value; + Self { max_open_fds, cache_size_in_bytes } + } +} + +#[cfg(feature = "rocksdb")] +impl RocksDbOptions { + pub fn with_path(self, path: &Path) -> RocksDbConfig { + RocksDbConfig::new(path) + .with_cache_size(self.cache_size_in_bytes) + .with_max_open_files(self.max_open_fds) + } +} From 89f416812eefc1e9524be316570b83da91f5f12d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 Mar 2026 17:16:24 +0100 Subject: [PATCH 02/12] docs --- docs/external/src/operator/usage.md | 57 +++++++++++++++++++++++++++++ docs/internal/src/store.md | 49 ++++++++++++++++++++++++- 2 files changed, 105 insertions(+), 1 deletion(-) diff --git a/docs/external/src/operator/usage.md b/docs/external/src/operator/usage.md index ce4b699b14..1524ba445b 100644 --- a/docs/external/src/operator/usage.md +++ b/docs/external/src/operator/usage.md @@ -138,6 +138,63 @@ You can inspect the service file with `systemctl cat miden-node` or alternativel our repository in the `packaging` folder. For the bootstrapping process be sure to specify the data-directory as expected by the systemd file. +## RocksDB tuning + +The store component uses [RocksDB](https://rocksdb.org/) to persist the account and nullifier trees. Two +independent RocksDB instances are opened — one per tree — and each can be tuned independently via CLI flags or the +corresponding environment variables. + +### Tuneable parameters + +| Flag | Env variable | Default | Description | +|---|---|---|---| +| `--account_tree.rocksdb.max_open_fds` | `ACCOUNT_TREE__ROCKSDB__MAX_OPEN_FDS` | `64` | Maximum number of file descriptors the account-tree RocksDB instance may keep open simultaneously. | +| `--account_tree.rocksdb.max_cache_size` | `ACCOUNT_TREE__ROCKSDB__CACHE_SIZE` | `2147483648` (2 GiB) | Size in bytes of the shared LRU block cache used by all column families in the account-tree instance. | +| `--nullifier_tree.rocksdb.max_open_fds` | `NULLIFIER_TREE__ROCKSDB__MAX_OPEN_FDS` | `64` | Maximum number of file descriptors the nullifier-tree RocksDB instance may keep open simultaneously. | +| `--nullifier_tree.rocksdb.max_cache_size` | `NULLIFIER_TREE__ROCKSDB__CACHE_SIZE` | `2147483648` (2 GiB) | Size in bytes of the shared LRU block cache used by all column families in the nullifier-tree instance. | + +### Host-specific guidance + +**Block cache** (`max_cache_size`): The block cache is the single most impactful knob. Set it as large as the host +can comfortably spare. A node that has recently restarted needs to warm the cache before reaching peak read +throughput, so prefer larger values on production machines. A combined allocation of **4–8 GiB** (2–4 GiB per tree) +is a reasonable starting point for a machine with 16+ GiB of RAM. + +**Open file descriptors** (`max_open_fds`): RocksDB keeps SST files open for fast random access. The default of `64` +is intentionally conservative to stay well within typical OS `ulimit` values. On a dedicated server where the OS +limit has been raised (e.g., `ulimit -n 65536`), increasing this to `512` or higher can reduce latency spikes +caused by re-opening files during compaction. + +**Compaction parallelism and background jobs**: These are automatically set to the number of CPU cores detected at +startup (via `rayon::current_num_threads()`) and are currently not configurable at runtime. Ensure the node process +is not pinned to a small CPU set if you want compaction to keep up with write throughput. + +**WAL**: Writes skip the per-write `fsync` (`set_sync(false)`) for throughput. A 512 MiB WAL cap is applied +globally. A hard node crash between flushes may require replaying the WAL on next startup, which RocksDB handles +automatically. + +### Example: high-memory production host + +```sh +miden-node bundled start \ + --data-directory data \ + --rpc.url http://0.0.0.0:57291 \ + --account_tree.rocksdb.max_cache_size 4294967296 \ + --account_tree.rocksdb.max_open_fds 512 \ + --nullifier_tree.rocksdb.max_cache_size 4294967296 \ + --nullifier_tree.rocksdb.max_open_fds 512 +``` + +Or equivalently via environment variables: + +```sh +export ACCOUNT_TREE__ROCKSDB__CACHE_SIZE=4294967296 +export ACCOUNT_TREE__ROCKSDB__MAX_OPEN_FDS=512 +export NULLIFIER_TREE__ROCKSDB__CACHE_SIZE=4294967296 +export NULLIFIER_TREE__ROCKSDB__MAX_OPEN_FDS=512 +miden-node bundled start --data-directory data --rpc.url http://0.0.0.0:57291 +``` + ## Environment variables Most configuration options can also be configured using environment variables as an alternative to providing the values diff --git a/docs/internal/src/store.md b/docs/internal/src/store.md index 1929b7c491..277f1bbbac 100644 --- a/docs/internal/src/store.md +++ b/docs/internal/src/store.md @@ -2,7 +2,7 @@ This component persists the chain state in a `sqlite` database. It also stores each block's raw data as a file. -Mekle data structures are kept in-memory and are rebuilt on startup. Other data like account, note and nullifier +Merkle data structures are kept in-memory and are rebuilt on startup. Other data like account, note and nullifier information is always read from disk. We will need to revisit this in the future but for now this is performant enough. ## Migrations @@ -14,6 +14,53 @@ Note that the migration logic includes both a schema number _and_ a hash based o on node startup to ensure that any existing database matches the expected schema. If you're seeing database failures on startup its likely that you created the database _before_ making schema changes resulting in different schema hashes. +## RocksDB tree storage + +The account and nullifier trees are persisted in separate RocksDB instances under +`/accounttree` and `/nullifiertree` respectively. Both are +managed by `crates/large-smt-backend-rocksdb`. + +### Column families + +| Column family | Contents | +|---|---| +| `leaves` | SMT leaf nodes keyed by their logical `u64` index (big-endian). | +| `st24`–`st56` | Serialised `Subtree` objects at depths 24, 32, 40, 48, 56. | +| `metadata` | SMT root hash, leaf count, entry count. | +| `depth24` | Cached depth-24 inner-node hashes for fast top-level reconstruction on startup. | + +### Fixed tuning applied at startup + +The following settings are applied unconditionally to every opened instance and cannot be changed +via CLI flags: + +| Setting | Value | Rationale | +|---|---|---| +| Compaction threads (`increase_parallelism`) | `rayon::current_num_threads()` | Match CPU core count to avoid compaction falling behind under write load. | +| Background flush/compaction jobs | `rayon::current_num_threads()` | Same as above. | +| Max WAL size | 512 MiB | Bounds recovery time on restart. | +| Memtable size per CF | 128 MiB, up to 3 in-flight | Batches writes before flushing; reduces write amplification. | +| Target SST file size | 512 MiB (multiplier ×2 per level) | Reduces the number of files and keeps bloom filters effective. | +| Compaction style | Level | Predictable read/write amplification for SMT access patterns. | +| Compression | LZ4 | Fast compression with decent ratio for node data. | +| Bloom filter bits (leaves / `st32`–`st40`) | 10.0 bits/key | Tuned for point-lookup miss rate vs. memory trade-off. | +| Bloom filter bits (`st24`) | 8.0 bits/key | Shallower subtrees are queried less frequently. | +| Bloom filter bits (`st48`–`st56`) | 12.0 bits/key | Deeper subtrees are larger; more bits reduce false-positive cost. | +| WAL sync per write | disabled (`set_sync(false)`) | Throughput optimisation; RocksDB will replay the WAL on an unclean shutdown. | + +### Runtime-tuneable parameters + +Two parameters per tree can be adjusted at launch via CLI flags or environment variables: + +| Flag | Default | Notes | +|---|---|---| +| `--account_tree.rocksdb.max_cache_size` | 2 GiB | Shared LRU block cache across all CFs. Larger is better; size to available RAM. | +| `--account_tree.rocksdb.max_open_fds` | 64 | Raise to 512+ on machines with a high `ulimit -n`. | +| `--nullifier_tree.rocksdb.max_cache_size` | 2 GiB | Same as above for the nullifier tree. | +| `--nullifier_tree.rocksdb.max_open_fds` | 64 | Same as above for the nullifier tree. | + +See the [operator usage guide](../../../external/src/operator/usage.md) for deployment examples. + ## Architecture The store consists mainly of a gRPC server which answers requests from the RPC and block-producer components, as well as From 28b22a0efa48e4f19f7b5e3ab39f424ec92eab43 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 Mar 2026 17:18:49 +0100 Subject: [PATCH 03/12] shorten --- docs/external/src/operator/usage.md | 51 +++++------------------------ docs/internal/src/store.md | 50 +++++----------------------- 2 files changed, 18 insertions(+), 83 deletions(-) diff --git a/docs/external/src/operator/usage.md b/docs/external/src/operator/usage.md index 1524ba445b..840bee2cc4 100644 --- a/docs/external/src/operator/usage.md +++ b/docs/external/src/operator/usage.md @@ -140,40 +140,17 @@ expected by the systemd file. ## RocksDB tuning -The store component uses [RocksDB](https://rocksdb.org/) to persist the account and nullifier trees. Two -independent RocksDB instances are opened — one per tree — and each can be tuned independently via CLI flags or the -corresponding environment variables. +The store uses RocksDB for the account and nullifier trees, one instance each. The two most impactful knobs per tree +are exposed as CLI flags (also available as environment variables): -### Tuneable parameters +| Flag | Default | Notes | +|---|---|---| +| `--account_tree.rocksdb.max_cache_size` | 2 GiB | Shared LRU block cache. Increase on memory-rich hosts. | +| `--account_tree.rocksdb.max_open_fds` | 64 | Raise to 512+ when `ulimit -n` allows. | +| `--nullifier_tree.rocksdb.max_cache_size` | 2 GiB | Same as above for the nullifier tree. | +| `--nullifier_tree.rocksdb.max_open_fds` | 64 | Same as above for the nullifier tree. | -| Flag | Env variable | Default | Description | -|---|---|---|---| -| `--account_tree.rocksdb.max_open_fds` | `ACCOUNT_TREE__ROCKSDB__MAX_OPEN_FDS` | `64` | Maximum number of file descriptors the account-tree RocksDB instance may keep open simultaneously. | -| `--account_tree.rocksdb.max_cache_size` | `ACCOUNT_TREE__ROCKSDB__CACHE_SIZE` | `2147483648` (2 GiB) | Size in bytes of the shared LRU block cache used by all column families in the account-tree instance. | -| `--nullifier_tree.rocksdb.max_open_fds` | `NULLIFIER_TREE__ROCKSDB__MAX_OPEN_FDS` | `64` | Maximum number of file descriptors the nullifier-tree RocksDB instance may keep open simultaneously. | -| `--nullifier_tree.rocksdb.max_cache_size` | `NULLIFIER_TREE__ROCKSDB__CACHE_SIZE` | `2147483648` (2 GiB) | Size in bytes of the shared LRU block cache used by all column families in the nullifier-tree instance. | - -### Host-specific guidance - -**Block cache** (`max_cache_size`): The block cache is the single most impactful knob. Set it as large as the host -can comfortably spare. A node that has recently restarted needs to warm the cache before reaching peak read -throughput, so prefer larger values on production machines. A combined allocation of **4–8 GiB** (2–4 GiB per tree) -is a reasonable starting point for a machine with 16+ GiB of RAM. - -**Open file descriptors** (`max_open_fds`): RocksDB keeps SST files open for fast random access. The default of `64` -is intentionally conservative to stay well within typical OS `ulimit` values. On a dedicated server where the OS -limit has been raised (e.g., `ulimit -n 65536`), increasing this to `512` or higher can reduce latency spikes -caused by re-opening files during compaction. - -**Compaction parallelism and background jobs**: These are automatically set to the number of CPU cores detected at -startup (via `rayon::current_num_threads()`) and are currently not configurable at runtime. Ensure the node process -is not pinned to a small CPU set if you want compaction to keep up with write throughput. - -**WAL**: Writes skip the per-write `fsync` (`set_sync(false)`) for throughput. A 512 MiB WAL cap is applied -globally. A hard node crash between flushes may require replaying the WAL on next startup, which RocksDB handles -automatically. - -### Example: high-memory production host +Compaction parallelism is set automatically to the number of available CPU cores. ```sh miden-node bundled start \ @@ -185,16 +162,6 @@ miden-node bundled start \ --nullifier_tree.rocksdb.max_open_fds 512 ``` -Or equivalently via environment variables: - -```sh -export ACCOUNT_TREE__ROCKSDB__CACHE_SIZE=4294967296 -export ACCOUNT_TREE__ROCKSDB__MAX_OPEN_FDS=512 -export NULLIFIER_TREE__ROCKSDB__CACHE_SIZE=4294967296 -export NULLIFIER_TREE__ROCKSDB__MAX_OPEN_FDS=512 -miden-node bundled start --data-directory data --rpc.url http://0.0.0.0:57291 -``` - ## Environment variables Most configuration options can also be configured using environment variables as an alternative to providing the values diff --git a/docs/internal/src/store.md b/docs/internal/src/store.md index 277f1bbbac..11b85e3785 100644 --- a/docs/internal/src/store.md +++ b/docs/internal/src/store.md @@ -17,49 +17,17 @@ startup its likely that you created the database _before_ making schema changes ## RocksDB tree storage The account and nullifier trees are persisted in separate RocksDB instances under -`/accounttree` and `/nullifiertree` respectively. Both are -managed by `crates/large-smt-backend-rocksdb`. +`/accounttree` and `/nullifiertree`, managed by +`crates/large-smt-backend-rocksdb`. Column families: `leaves`, `st24`–`st56` (subtrees at each +depth), `metadata` (root/counts), `depth24` (cached depth-24 hashes for fast startup). -### Column families +Compaction parallelism and background jobs are set to `rayon::current_num_threads()` automatically. +WAL sync per write is disabled for throughput; a 512 MiB WAL cap bounds recovery time. Bloom filter +bits vary by depth (8.0–12.0) and memtables are 128 MiB per CF. See `RocksDbStorage::open` for the +full fixed configuration. -| Column family | Contents | -|---|---| -| `leaves` | SMT leaf nodes keyed by their logical `u64` index (big-endian). | -| `st24`–`st56` | Serialised `Subtree` objects at depths 24, 32, 40, 48, 56. | -| `metadata` | SMT root hash, leaf count, entry count. | -| `depth24` | Cached depth-24 inner-node hashes for fast top-level reconstruction on startup. | - -### Fixed tuning applied at startup - -The following settings are applied unconditionally to every opened instance and cannot be changed -via CLI flags: - -| Setting | Value | Rationale | -|---|---|---| -| Compaction threads (`increase_parallelism`) | `rayon::current_num_threads()` | Match CPU core count to avoid compaction falling behind under write load. | -| Background flush/compaction jobs | `rayon::current_num_threads()` | Same as above. | -| Max WAL size | 512 MiB | Bounds recovery time on restart. | -| Memtable size per CF | 128 MiB, up to 3 in-flight | Batches writes before flushing; reduces write amplification. | -| Target SST file size | 512 MiB (multiplier ×2 per level) | Reduces the number of files and keeps bloom filters effective. | -| Compaction style | Level | Predictable read/write amplification for SMT access patterns. | -| Compression | LZ4 | Fast compression with decent ratio for node data. | -| Bloom filter bits (leaves / `st32`–`st40`) | 10.0 bits/key | Tuned for point-lookup miss rate vs. memory trade-off. | -| Bloom filter bits (`st24`) | 8.0 bits/key | Shallower subtrees are queried less frequently. | -| Bloom filter bits (`st48`–`st56`) | 12.0 bits/key | Deeper subtrees are larger; more bits reduce false-positive cost. | -| WAL sync per write | disabled (`set_sync(false)`) | Throughput optimisation; RocksDB will replay the WAL on an unclean shutdown. | - -### Runtime-tuneable parameters - -Two parameters per tree can be adjusted at launch via CLI flags or environment variables: - -| Flag | Default | Notes | -|---|---|---| -| `--account_tree.rocksdb.max_cache_size` | 2 GiB | Shared LRU block cache across all CFs. Larger is better; size to available RAM. | -| `--account_tree.rocksdb.max_open_fds` | 64 | Raise to 512+ on machines with a high `ulimit -n`. | -| `--nullifier_tree.rocksdb.max_cache_size` | 2 GiB | Same as above for the nullifier tree. | -| `--nullifier_tree.rocksdb.max_open_fds` | 64 | Same as above for the nullifier tree. | - -See the [operator usage guide](../../../external/src/operator/usage.md) for deployment examples. +Runtime-tuneable parameters (`--{account,nullifier}_tree.rocksdb.{max_cache_size,max_open_fds}`) +are documented in the [operator usage guide](../../../external/src/operator/usage.md). ## Architecture From eafc85965b889f052cf3c92c660520398937f73d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 Mar 2026 17:19:05 +0100 Subject: [PATCH 04/12] toml --- crates/utils/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 92aa378188..af783b7368 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -16,8 +16,8 @@ workspace = true [features] # Enables utility functions for testing traces created by some other crate's stack. -testing = ["miden-protocol/testing"] rocksdb = ["dep:miden-large-smt-backend-rocksdb"] +testing = ["miden-protocol/testing"] [dependencies] anyhow = { workspace = true } @@ -47,7 +47,7 @@ tracing-subscriber = { workspace = true } url = { workspace = true } # RocksDbConfig is needed due to orphan rules -miden-large-smt-backend-rocksdb = { workspace = true, optional = true } +miden-large-smt-backend-rocksdb = { optional = true, workspace = true } [dev-dependencies] thiserror = { workspace = true } From c54abcd9efa16ca8dce14a1a11e63f832d1d0998 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 Mar 2026 17:20:43 +0100 Subject: [PATCH 05/12] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8612b4ab0f..7ab5964c9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Enhancements +- Expose per-tree RocksDB tuning options ([#1782](https://github.com/0xMiden/node/pull/1782)). - Added verbose `info!`-level logging to the network transaction builder for transaction execution, note filtering failures, and transaction outcomes ([#1770](https://github.com/0xMiden/node/pull/1770)). - [BREAKING] Move block proving from Blocker Producer to the Store ([#1579](https://github.com/0xMiden/node/pull/1579)). - [BREAKING] Updated miden-base dependencies to use `next` branch; renamed `NoteInputs` to `NoteStorage`, `.inputs()` to `.storage()`, and database `inputs` column to `storage` ([#1595](https://github.com/0xMiden/node/pull/1595)). From 5062d1eff524e3435d48352a27b6f52726f3d728 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 Mar 2026 17:47:32 +0100 Subject: [PATCH 06/12] mod rocksdb --- crates/utils/src/clap.rs | 123 +++++-------------------------- crates/utils/src/clap/rocksdb.rs | 95 ++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 106 deletions(-) create mode 100644 crates/utils/src/clap/rocksdb.rs diff --git a/crates/utils/src/clap.rs b/crates/utils/src/clap.rs index 5384993277..8201f79ef8 100644 --- a/crates/utils/src/clap.rs +++ b/crates/utils/src/clap.rs @@ -1,12 +1,11 @@ //! Public module for share clap pieces to reduce duplication use std::num::{NonZeroU32, NonZeroU64}; -#[cfg(feature = "rocksdb")] -use std::path::Path; use std::time::Duration; #[cfg(feature = "rocksdb")] -use miden_large_smt_backend_rocksdb::RocksDbConfig; +mod rocksdb; +pub use rocksdb::*; const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); const TEST_REQUEST_TIMEOUT: Duration = Duration::from_secs(5); @@ -154,7 +153,9 @@ pub struct StorageOptions { impl Default for StorageOptions { fn default() -> Self { Self { + #[cfg(feature = "rocksdb")] account_tree: RocksDbOptions::default().into(), + #[cfg(feature = "rocksdb")] nullifier_tree: RocksDbOptions::default().into(), } } @@ -165,109 +166,19 @@ impl StorageOptions { /// /// These values were determined during development of `LargeSmt` pub fn bench() -> Self { - let account_tree = AccountTreeRocksDbOptions { - max_open_fds: 512, - cache_size_in_bytes: 2 << 30, - }; - let nullifier_tree = NullifierTreeRocksDbOptions { - max_open_fds: 512, - cache_size_in_bytes: 2 << 30, - }; - Self { account_tree, nullifier_tree } - } -} - -/// Per usage options for rocksdb configuration -#[cfg(feature = "rocksdb")] -#[derive(clap::Args, Clone, Debug, PartialEq, Eq)] -pub struct NullifierTreeRocksDbOptions { - #[arg( - long = "nullifier_tree.rocksdb.max_open_fds", - default_value_t = 64, - value_name = "NULLIFIER_TREE__ROCKSDB__MAX_OPEN_FDS" - )] - pub max_open_fds: i32, - #[arg( - long = "nullifier_tree.rocksdb.max_cache_size", - default_value_t = 2 * 1024 * 1024 * 1024, - value_name = "NULLIFIER_TREE__ROCKSDB__CACHE_SIZE" - )] - pub cache_size_in_bytes: usize, -} - -/// Per usage options for rocksdb configuration -#[cfg(feature = "rocksdb")] -#[derive(clap::Args, Clone, Debug, PartialEq, Eq)] -pub struct AccountTreeRocksDbOptions { - #[arg( - long = "account_tree.rocksdb.max_open_fds", - default_value_t = 64, - value_name = "ACCOUNT_TREE__ROCKSDB__MAX_OPEN_FDS" - )] - pub max_open_fds: i32, - #[arg( - long = "account_tree.rocksdb.max_cache_size", - default_value_t = 2<<30, - value_name = "ACCOUNT_TREE__ROCKSDB__CACHE_SIZE" - )] - pub cache_size_in_bytes: usize, -} - -/// General confiration options for rocksdb. -#[cfg(feature = "rocksdb")] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct RocksDbOptions { - pub max_open_fds: i32, - pub cache_size_in_bytes: usize, -} - -#[cfg(feature = "rocksdb")] -impl Default for RocksDbOptions { - fn default() -> Self { - Self { - max_open_fds: 512, - cache_size_in_bytes: 2 << 30, + #[cfg(feature = "rocksdb")] + { + let account_tree = AccountTreeRocksDbOptions { + max_open_fds: self::rocksdb::BENCH_ROCKSDB_MAX_OPEN_FDS, + cache_size_in_bytes: self::rocksdb::DEFAULT_ROCKSDB_CACHE_SIZE, + }; + let nullifier_tree = NullifierTreeRocksDbOptions { + max_open_fds: BENCH_ROCKSDB_MAX_OPEN_FDS, + cache_size_in_bytes: DEFAULT_ROCKSDB_CACHE_SIZE, + }; + Self { account_tree, nullifier_tree } } - } -} - -#[cfg(feature = "rocksdb")] -impl From for RocksDbOptions { - fn from(value: AccountTreeRocksDbOptions) -> Self { - let AccountTreeRocksDbOptions { max_open_fds, cache_size_in_bytes } = value; - Self { max_open_fds, cache_size_in_bytes } - } -} - -#[cfg(feature = "rocksdb")] -impl From for RocksDbOptions { - fn from(value: NullifierTreeRocksDbOptions) -> Self { - let NullifierTreeRocksDbOptions { max_open_fds, cache_size_in_bytes } = value; - Self { max_open_fds, cache_size_in_bytes } - } -} - -#[cfg(feature = "rocksdb")] -impl From for AccountTreeRocksDbOptions { - fn from(value: RocksDbOptions) -> Self { - let RocksDbOptions { max_open_fds, cache_size_in_bytes } = value; - Self { max_open_fds, cache_size_in_bytes } - } -} - -#[cfg(feature = "rocksdb")] -impl From for NullifierTreeRocksDbOptions { - fn from(value: RocksDbOptions) -> Self { - let RocksDbOptions { max_open_fds, cache_size_in_bytes } = value; - Self { max_open_fds, cache_size_in_bytes } - } -} - -#[cfg(feature = "rocksdb")] -impl RocksDbOptions { - pub fn with_path(self, path: &Path) -> RocksDbConfig { - RocksDbConfig::new(path) - .with_cache_size(self.cache_size_in_bytes) - .with_max_open_files(self.max_open_fds) + #[cfg(not(feature = "rocksdb"))] + Self::default() } } diff --git a/crates/utils/src/clap/rocksdb.rs b/crates/utils/src/clap/rocksdb.rs new file mode 100644 index 0000000000..bab45c1ef8 --- /dev/null +++ b/crates/utils/src/clap/rocksdb.rs @@ -0,0 +1,95 @@ +//! RocksDB storage backend specific CLI argument parsing and types. + +use std::path::Path; + +use miden_large_smt_backend_rocksdb::RocksDbConfig; + +pub(crate) const DEFAULT_ROCKSDB_MAX_OPEN_FDS: i32 = 64; +pub(crate) const DEFAULT_ROCKSDB_CACHE_SIZE: usize = 2 << 30; +pub(crate) const BENCH_ROCKSDB_MAX_OPEN_FDS: i32 = 512; + +/// Per usage options for rocksdb configuration +#[derive(clap::Args, Clone, Debug, PartialEq, Eq)] +pub struct NullifierTreeRocksDbOptions { + #[arg( + long = "nullifier_tree.rocksdb.max_open_fds", + default_value_t = DEFAULT_ROCKSDB_MAX_OPEN_FDS, + value_name = "NULLIFIER_TREE__ROCKSDB__MAX_OPEN_FDS" + )] + pub max_open_fds: i32, + #[arg( + long = "nullifier_tree.rocksdb.max_cache_size", + default_value_t = DEFAULT_ROCKSDB_CACHE_SIZE, + value_name = "NULLIFIER_TREE__ROCKSDB__CACHE_SIZE" + )] + pub cache_size_in_bytes: usize, +} + +/// Per usage options for rocksdb configuration +#[derive(clap::Args, Clone, Debug, PartialEq, Eq)] +pub struct AccountTreeRocksDbOptions { + #[arg( + long = "account_tree.rocksdb.max_open_fds", + default_value_t = DEFAULT_ROCKSDB_MAX_OPEN_FDS, + value_name = "ACCOUNT_TREE__ROCKSDB__MAX_OPEN_FDS" + )] + pub max_open_fds: i32, + #[arg( + long = "account_tree.rocksdb.max_cache_size", + default_value_t = DEFAULT_ROCKSDB_CACHE_SIZE, + value_name = "ACCOUNT_TREE__ROCKSDB__CACHE_SIZE" + )] + pub cache_size_in_bytes: usize, +} + +/// General confiration options for rocksdb. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RocksDbOptions { + pub max_open_fds: i32, + pub cache_size_in_bytes: usize, +} + +impl Default for RocksDbOptions { + fn default() -> Self { + Self { + max_open_fds: DEFAULT_ROCKSDB_MAX_OPEN_FDS, + cache_size_in_bytes: DEFAULT_ROCKSDB_CACHE_SIZE, + } + } +} + +impl From for RocksDbOptions { + fn from(value: AccountTreeRocksDbOptions) -> Self { + let AccountTreeRocksDbOptions { max_open_fds, cache_size_in_bytes } = value; + Self { max_open_fds, cache_size_in_bytes } + } +} + +impl From for RocksDbOptions { + fn from(value: NullifierTreeRocksDbOptions) -> Self { + let NullifierTreeRocksDbOptions { max_open_fds, cache_size_in_bytes } = value; + Self { max_open_fds, cache_size_in_bytes } + } +} + +impl From for AccountTreeRocksDbOptions { + fn from(value: RocksDbOptions) -> Self { + let RocksDbOptions { max_open_fds, cache_size_in_bytes } = value; + Self { max_open_fds, cache_size_in_bytes } + } +} + +impl From for NullifierTreeRocksDbOptions { + fn from(value: RocksDbOptions) -> Self { + let RocksDbOptions { max_open_fds, cache_size_in_bytes } = value; + Self { max_open_fds, cache_size_in_bytes } + } +} + +impl RocksDbOptions { + pub fn with_path(self, path: &Path) -> RocksDbConfig { + RocksDbConfig::new(path) + .with_cache_size(self.cache_size_in_bytes) + .with_max_open_files(self.max_open_fds) + } +} From 7ead823ba633ebe31f65f4a334e9ebd3d68c2105 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 Mar 2026 17:52:10 +0100 Subject: [PATCH 07/12] fix tests --- crates/block-producer/src/server/tests.rs | 3 ++- crates/rpc/src/tests.rs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index d51d6aa6e3..113c42b787 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -3,7 +3,7 @@ use std::time::Duration; use miden_node_proto::generated::block_producer::api_client as block_producer_client; use miden_node_store::{GenesisState, Store}; -use miden_node_utils::clap::GrpcOptionsInternal; +use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions}; use miden_node_utils::fee::test_fee_params; use miden_node_validator::{Validator, ValidatorSigner}; use miden_protocol::testing::random_secret_key::random_secret_key; @@ -156,6 +156,7 @@ async fn start_store( block_prover_url: None, data_directory: dir, grpc_options: GrpcOptionsInternal::bench(), + storage_options: StorageOptions::bench(), } .serve() .await diff --git a/crates/rpc/src/tests.rs b/crates/rpc/src/tests.rs index 4e2b4a5074..24f7e6054e 100644 --- a/crates/rpc/src/tests.rs +++ b/crates/rpc/src/tests.rs @@ -9,7 +9,7 @@ use miden_node_proto::generated::rpc::api_client::ApiClient as ProtoClient; use miden_node_proto::generated::{self as proto}; use miden_node_store::Store; use miden_node_store::genesis::config::GenesisConfig; -use miden_node_utils::clap::{GrpcOptionsExternal, GrpcOptionsInternal}; +use miden_node_utils::clap::{GrpcOptionsExternal, GrpcOptionsInternal, StorageOptions}; use miden_node_utils::fee::test_fee; use miden_node_utils::limiter::{ QueryParamAccountIdLimit, @@ -477,6 +477,7 @@ async fn start_store(store_listener: TcpListener) -> (Runtime, TempDir, Word, So block_producer_listener, data_directory: dir, grpc_options: GrpcOptionsInternal::test(), + storage_options: StorageOptions::default(), } .serve() .await @@ -519,6 +520,7 @@ async fn restart_store(store_addr: SocketAddr, data_directory: &std::path::Path) block_producer_listener, data_directory: dir, grpc_options: GrpcOptionsInternal::test(), + storage_options: StorageOptions::default(), } .serve() .await From 4cd81f24406caa59d0bee787a61e86e74bfe0cd8 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 Mar 2026 21:45:30 +0100 Subject: [PATCH 08/12] broken link --- docs/internal/src/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internal/src/store.md b/docs/internal/src/store.md index 11b85e3785..a2b06a76d2 100644 --- a/docs/internal/src/store.md +++ b/docs/internal/src/store.md @@ -27,7 +27,7 @@ bits vary by depth (8.0–12.0) and memtables are 128 MiB per CF. See `RocksDbSt full fixed configuration. Runtime-tuneable parameters (`--{account,nullifier}_tree.rocksdb.{max_cache_size,max_open_fds}`) -are documented in the [operator usage guide](../../../external/src/operator/usage.md). +are documented in the operator usage guide (`docs/external/src/operator/usage.md`). ## Architecture From e64eff0f533a1556a7b517f7e5d72e2e295f373c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 Mar 2026 21:46:29 +0100 Subject: [PATCH 09/12] fixup book --- docs/internal/src/store.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/internal/src/store.md b/docs/internal/src/store.md index a2b06a76d2..1e88bcafde 100644 --- a/docs/internal/src/store.md +++ b/docs/internal/src/store.md @@ -24,10 +24,9 @@ depth), `metadata` (root/counts), `depth24` (cached depth-24 hashes for fast sta Compaction parallelism and background jobs are set to `rayon::current_num_threads()` automatically. WAL sync per write is disabled for throughput; a 512 MiB WAL cap bounds recovery time. Bloom filter bits vary by depth (8.0–12.0) and memtables are 128 MiB per CF. See `RocksDbStorage::open` for the -full fixed configuration. +full fixed configuration. Runtime-tuneable parameters are documented in the +[operator usage guide](https://github.com/0xMiden/node/blob/next/docs/external/src/operator/usage.md#rocksdb-tuning). -Runtime-tuneable parameters (`--{account,nullifier}_tree.rocksdb.{max_cache_size,max_open_fds}`) -are documented in the operator usage guide (`docs/external/src/operator/usage.md`). ## Architecture From 0f8a94c65f1662fe001f6cf5b3109cc83fac37fa Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 12 Mar 2026 15:23:39 +0100 Subject: [PATCH 10/12] address review --- crates/utils/src/clap.rs | 1 + docs/internal/src/store.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/utils/src/clap.rs b/crates/utils/src/clap.rs index 8201f79ef8..421b39bf44 100644 --- a/crates/utils/src/clap.rs +++ b/crates/utils/src/clap.rs @@ -5,6 +5,7 @@ use std::time::Duration; #[cfg(feature = "rocksdb")] mod rocksdb; +#[cfg(feature = "rocksdb")] pub use rocksdb::*; const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); diff --git a/docs/internal/src/store.md b/docs/internal/src/store.md index 1e88bcafde..67ad1d4d9b 100644 --- a/docs/internal/src/store.md +++ b/docs/internal/src/store.md @@ -23,7 +23,7 @@ depth), `metadata` (root/counts), `depth24` (cached depth-24 hashes for fast sta Compaction parallelism and background jobs are set to `rayon::current_num_threads()` automatically. WAL sync per write is disabled for throughput; a 512 MiB WAL cap bounds recovery time. Bloom filter -bits vary by depth (8.0–12.0) and memtables are 128 MiB per CF. See `RocksDbStorage::open` for the +bits vary by depth (8.0–12.0) and memtables are 128 MiB per column family. See `RocksDbStorage::open` for the full fixed configuration. Runtime-tuneable parameters are documented in the [operator usage guide](https://github.com/0xMiden/node/blob/next/docs/external/src/operator/usage.md#rocksdb-tuning). From 0f626366964683081863cc992b445ffd066c8365 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 12 Mar 2026 16:09:05 +0100 Subject: [PATCH 11/12] yes --- crates/utils/src/clap/rocksdb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/utils/src/clap/rocksdb.rs b/crates/utils/src/clap/rocksdb.rs index bab45c1ef8..6e731971f1 100644 --- a/crates/utils/src/clap/rocksdb.rs +++ b/crates/utils/src/clap/rocksdb.rs @@ -1,4 +1,4 @@ -//! RocksDB storage backend specific CLI argument parsing and types. +//! `RocksDB` storage backend specific CLI argument parsing and types. use std::path::Path; From 3602e972da6079ff44e8d276dff70df237415f50 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 12 Mar 2026 16:26:34 +0100 Subject: [PATCH 12/12] default impl changes --- bin/node/src/commands/bundled.rs | 2 +- bin/stress-test/src/store/mod.rs | 5 ++++- crates/utils/src/clap.rs | 13 +------------ crates/utils/src/clap/rocksdb.rs | 12 ++++++++++++ 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/bin/node/src/commands/bundled.rs b/bin/node/src/commands/bundled.rs index 2ec2088aae..2db11c0652 100644 --- a/bin/node/src/commands/bundled.rs +++ b/bin/node/src/commands/bundled.rs @@ -141,7 +141,7 @@ impl BundledCommand { } } - #[expect(clippy::too_many_lines)] + #[expect(clippy::too_many_lines, clippy::too_many_arguments)] async fn start( rpc_url: Url, block_prover_url: Option, diff --git a/bin/stress-test/src/store/mod.rs b/bin/stress-test/src/store/mod.rs index d973ca1f0e..f1ea6e0f77 100644 --- a/bin/stress-test/src/store/mod.rs +++ b/bin/stress-test/src/store/mod.rs @@ -5,6 +5,7 @@ use futures::{StreamExt, stream}; use miden_node_proto::generated::store::rpc_client::RpcClient; use miden_node_proto::generated::{self as proto}; use miden_node_store::state::State; +use miden_node_utils::clap::StorageOptions; use miden_node_utils::tracing::grpc::OtelInterceptor; use miden_protocol::account::AccountId; use miden_protocol::note::{NoteDetails, NoteTag}; @@ -490,7 +491,9 @@ struct SyncChainMmrRun { pub async fn load_state(data_directory: &Path) { let start = Instant::now(); let (termination_ask, _) = tokio::sync::mpsc::channel(1); - let _state = State::load(data_directory, Default::default(), termination_ask).await.unwrap(); + let _state = State::load(data_directory, StorageOptions::default(), termination_ask) + .await + .unwrap(); let elapsed = start.elapsed(); // Get database path and run SQL commands to count records diff --git a/crates/utils/src/clap.rs b/crates/utils/src/clap.rs index 421b39bf44..079a619d35 100644 --- a/crates/utils/src/clap.rs +++ b/crates/utils/src/clap.rs @@ -141,7 +141,7 @@ impl GrpcOptionsExternal { /// Collection of per usage storage backend configurations. /// /// Note: Currently only contains `rocksdb` related configuration. -#[derive(clap::Args, Clone, Debug, PartialEq, Eq)] +#[derive(clap::Args, Clone, Debug, Default, PartialEq, Eq)] pub struct StorageOptions { #[cfg(feature = "rocksdb")] #[clap(flatten)] @@ -151,17 +151,6 @@ pub struct StorageOptions { pub nullifier_tree: NullifierTreeRocksDbOptions, } -impl Default for StorageOptions { - fn default() -> Self { - Self { - #[cfg(feature = "rocksdb")] - account_tree: RocksDbOptions::default().into(), - #[cfg(feature = "rocksdb")] - nullifier_tree: RocksDbOptions::default().into(), - } - } -} - impl StorageOptions { /// Benchmark setup. /// diff --git a/crates/utils/src/clap/rocksdb.rs b/crates/utils/src/clap/rocksdb.rs index 6e731971f1..9b752205d2 100644 --- a/crates/utils/src/clap/rocksdb.rs +++ b/crates/utils/src/clap/rocksdb.rs @@ -25,6 +25,12 @@ pub struct NullifierTreeRocksDbOptions { pub cache_size_in_bytes: usize, } +impl Default for NullifierTreeRocksDbOptions { + fn default() -> Self { + RocksDbOptions::default().into() + } +} + /// Per usage options for rocksdb configuration #[derive(clap::Args, Clone, Debug, PartialEq, Eq)] pub struct AccountTreeRocksDbOptions { @@ -42,6 +48,12 @@ pub struct AccountTreeRocksDbOptions { pub cache_size_in_bytes: usize, } +impl Default for AccountTreeRocksDbOptions { + fn default() -> Self { + RocksDbOptions::default().into() + } +} + /// General confiration options for rocksdb. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct RocksDbOptions {