feat/rocksdb: expose rocksdb config per LargeSmt tree

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Enhancements
 
+- Expose per-tree RocksDB tuning options ([#1782](https://github.com/0xMiden/node/pull/1782)).
 - Added verbose `info!`-level logging to the network transaction builder for transaction execution, note filtering failures, and transaction outcomes ([#1770](https://github.com/0xMiden/node/pull/1770)).
 - [BREAKING] Move block proving from Blocker Producer to the Store ([#1579](https://github.com/0xMiden/node/pull/1579)).
 - [BREAKING] Updated miden-base dependencies to use `next` branch; renamed `NoteInputs` to `NoteStorage`, `.inputs()` to `.storage()`, and database `inputs` column to `storage` ([#1595](https://github.com/0xMiden/node/pull/1595)).

bin/node/src/commands/bundled.rs

@@ -5,7 +5,7 @@ use anyhow::Context;
 use miden_node_block_producer::BlockProducer;
 use miden_node_rpc::Rpc;
 use miden_node_store::Store;
-use miden_node_utils::clap::GrpcOptionsExternal;
+use miden_node_utils::clap::{GrpcOptionsExternal, StorageOptions};
 use miden_node_utils::grpc::UrlExt;
 use miden_node_validator::{Validator, ValidatorSigner};
 use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SecretKey;
@@ -84,6 +84,9 @@ pub enum BundledCommand {
 
         #[command(flatten)]
         grpc_options: GrpcOptionsExternal,
+
+        #[command(flatten)]
+        storage_options: StorageOptions,
     },
 }
 
@@ -121,6 +124,7 @@ impl BundledCommand {
                 validator,
                 enable_otel: _,
                 grpc_options,
+                storage_options,
             } => {
                 Self::start(
                     rpc_url,
@@ -130,13 +134,14 @@ impl BundledCommand {
                     ntx_builder,
                     validator,
                     grpc_options,
+                    storage_options,
                 )
                 .await
             },
         }
     }
 
-    #[expect(clippy::too_many_lines)]
+    #[expect(clippy::too_many_lines, clippy::too_many_arguments)]
     async fn start(
         rpc_url: Url,
         block_prover_url: Option<Url>,
@@ -145,6 +150,7 @@ impl BundledCommand {
         ntx_builder: NtxBuilderConfig,
         validator: BundledValidatorConfig,
         grpc_options: GrpcOptionsExternal,
+        storage_options: StorageOptions,
     ) -> anyhow::Result<()> {
         // Start listening on all gRPC urls so that inter-component connections can be created
         // before each component is fully started up.
@@ -202,6 +208,7 @@ impl BundledCommand {
                     data_directory: data_directory_clone,
                     block_prover_url,
                     grpc_options: grpc_options.into(),
+                    storage_options,
                 }
                 .serve()
                 .await

bin/node/src/commands/store.rs

@@ -3,7 +3,7 @@ use std::path::{Path, PathBuf};
 use anyhow::Context;
 use miden_node_store::Store;
 use miden_node_store::genesis::GenesisBlock;
-use miden_node_utils::clap::GrpcOptionsInternal;
+use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions};
 use miden_node_utils::fs::ensure_empty_directory;
 use miden_node_utils::grpc::UrlExt;
 use miden_protocol::block::ProvenBlock;
@@ -67,6 +67,9 @@ pub enum StoreCommand {
 
         #[command(flatten)]
         grpc_options: GrpcOptionsInternal,
+
+        #[command(flatten)]
+        storage_options: StorageOptions,
     },
 }
 
@@ -86,6 +89,7 @@ impl StoreCommand {
                 data_directory,
                 enable_otel: _,
                 grpc_options,
+                storage_options,
             } => {
                 Self::start(
                     rpc_url,
@@ -94,6 +98,7 @@ impl StoreCommand {
                     block_prover_url,
                     data_directory,
                     grpc_options,
+                    storage_options,
                 )
                 .await
             },
@@ -115,6 +120,7 @@ impl StoreCommand {
         block_prover_url: Option<Url>,
         data_directory: PathBuf,
         grpc_options: GrpcOptionsInternal,
+        storage_options: StorageOptions,
     ) -> anyhow::Result<()> {
         let rpc_listener = rpc_url
             .to_socket()
@@ -144,6 +150,7 @@ impl StoreCommand {
             block_producer_listener,
             data_directory,
             grpc_options,
+            storage_options,
         }
         .serve()
         .await

bin/stress-test/src/seeding/mod.rs

@@ -9,7 +9,7 @@ use miden_node_block_producer::store::StoreClient;
 use miden_node_proto::domain::batch::BatchInputs;
 use miden_node_proto::generated::store::rpc_client::RpcClient;
 use miden_node_store::{DataDirectory, GenesisState, Store};
-use miden_node_utils::clap::GrpcOptionsInternal;
+use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions};
 use miden_node_utils::tracing::grpc::OtelInterceptor;
 use miden_protocol::account::auth::AuthScheme;
 use miden_protocol::account::delta::AccountUpdateDetails;
@@ -555,6 +555,7 @@ pub async fn start_store(
             block_producer_listener,
             data_directory: dir,
             grpc_options: GrpcOptionsInternal::bench(),
+            storage_options: StorageOptions::bench(),
         }
         .serve()
         .await

bin/stress-test/src/store/mod.rs

@@ -5,6 +5,7 @@ use futures::{StreamExt, stream};
 use miden_node_proto::generated::store::rpc_client::RpcClient;
 use miden_node_proto::generated::{self as proto};
 use miden_node_store::state::State;
+use miden_node_utils::clap::StorageOptions;
 use miden_node_utils::tracing::grpc::OtelInterceptor;
 use miden_protocol::account::AccountId;
 use miden_protocol::note::{NoteDetails, NoteTag};
@@ -490,7 +491,9 @@ struct SyncChainMmrRun {
 pub async fn load_state(data_directory: &Path) {
     let start = Instant::now();
     let (termination_ask, _) = tokio::sync::mpsc::channel(1);
-    let _state = State::load(data_directory, termination_ask).await.unwrap();
+    let _state = State::load(data_directory, StorageOptions::default(), termination_ask)
+        .await
+        .unwrap();
     let elapsed = start.elapsed();
 
     // Get database path and run SQL commands to count records

crates/block-producer/src/server/tests.rs

@@ -3,7 +3,7 @@ use std::time::Duration;
 
 use miden_node_proto::generated::block_producer::api_client as block_producer_client;
 use miden_node_store::{GenesisState, Store};
-use miden_node_utils::clap::GrpcOptionsInternal;
+use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions};
 use miden_node_utils::fee::test_fee_params;
 use miden_node_validator::{Validator, ValidatorSigner};
 use miden_protocol::testing::random_secret_key::random_secret_key;
@@ -159,6 +159,7 @@ async fn start_store(
             block_prover_url: None,
             data_directory: dir,
             grpc_options: GrpcOptionsInternal::bench(),
+            storage_options: StorageOptions::bench(),
         }
         .serve()
         .await

crates/rpc/src/tests.rs

@@ -9,7 +9,7 @@ use miden_node_proto::generated::rpc::api_client::ApiClient as ProtoClient;
 use miden_node_proto::generated::{self as proto};
 use miden_node_store::Store;
 use miden_node_store::genesis::config::GenesisConfig;
-use miden_node_utils::clap::{GrpcOptionsExternal, GrpcOptionsInternal};
+use miden_node_utils::clap::{GrpcOptionsExternal, GrpcOptionsInternal, StorageOptions};
 use miden_node_utils::fee::test_fee;
 use miden_node_utils::limiter::{
     QueryParamAccountIdLimit,
@@ -480,6 +480,7 @@ async fn start_store(store_listener: TcpListener) -> (Runtime, TempDir, Word, So
             block_producer_listener,
             data_directory: dir,
             grpc_options: GrpcOptionsInternal::test(),
+            storage_options: StorageOptions::default(),
         }
         .serve()
         .await
@@ -522,6 +523,7 @@ async fn restart_store(store_addr: SocketAddr, data_directory: &std::path::Path)
             block_producer_listener,
             data_directory: dir,
             grpc_options: GrpcOptionsInternal::test(),
+            storage_options: StorageOptions::default(),
         }
         .serve()
         .await

crates/store/Cargo.toml

@@ -72,7 +72,7 @@ termtree              = "1.0"
 
 [features]
 default = ["rocksdb"]
-rocksdb = ["dep:miden-large-smt-backend-rocksdb"]
+rocksdb = ["dep:miden-large-smt-backend-rocksdb", "miden-node-utils/rocksdb"]
 
 [[bench]]
 harness           = false

crates/store/src/server/mod.rs

@@ -10,7 +10,7 @@ use miden_node_proto_build::{
     store_ntx_builder_api_descriptor,
     store_rpc_api_descriptor,
 };
-use miden_node_utils::clap::GrpcOptionsInternal;
+use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions};
 use miden_node_utils::panic::{CatchPanicLayer, catch_panic_layer_fn};
 use miden_node_utils::tracing::grpc::grpc_trace_fn;
 use tokio::net::TcpListener;
@@ -41,6 +41,7 @@ pub struct Store {
     /// URL for the Block Prover client. Uses local prover if `None`.
     pub block_prover_url: Option<Url>,
     pub data_directory: PathBuf,
+    pub storage_options: StorageOptions,
     pub grpc_options: GrpcOptionsInternal,
 }
 
@@ -90,7 +91,7 @@ impl Store {
         let (termination_ask, mut termination_signal) =
             tokio::sync::mpsc::channel::<ApplyBlockError>(1);
         let state = Arc::new(
-            State::load(&self.data_directory, termination_ask)
+            State::load(&self.data_directory, self.storage_options, termination_ask)
                 .await
                 .context("failed to load state")?,
         );

crates/store/src/state/loader.rs

@@ -14,7 +14,8 @@ use std::path::Path;
 
 use miden_crypto::merkle::mmr::Mmr;
 #[cfg(feature = "rocksdb")]
-use miden_large_smt_backend_rocksdb::{RocksDbConfig, RocksDbStorage};
+use miden_large_smt_backend_rocksdb::RocksDbStorage;
+use miden_node_utils::clap::RocksDbOptions;
 use miden_protocol::block::account_tree::{AccountTree, account_id_to_smt_key};
 use miden_protocol::block::nullifier_tree::NullifierTree;
 use miden_protocol::block::{BlockNumber, Blockchain};
@@ -98,8 +99,14 @@ fn block_num_to_nullifier_leaf(block_num: BlockNumber) -> Word {
 /// which detects divergence between persistent storage and the database. If divergence is detected,
 /// the user should manually delete the tree storage directories and restart the node.
 pub trait StorageLoader: SmtStorage + Sized {
+    /// A configuration type for the implementation.
+    type Config: std::fmt::Debug + std::default::Default;
     /// Creates a storage backend for the given domain.
-    fn create(data_dir: &Path, domain: &'static str) -> Result<Self, StateInitializationError>;
+    fn create(
+        data_dir: &Path,
+        storage_options: &Self::Config,
+        domain: &'static str,
+    ) -> Result<Self, StateInitializationError>;
 
     /// Loads an account tree, either from persistent storage or by rebuilding from DB.
     fn load_account_tree(
@@ -119,7 +126,12 @@ pub trait StorageLoader: SmtStorage + Sized {
 
 #[cfg(not(feature = "rocksdb"))]
 impl StorageLoader for MemoryStorage {
-    fn create(_data_dir: &Path, _domain: &'static str) -> Result<Self, StateInitializationError> {
+    type Config = ();
+    fn create(
+        _data_dir: &Path,
+        _storage_options: &Self::Config,
+        _domain: &'static str,
+    ) -> Result<Self, StateInitializationError> {
         Ok(MemoryStorage::default())
     }
 
@@ -207,12 +219,17 @@ impl StorageLoader for MemoryStorage {
 
 #[cfg(feature = "rocksdb")]
 impl StorageLoader for RocksDbStorage {
-    fn create(data_dir: &Path, domain: &'static str) -> Result<Self, StateInitializationError> {
+    type Config = RocksDbOptions;
+    fn create(
+        data_dir: &Path,
+        storage_options: &Self::Config,
+        domain: &'static str,
+    ) -> Result<Self, StateInitializationError> {
         let storage_path = data_dir.join(domain);
-
+        let config = storage_options.with_path(&storage_path);
         fs_err::create_dir_all(&storage_path)
             .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string()))?;
-        RocksDbStorage::open(RocksDbConfig::new(storage_path))
+        RocksDbStorage::open(config)
             .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string()))
     }
 

crates/store/src/state/mod.rs

@@ -21,6 +21,7 @@ use miden_node_proto::domain::account::{
     StorageMapRequest,
 };
 use miden_node_proto::domain::batch::BatchInputs;
+use miden_node_utils::clap::StorageOptions;
 use miden_node_utils::formatting::format_array;
 use miden_node_utils::limiter::{QueryParamLimiter, QueryParamStorageMapKeyTotalLimit};
 use miden_protocol::Word;
@@ -134,6 +135,7 @@ impl State {
     #[instrument(target = COMPONENT, skip_all)]
     pub async fn load(
         data_path: &Path,
+        storage_options: StorageOptions,
         termination_ask: tokio::sync::mpsc::Sender<ApplyBlockError>,
     ) -> Result<Self, StateInitializationError> {
         let data_directory = DataDirectory::load(data_path.to_path_buf())
@@ -152,10 +154,18 @@ impl State {
         let blockchain = load_mmr(&mut db).await?;
         let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS);
 
-        let account_storage = TreeStorage::create(data_path, ACCOUNT_TREE_STORAGE_DIR)?;
+        let account_storage = TreeStorage::create(
+            data_path,
+            &storage_options.account_tree.into(),
+            ACCOUNT_TREE_STORAGE_DIR,
+        )?;
         let account_tree = account_storage.load_account_tree(&mut db).await?;
 
-        let nullifier_storage = TreeStorage::create(data_path, NULLIFIER_TREE_STORAGE_DIR)?;
+        let nullifier_storage = TreeStorage::create(
+            data_path,
+            &storage_options.nullifier_tree.into(),
+            NULLIFIER_TREE_STORAGE_DIR,
+        )?;
         let nullifier_tree = nullifier_storage.load_nullifier_tree(&mut db).await?;
 
         // Verify that tree roots match the expected roots from the database.

crates/utils/Cargo.toml

@@ -16,6 +16,7 @@ workspace = true
 
 [features]
 # Enables utility functions for testing traces created by some other crate's stack.
+rocksdb = ["dep:miden-large-smt-backend-rocksdb"]
 testing = ["miden-protocol/testing"]
 
 [dependencies]
@@ -46,5 +47,8 @@ tracing-opentelemetry = { version = "0.32" }
 tracing-subscriber    = { workspace = true }
 url                   = { workspace = true }
 
+# RocksDbConfig is needed due to orphan rules
+miden-large-smt-backend-rocksdb = { optional = true, workspace = true }
+
 [dev-dependencies]
 thiserror = { workspace = true }