diff --git a/CHANGELOG.md b/CHANGELOG.md index 393ec26801..7cedd8c0e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Enhancements +- Expose per-tree RocksDB tuning options ([#1782](https://github.com/0xMiden/node/pull/1782)). - Added verbose `info!`-level logging to the network transaction builder for transaction execution, note filtering failures, and transaction outcomes ([#1770](https://github.com/0xMiden/node/pull/1770)). - [BREAKING] Move block proving from Blocker Producer to the Store ([#1579](https://github.com/0xMiden/node/pull/1579)). - [BREAKING] Updated miden-base dependencies to use `next` branch; renamed `NoteInputs` to `NoteStorage`, `.inputs()` to `.storage()`, and database `inputs` column to `storage` ([#1595](https://github.com/0xMiden/node/pull/1595)). diff --git a/Cargo.lock b/Cargo.lock index ee430586f0..16ea17404b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3234,6 +3234,7 @@ dependencies = [ "humantime", "itertools 0.14.0", "lru", + "miden-large-smt-backend-rocksdb", "miden-protocol", "opentelemetry", "opentelemetry-otlp", diff --git a/bin/node/src/commands/bundled.rs b/bin/node/src/commands/bundled.rs index b52907966d..2db11c0652 100644 --- a/bin/node/src/commands/bundled.rs +++ b/bin/node/src/commands/bundled.rs @@ -5,7 +5,7 @@ use anyhow::Context; use miden_node_block_producer::BlockProducer; use miden_node_rpc::Rpc; use miden_node_store::Store; -use miden_node_utils::clap::GrpcOptionsExternal; +use miden_node_utils::clap::{GrpcOptionsExternal, StorageOptions}; use miden_node_utils::grpc::UrlExt; use miden_node_validator::{Validator, ValidatorSigner}; use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SecretKey; @@ -84,6 +84,9 @@ pub enum BundledCommand { #[command(flatten)] grpc_options: GrpcOptionsExternal, + + #[command(flatten)] + storage_options: StorageOptions, }, } @@ -121,6 +124,7 @@ impl BundledCommand { validator, enable_otel: _, grpc_options, + storage_options, } => { Self::start( rpc_url, @@ -130,13 +134,14 @@ impl BundledCommand { ntx_builder, validator, grpc_options, + storage_options, ) .await }, } } - #[expect(clippy::too_many_lines)] + #[expect(clippy::too_many_lines, clippy::too_many_arguments)] async fn start( rpc_url: Url, block_prover_url: Option, @@ -145,6 +150,7 @@ impl BundledCommand { ntx_builder: NtxBuilderConfig, validator: BundledValidatorConfig, grpc_options: GrpcOptionsExternal, + storage_options: StorageOptions, ) -> anyhow::Result<()> { // Start listening on all gRPC urls so that inter-component connections can be created // before each component is fully started up. @@ -202,6 +208,7 @@ impl BundledCommand { data_directory: data_directory_clone, block_prover_url, grpc_options: grpc_options.into(), + storage_options, } .serve() .await diff --git a/bin/node/src/commands/store.rs b/bin/node/src/commands/store.rs index 39c8fad73c..53ea3dae96 100644 --- a/bin/node/src/commands/store.rs +++ b/bin/node/src/commands/store.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use anyhow::Context; use miden_node_store::Store; use miden_node_store::genesis::GenesisBlock; -use miden_node_utils::clap::GrpcOptionsInternal; +use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions}; use miden_node_utils::fs::ensure_empty_directory; use miden_node_utils::grpc::UrlExt; use miden_protocol::block::ProvenBlock; @@ -67,6 +67,9 @@ pub enum StoreCommand { #[command(flatten)] grpc_options: GrpcOptionsInternal, + + #[command(flatten)] + storage_options: StorageOptions, }, } @@ -86,6 +89,7 @@ impl StoreCommand { data_directory, enable_otel: _, grpc_options, + storage_options, } => { Self::start( rpc_url, @@ -94,6 +98,7 @@ impl StoreCommand { block_prover_url, data_directory, grpc_options, + storage_options, ) .await }, @@ -115,6 +120,7 @@ impl StoreCommand { block_prover_url: Option, data_directory: PathBuf, grpc_options: GrpcOptionsInternal, + storage_options: StorageOptions, ) -> anyhow::Result<()> { let rpc_listener = rpc_url .to_socket() @@ -144,6 +150,7 @@ impl StoreCommand { block_producer_listener, data_directory, grpc_options, + storage_options, } .serve() .await diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index 8176fdf1bb..f6ba165109 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -9,7 +9,7 @@ use miden_node_block_producer::store::StoreClient; use miden_node_proto::domain::batch::BatchInputs; use miden_node_proto::generated::store::rpc_client::RpcClient; use miden_node_store::{DataDirectory, GenesisState, Store}; -use miden_node_utils::clap::GrpcOptionsInternal; +use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions}; use miden_node_utils::tracing::grpc::OtelInterceptor; use miden_protocol::account::auth::AuthScheme; use miden_protocol::account::delta::AccountUpdateDetails; @@ -555,6 +555,7 @@ pub async fn start_store( block_producer_listener, data_directory: dir, grpc_options: GrpcOptionsInternal::bench(), + storage_options: StorageOptions::bench(), } .serve() .await diff --git a/bin/stress-test/src/store/mod.rs b/bin/stress-test/src/store/mod.rs index 314a5e95d0..f1ea6e0f77 100644 --- a/bin/stress-test/src/store/mod.rs +++ b/bin/stress-test/src/store/mod.rs @@ -5,6 +5,7 @@ use futures::{StreamExt, stream}; use miden_node_proto::generated::store::rpc_client::RpcClient; use miden_node_proto::generated::{self as proto}; use miden_node_store::state::State; +use miden_node_utils::clap::StorageOptions; use miden_node_utils::tracing::grpc::OtelInterceptor; use miden_protocol::account::AccountId; use miden_protocol::note::{NoteDetails, NoteTag}; @@ -490,7 +491,9 @@ struct SyncChainMmrRun { pub async fn load_state(data_directory: &Path) { let start = Instant::now(); let (termination_ask, _) = tokio::sync::mpsc::channel(1); - let _state = State::load(data_directory, termination_ask).await.unwrap(); + let _state = State::load(data_directory, StorageOptions::default(), termination_ask) + .await + .unwrap(); let elapsed = start.elapsed(); // Get database path and run SQL commands to count records diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index f0f3dcc175..3e5da20254 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -3,7 +3,7 @@ use std::time::Duration; use miden_node_proto::generated::block_producer::api_client as block_producer_client; use miden_node_store::{GenesisState, Store}; -use miden_node_utils::clap::GrpcOptionsInternal; +use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions}; use miden_node_utils::fee::test_fee_params; use miden_node_validator::{Validator, ValidatorSigner}; use miden_protocol::testing::random_secret_key::random_secret_key; @@ -159,6 +159,7 @@ async fn start_store( block_prover_url: None, data_directory: dir, grpc_options: GrpcOptionsInternal::bench(), + storage_options: StorageOptions::bench(), } .serve() .await diff --git a/crates/rpc/src/tests.rs b/crates/rpc/src/tests.rs index a54f44318d..89b7a23c4a 100644 --- a/crates/rpc/src/tests.rs +++ b/crates/rpc/src/tests.rs @@ -9,7 +9,7 @@ use miden_node_proto::generated::rpc::api_client::ApiClient as ProtoClient; use miden_node_proto::generated::{self as proto}; use miden_node_store::Store; use miden_node_store::genesis::config::GenesisConfig; -use miden_node_utils::clap::{GrpcOptionsExternal, GrpcOptionsInternal}; +use miden_node_utils::clap::{GrpcOptionsExternal, GrpcOptionsInternal, StorageOptions}; use miden_node_utils::fee::test_fee; use miden_node_utils::limiter::{ QueryParamAccountIdLimit, @@ -480,6 +480,7 @@ async fn start_store(store_listener: TcpListener) -> (Runtime, TempDir, Word, So block_producer_listener, data_directory: dir, grpc_options: GrpcOptionsInternal::test(), + storage_options: StorageOptions::default(), } .serve() .await @@ -522,6 +523,7 @@ async fn restart_store(store_addr: SocketAddr, data_directory: &std::path::Path) block_producer_listener, data_directory: dir, grpc_options: GrpcOptionsInternal::test(), + storage_options: StorageOptions::default(), } .serve() .await diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index f22555e10a..7c8135d97a 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -72,7 +72,7 @@ termtree = "1.0" [features] default = ["rocksdb"] -rocksdb = ["dep:miden-large-smt-backend-rocksdb"] +rocksdb = ["dep:miden-large-smt-backend-rocksdb", "miden-node-utils/rocksdb"] [[bench]] harness = false diff --git a/crates/store/src/server/mod.rs b/crates/store/src/server/mod.rs index 99b6806d15..d1b218be3f 100644 --- a/crates/store/src/server/mod.rs +++ b/crates/store/src/server/mod.rs @@ -10,7 +10,7 @@ use miden_node_proto_build::{ store_ntx_builder_api_descriptor, store_rpc_api_descriptor, }; -use miden_node_utils::clap::GrpcOptionsInternal; +use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions}; use miden_node_utils::panic::{CatchPanicLayer, catch_panic_layer_fn}; use miden_node_utils::tracing::grpc::grpc_trace_fn; use tokio::net::TcpListener; @@ -41,6 +41,7 @@ pub struct Store { /// URL for the Block Prover client. Uses local prover if `None`. pub block_prover_url: Option, pub data_directory: PathBuf, + pub storage_options: StorageOptions, pub grpc_options: GrpcOptionsInternal, } @@ -90,7 +91,7 @@ impl Store { let (termination_ask, mut termination_signal) = tokio::sync::mpsc::channel::(1); let state = Arc::new( - State::load(&self.data_directory, termination_ask) + State::load(&self.data_directory, self.storage_options, termination_ask) .await .context("failed to load state")?, ); diff --git a/crates/store/src/state/loader.rs b/crates/store/src/state/loader.rs index d632a1c0c4..20ef9cdc04 100644 --- a/crates/store/src/state/loader.rs +++ b/crates/store/src/state/loader.rs @@ -14,7 +14,8 @@ use std::path::Path; use miden_crypto::merkle::mmr::Mmr; #[cfg(feature = "rocksdb")] -use miden_large_smt_backend_rocksdb::{RocksDbConfig, RocksDbStorage}; +use miden_large_smt_backend_rocksdb::RocksDbStorage; +use miden_node_utils::clap::RocksDbOptions; use miden_protocol::block::account_tree::{AccountTree, account_id_to_smt_key}; use miden_protocol::block::nullifier_tree::NullifierTree; use miden_protocol::block::{BlockNumber, Blockchain}; @@ -98,8 +99,14 @@ fn block_num_to_nullifier_leaf(block_num: BlockNumber) -> Word { /// which detects divergence between persistent storage and the database. If divergence is detected, /// the user should manually delete the tree storage directories and restart the node. pub trait StorageLoader: SmtStorage + Sized { + /// A configuration type for the implementation. + type Config: std::fmt::Debug + std::default::Default; /// Creates a storage backend for the given domain. - fn create(data_dir: &Path, domain: &'static str) -> Result; + fn create( + data_dir: &Path, + storage_options: &Self::Config, + domain: &'static str, + ) -> Result; /// Loads an account tree, either from persistent storage or by rebuilding from DB. fn load_account_tree( @@ -119,7 +126,12 @@ pub trait StorageLoader: SmtStorage + Sized { #[cfg(not(feature = "rocksdb"))] impl StorageLoader for MemoryStorage { - fn create(_data_dir: &Path, _domain: &'static str) -> Result { + type Config = (); + fn create( + _data_dir: &Path, + _storage_options: &Self::Config, + _domain: &'static str, + ) -> Result { Ok(MemoryStorage::default()) } @@ -207,12 +219,17 @@ impl StorageLoader for MemoryStorage { #[cfg(feature = "rocksdb")] impl StorageLoader for RocksDbStorage { - fn create(data_dir: &Path, domain: &'static str) -> Result { + type Config = RocksDbOptions; + fn create( + data_dir: &Path, + storage_options: &Self::Config, + domain: &'static str, + ) -> Result { let storage_path = data_dir.join(domain); - + let config = storage_options.with_path(&storage_path); fs_err::create_dir_all(&storage_path) .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string()))?; - RocksDbStorage::open(RocksDbConfig::new(storage_path)) + RocksDbStorage::open(config) .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string())) } diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index a996e1956a..a511eee95d 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -21,6 +21,7 @@ use miden_node_proto::domain::account::{ StorageMapRequest, }; use miden_node_proto::domain::batch::BatchInputs; +use miden_node_utils::clap::StorageOptions; use miden_node_utils::formatting::format_array; use miden_node_utils::limiter::{QueryParamLimiter, QueryParamStorageMapKeyTotalLimit}; use miden_protocol::Word; @@ -134,6 +135,7 @@ impl State { #[instrument(target = COMPONENT, skip_all)] pub async fn load( data_path: &Path, + storage_options: StorageOptions, termination_ask: tokio::sync::mpsc::Sender, ) -> Result { let data_directory = DataDirectory::load(data_path.to_path_buf()) @@ -152,10 +154,18 @@ impl State { let blockchain = load_mmr(&mut db).await?; let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS); - let account_storage = TreeStorage::create(data_path, ACCOUNT_TREE_STORAGE_DIR)?; + let account_storage = TreeStorage::create( + data_path, + &storage_options.account_tree.into(), + ACCOUNT_TREE_STORAGE_DIR, + )?; let account_tree = account_storage.load_account_tree(&mut db).await?; - let nullifier_storage = TreeStorage::create(data_path, NULLIFIER_TREE_STORAGE_DIR)?; + let nullifier_storage = TreeStorage::create( + data_path, + &storage_options.nullifier_tree.into(), + NULLIFIER_TREE_STORAGE_DIR, + )?; let nullifier_tree = nullifier_storage.load_nullifier_tree(&mut db).await?; // Verify that tree roots match the expected roots from the database. diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 3cc2c0a34d..01f536ffee 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -16,6 +16,7 @@ workspace = true [features] # Enables utility functions for testing traces created by some other crate's stack. +rocksdb = ["dep:miden-large-smt-backend-rocksdb"] testing = ["miden-protocol/testing"] [dependencies] @@ -46,5 +47,8 @@ tracing-opentelemetry = { version = "0.32" } tracing-subscriber = { workspace = true } url = { workspace = true } +# RocksDbConfig is needed due to orphan rules +miden-large-smt-backend-rocksdb = { optional = true, workspace = true } + [dev-dependencies] thiserror = { workspace = true } diff --git a/crates/utils/src/clap.rs b/crates/utils/src/clap.rs index 0d85dce5ad..079a619d35 100644 --- a/crates/utils/src/clap.rs +++ b/crates/utils/src/clap.rs @@ -3,6 +3,11 @@ use std::num::{NonZeroU32, NonZeroU64}; use std::time::Duration; +#[cfg(feature = "rocksdb")] +mod rocksdb; +#[cfg(feature = "rocksdb")] +pub use rocksdb::*; + const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); const TEST_REQUEST_TIMEOUT: Duration = Duration::from_secs(5); const DEFAULT_MAX_CONNECTION_AGE: Duration = Duration::from_mins(30); @@ -132,3 +137,38 @@ impl GrpcOptionsExternal { } } } + +/// Collection of per usage storage backend configurations. +/// +/// Note: Currently only contains `rocksdb` related configuration. +#[derive(clap::Args, Clone, Debug, Default, PartialEq, Eq)] +pub struct StorageOptions { + #[cfg(feature = "rocksdb")] + #[clap(flatten)] + pub account_tree: AccountTreeRocksDbOptions, + #[cfg(feature = "rocksdb")] + #[clap(flatten)] + pub nullifier_tree: NullifierTreeRocksDbOptions, +} + +impl StorageOptions { + /// Benchmark setup. + /// + /// These values were determined during development of `LargeSmt` + pub fn bench() -> Self { + #[cfg(feature = "rocksdb")] + { + let account_tree = AccountTreeRocksDbOptions { + max_open_fds: self::rocksdb::BENCH_ROCKSDB_MAX_OPEN_FDS, + cache_size_in_bytes: self::rocksdb::DEFAULT_ROCKSDB_CACHE_SIZE, + }; + let nullifier_tree = NullifierTreeRocksDbOptions { + max_open_fds: BENCH_ROCKSDB_MAX_OPEN_FDS, + cache_size_in_bytes: DEFAULT_ROCKSDB_CACHE_SIZE, + }; + Self { account_tree, nullifier_tree } + } + #[cfg(not(feature = "rocksdb"))] + Self::default() + } +} diff --git a/crates/utils/src/clap/rocksdb.rs b/crates/utils/src/clap/rocksdb.rs new file mode 100644 index 0000000000..9b752205d2 --- /dev/null +++ b/crates/utils/src/clap/rocksdb.rs @@ -0,0 +1,107 @@ +//! `RocksDB` storage backend specific CLI argument parsing and types. + +use std::path::Path; + +use miden_large_smt_backend_rocksdb::RocksDbConfig; + +pub(crate) const DEFAULT_ROCKSDB_MAX_OPEN_FDS: i32 = 64; +pub(crate) const DEFAULT_ROCKSDB_CACHE_SIZE: usize = 2 << 30; +pub(crate) const BENCH_ROCKSDB_MAX_OPEN_FDS: i32 = 512; + +/// Per usage options for rocksdb configuration +#[derive(clap::Args, Clone, Debug, PartialEq, Eq)] +pub struct NullifierTreeRocksDbOptions { + #[arg( + long = "nullifier_tree.rocksdb.max_open_fds", + default_value_t = DEFAULT_ROCKSDB_MAX_OPEN_FDS, + value_name = "NULLIFIER_TREE__ROCKSDB__MAX_OPEN_FDS" + )] + pub max_open_fds: i32, + #[arg( + long = "nullifier_tree.rocksdb.max_cache_size", + default_value_t = DEFAULT_ROCKSDB_CACHE_SIZE, + value_name = "NULLIFIER_TREE__ROCKSDB__CACHE_SIZE" + )] + pub cache_size_in_bytes: usize, +} + +impl Default for NullifierTreeRocksDbOptions { + fn default() -> Self { + RocksDbOptions::default().into() + } +} + +/// Per usage options for rocksdb configuration +#[derive(clap::Args, Clone, Debug, PartialEq, Eq)] +pub struct AccountTreeRocksDbOptions { + #[arg( + long = "account_tree.rocksdb.max_open_fds", + default_value_t = DEFAULT_ROCKSDB_MAX_OPEN_FDS, + value_name = "ACCOUNT_TREE__ROCKSDB__MAX_OPEN_FDS" + )] + pub max_open_fds: i32, + #[arg( + long = "account_tree.rocksdb.max_cache_size", + default_value_t = DEFAULT_ROCKSDB_CACHE_SIZE, + value_name = "ACCOUNT_TREE__ROCKSDB__CACHE_SIZE" + )] + pub cache_size_in_bytes: usize, +} + +impl Default for AccountTreeRocksDbOptions { + fn default() -> Self { + RocksDbOptions::default().into() + } +} + +/// General confiration options for rocksdb. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RocksDbOptions { + pub max_open_fds: i32, + pub cache_size_in_bytes: usize, +} + +impl Default for RocksDbOptions { + fn default() -> Self { + Self { + max_open_fds: DEFAULT_ROCKSDB_MAX_OPEN_FDS, + cache_size_in_bytes: DEFAULT_ROCKSDB_CACHE_SIZE, + } + } +} + +impl From for RocksDbOptions { + fn from(value: AccountTreeRocksDbOptions) -> Self { + let AccountTreeRocksDbOptions { max_open_fds, cache_size_in_bytes } = value; + Self { max_open_fds, cache_size_in_bytes } + } +} + +impl From for RocksDbOptions { + fn from(value: NullifierTreeRocksDbOptions) -> Self { + let NullifierTreeRocksDbOptions { max_open_fds, cache_size_in_bytes } = value; + Self { max_open_fds, cache_size_in_bytes } + } +} + +impl From for AccountTreeRocksDbOptions { + fn from(value: RocksDbOptions) -> Self { + let RocksDbOptions { max_open_fds, cache_size_in_bytes } = value; + Self { max_open_fds, cache_size_in_bytes } + } +} + +impl From for NullifierTreeRocksDbOptions { + fn from(value: RocksDbOptions) -> Self { + let RocksDbOptions { max_open_fds, cache_size_in_bytes } = value; + Self { max_open_fds, cache_size_in_bytes } + } +} + +impl RocksDbOptions { + pub fn with_path(self, path: &Path) -> RocksDbConfig { + RocksDbConfig::new(path) + .with_cache_size(self.cache_size_in_bytes) + .with_max_open_files(self.max_open_fds) + } +} diff --git a/docs/external/src/operator/usage.md b/docs/external/src/operator/usage.md index ce4b699b14..840bee2cc4 100644 --- a/docs/external/src/operator/usage.md +++ b/docs/external/src/operator/usage.md @@ -138,6 +138,30 @@ You can inspect the service file with `systemctl cat miden-node` or alternativel our repository in the `packaging` folder. For the bootstrapping process be sure to specify the data-directory as expected by the systemd file. +## RocksDB tuning + +The store uses RocksDB for the account and nullifier trees, one instance each. The two most impactful knobs per tree +are exposed as CLI flags (also available as environment variables): + +| Flag | Default | Notes | +|---|---|---| +| `--account_tree.rocksdb.max_cache_size` | 2 GiB | Shared LRU block cache. Increase on memory-rich hosts. | +| `--account_tree.rocksdb.max_open_fds` | 64 | Raise to 512+ when `ulimit -n` allows. | +| `--nullifier_tree.rocksdb.max_cache_size` | 2 GiB | Same as above for the nullifier tree. | +| `--nullifier_tree.rocksdb.max_open_fds` | 64 | Same as above for the nullifier tree. | + +Compaction parallelism is set automatically to the number of available CPU cores. + +```sh +miden-node bundled start \ + --data-directory data \ + --rpc.url http://0.0.0.0:57291 \ + --account_tree.rocksdb.max_cache_size 4294967296 \ + --account_tree.rocksdb.max_open_fds 512 \ + --nullifier_tree.rocksdb.max_cache_size 4294967296 \ + --nullifier_tree.rocksdb.max_open_fds 512 +``` + ## Environment variables Most configuration options can also be configured using environment variables as an alternative to providing the values diff --git a/docs/internal/src/store.md b/docs/internal/src/store.md index 1929b7c491..67ad1d4d9b 100644 --- a/docs/internal/src/store.md +++ b/docs/internal/src/store.md @@ -2,7 +2,7 @@ This component persists the chain state in a `sqlite` database. It also stores each block's raw data as a file. -Mekle data structures are kept in-memory and are rebuilt on startup. Other data like account, note and nullifier +Merkle data structures are kept in-memory and are rebuilt on startup. Other data like account, note and nullifier information is always read from disk. We will need to revisit this in the future but for now this is performant enough. ## Migrations @@ -14,6 +14,20 @@ Note that the migration logic includes both a schema number _and_ a hash based o on node startup to ensure that any existing database matches the expected schema. If you're seeing database failures on startup its likely that you created the database _before_ making schema changes resulting in different schema hashes. +## RocksDB tree storage + +The account and nullifier trees are persisted in separate RocksDB instances under +`/accounttree` and `/nullifiertree`, managed by +`crates/large-smt-backend-rocksdb`. Column families: `leaves`, `st24`–`st56` (subtrees at each +depth), `metadata` (root/counts), `depth24` (cached depth-24 hashes for fast startup). + +Compaction parallelism and background jobs are set to `rayon::current_num_threads()` automatically. +WAL sync per write is disabled for throughput; a 512 MiB WAL cap bounds recovery time. Bloom filter +bits vary by depth (8.0–12.0) and memtables are 128 MiB per column family. See `RocksDbStorage::open` for the +full fixed configuration. Runtime-tuneable parameters are documented in the +[operator usage guide](https://github.com/0xMiden/node/blob/next/docs/external/src/operator/usage.md#rocksdb-tuning). + + ## Architecture The store consists mainly of a gRPC server which answers requests from the RPC and block-producer components, as well as