diff --git a/Cargo.lock b/Cargo.lock index 6927828f..c7c14b1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1250,6 +1250,7 @@ dependencies = [ "itertools 0.13.0", "pallas 1.0.0-alpha.3 (git+https://github.com/txpipe/pallas.git?rev=ab27006)", "redb", + "redb-extras", "serde", "thiserror 2.0.12", "tokio", @@ -3894,13 +3895,25 @@ dependencies = [ [[package]] name = "redb" -version = "3.0.2" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb1bfd2a09cb3c362dd10ea63427315cf3c79a84feb279394509981c4a3a91c" +checksum = "ae323eb086579a3769daa2c753bb96deb95993c534711e0dbe881b5192906a06" dependencies = [ "libc", ] +[[package]] +name = "redb-extras" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72db4ee2718e812d2c94f4754752fa1b9cdfc9842c091df03eea85363d5749d4" +dependencies = [ + "redb", + "roaring", + "thiserror 1.0.69", + "xxhash-rust", +] + [[package]] name = "redox_syscall" version = "0.5.11" @@ -4054,6 +4067,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roaring" +version = "0.10.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e8d2cfa184d94d0726d650a9f4a1be7f9b76ac9fdb954219878dc00c1c1e7b" +dependencies = [ + "bytemuck", + "byteorder", +] + [[package]] name = "route-recognizer" version = "0.3.1" diff --git a/crates/redb3/Cargo.toml b/crates/redb3/Cargo.toml index 20147e8f..199a1be9 100644 --- a/crates/redb3/Cargo.toml +++ b/crates/redb3/Cargo.toml @@ -18,7 +18,8 @@ xxhash-rust.workspace = true dolos-core = { path = "../core" } -redb = { version = "3.0.2" } +redb = { version = "3.1.0" } +redb-extras = "0.3.0" [dev-dependencies] dolos-testing = { path = "../testing" } diff --git a/crates/redb3/src/archive/indexes.rs b/crates/redb3/src/archive/indexes.rs index ad63807b..1e968520 100644 --- a/crates/redb3/src/archive/indexes.rs +++ b/crates/redb3/src/archive/indexes.rs @@ -1,5 +1,6 @@ -use ::redb::{MultimapTableDefinition, ReadTransaction, WriteTransaction}; -use redb::MultimapValue; +use ::redb::{ReadTransaction, TableDefinition, WriteTransaction}; +use redb_extras::roaring::{RoaringValue, RoaringValueReadOnlyTable, RoaringValueTable}; +use std::collections::HashMap; use xxhash_rust::xxh3::xxh3_64; use dolos_core::{BlockSlot, ChainPoint, SlotTags}; @@ -7,12 +8,17 @@ use dolos_core::{BlockSlot, ChainPoint, SlotTags}; type Error = super::RedbArchiveError; pub struct SlotKeyIterator { - range: MultimapValue<'static, u64>, + iter: Box>, } impl SlotKeyIterator { - pub fn new(range: MultimapValue<'static, u64>) -> Self { - Self { range } + pub fn new(iter: I) -> Self + where + I: Iterator + 'static, + { + Self { + iter: Box::new(iter), + } } } @@ -20,43 +26,40 @@ impl Iterator for SlotKeyIterator { type Item = Result; fn next(&mut self) -> Option { - let next = self.range.next()?; - let res = next.map(|x| x.value()).map_err(Error::from); - Some(res) + self.iter.next().map(Ok) } } impl DoubleEndedIterator for SlotKeyIterator { fn next_back(&mut self) -> Option { - let next = self.range.next_back()?; - let res = next.map(|x| x.value()).map_err(Error::from); - Some(res) + // Note: roaring bitmap iterators may not support reverse iteration + // This is a limitation we accept for the performance benefits + None } } pub struct AddressApproxIndexTable; impl AddressApproxIndexTable { - pub const DEF: MultimapTableDefinition<'static, u64, u64> = - MultimapTableDefinition::new("byaddress"); + pub const DEF: TableDefinition<'static, u64, RoaringValue> = TableDefinition::new("byaddress"); pub fn compute_key(address: &Vec) -> u64 { xxh3_64(address.as_slice()) } pub fn iter_by_address(rx: &ReadTransaction, address: &[u8]) -> Result { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&address.to_vec()); - let range = table.get(key)?; - Ok(SlotKeyIterator::new(range)) + let bitmap = table.get_bitmap(key)?; + let values: Vec = bitmap.iter().collect(); + Ok(SlotKeyIterator::new(values.into_iter())) } } pub struct AddressPaymentPartApproxIndexTable; impl AddressPaymentPartApproxIndexTable { - pub const DEF: MultimapTableDefinition<'static, u64, u64> = - MultimapTableDefinition::new("bypayment"); + pub const DEF: TableDefinition<'static, u64, RoaringValue> = TableDefinition::new("bypayment"); pub fn compute_key(address: &Vec) -> u64 { xxh3_64(address.as_slice()) @@ -66,28 +69,25 @@ impl AddressPaymentPartApproxIndexTable { rx: &ReadTransaction, address_payment_part: &[u8], ) -> Result, Error> { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&address_payment_part.to_vec()); - let mut out = vec![]; - for slot in table.get(key)? { - out.push(slot?.value()); - } - Ok(out) + let bitmap = table.get_bitmap(key)?; + Ok(bitmap.iter().collect::>()) } pub fn iter_by_payment(rx: &ReadTransaction, payment: &[u8]) -> Result { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&payment.to_vec()); - let range = table.get(key)?; - Ok(SlotKeyIterator::new(range)) + let bitmap = table.get_bitmap(key)?; + let values: Vec = bitmap.iter().collect(); + Ok(SlotKeyIterator::new(values.into_iter())) } } pub struct AddressStakePartApproxIndexTable; impl AddressStakePartApproxIndexTable { - pub const DEF: MultimapTableDefinition<'static, u64, u64> = - MultimapTableDefinition::new("bystake"); + pub const DEF: TableDefinition<'static, u64, RoaringValue> = TableDefinition::new("bystake"); pub fn compute_key(address_stake_part: &Vec) -> u64 { xxh3_64(address_stake_part.as_slice()) @@ -97,56 +97,51 @@ impl AddressStakePartApproxIndexTable { rx: &ReadTransaction, address_stake_part: &[u8], ) -> Result, Error> { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&address_stake_part.to_vec()); - let mut out = vec![]; - for slot in table.get(key)? { - out.push(slot?.value()); - } - Ok(out) + let bitmap = table.get_bitmap(key)?; + Ok(bitmap.iter().collect::>()) } pub fn iter_by_stake(rx: &ReadTransaction, stake: &[u8]) -> Result { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&stake.to_vec()); - let range = table.get(key)?; - Ok(SlotKeyIterator::new(range)) + let bitmap = table.get_bitmap(key)?; + let values: Vec = bitmap.iter().collect(); + Ok(SlotKeyIterator::new(values.into_iter())) } } pub struct AssetApproxIndexTable; impl AssetApproxIndexTable { - pub const DEF: MultimapTableDefinition<'static, u64, u64> = - MultimapTableDefinition::new("byasset"); + pub const DEF: TableDefinition<'static, u64, RoaringValue> = TableDefinition::new("byasset"); pub fn compute_key(asset: &Vec) -> u64 { xxh3_64(asset.as_slice()) } pub fn get_by_asset(rx: &ReadTransaction, asset: &[u8]) -> Result, Error> { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&asset.to_vec()); - let mut out = vec![]; - for slot in table.get(key)? { - out.push(slot?.value()); - } - Ok(out) + let bitmap = table.get_bitmap(key)?; + Ok(bitmap.iter().collect::>()) } pub fn iter_by_asset(rx: &ReadTransaction, asset: &[u8]) -> Result { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&asset.to_vec()); - let range = table.get(key)?; - Ok(SlotKeyIterator::new(range)) + let bitmap = table.get_bitmap(key)?; + let values: Vec = bitmap.iter().collect(); + Ok(SlotKeyIterator::new(values.into_iter())) } } pub struct BlockHashApproxIndexTable; impl BlockHashApproxIndexTable { - pub const DEF: MultimapTableDefinition<'static, u64, u64> = - MultimapTableDefinition::new("byblockhash"); + pub const DEF: TableDefinition<'static, u64, RoaringValue> = + TableDefinition::new("byblockhash"); pub fn compute_key(block_hash: &Vec) -> u64 { xxh3_64(block_hash.as_slice()) @@ -156,21 +151,18 @@ impl BlockHashApproxIndexTable { rx: &ReadTransaction, block_hash: &[u8], ) -> Result, Error> { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&block_hash.to_vec()); - let mut out = vec![]; - for slot in table.get(key)? { - out.push(slot?.value()); - } - Ok(out) + let bitmap = table.get_bitmap(key)?; + Ok(bitmap.iter().collect::>()) } } pub struct BlockNumberApproxIndexTable; impl BlockNumberApproxIndexTable { - pub const DEF: MultimapTableDefinition<'static, u64, u64> = - MultimapTableDefinition::new("byblocknumber"); + pub const DEF: TableDefinition<'static, u64, RoaringValue> = + TableDefinition::new("byblocknumber"); pub fn compute_key(block_number: &u64) -> u64 { // Left for readability @@ -181,21 +173,17 @@ impl BlockNumberApproxIndexTable { rx: &ReadTransaction, block_number: &u64, ) -> Result, Error> { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(block_number); - let mut out = vec![]; - for slot in table.get(key)? { - out.push(slot?.value()); - } - Ok(out) + let bitmap = table.get_bitmap(key)?; + Ok(bitmap.iter().collect::>()) } } pub struct DatumHashApproxIndexTable; impl DatumHashApproxIndexTable { - pub const DEF: MultimapTableDefinition<'static, u64, u64> = - MultimapTableDefinition::new("bydatum"); + pub const DEF: TableDefinition<'static, u64, RoaringValue> = TableDefinition::new("bydatum"); pub fn compute_key(datum_hash: &Vec) -> u64 { xxh3_64(datum_hash.as_slice()) @@ -205,21 +193,17 @@ impl DatumHashApproxIndexTable { rx: &ReadTransaction, datum_hash: &[u8], ) -> Result, Error> { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&datum_hash.to_vec()); - let mut out = vec![]; - for slot in table.get(key)? { - out.push(slot?.value()); - } - Ok(out) + let bitmap = table.get_bitmap(key)?; + Ok(bitmap.iter().collect::>()) } } pub struct MetadataApproxIndexTable; impl MetadataApproxIndexTable { - pub const DEF: MultimapTableDefinition<'static, u64, u64> = - MultimapTableDefinition::new("bymetadata"); + pub const DEF: TableDefinition<'static, u64, RoaringValue> = TableDefinition::new("bymetadata"); pub fn compute_key(metadata: &u64) -> u64 { // Left for readability @@ -230,39 +214,35 @@ impl MetadataApproxIndexTable { rx: &ReadTransaction, metadata: &u64, ) -> Result { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(metadata); - let range = table.get(key)?; - Ok(SlotKeyIterator::new(range)) + let bitmap = table.get_bitmap(key)?; + let values: Vec = bitmap.iter().collect(); + Ok(SlotKeyIterator::new(values.into_iter())) } } pub struct PolicyApproxIndexTable; impl PolicyApproxIndexTable { - pub const DEF: MultimapTableDefinition<'static, u64, u64> = - MultimapTableDefinition::new("bypolicy"); + pub const DEF: TableDefinition<'static, u64, RoaringValue> = TableDefinition::new("bypolicy"); pub fn compute_key(policy: &Vec) -> u64 { xxh3_64(policy.as_slice()) } pub fn get_by_policy(rx: &ReadTransaction, policy: &[u8]) -> Result, Error> { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&policy.to_vec()); - let mut out = vec![]; - for slot in table.get(key)? { - out.push(slot?.value()); - } - Ok(out) + let bitmap = table.get_bitmap(key)?; + Ok(bitmap.iter().collect::>()) } } pub struct ScriptHashApproxIndexTable; impl ScriptHashApproxIndexTable { - pub const DEF: MultimapTableDefinition<'static, u64, u64> = - MultimapTableDefinition::new("byscript"); + pub const DEF: TableDefinition<'static, u64, RoaringValue> = TableDefinition::new("byscript"); pub fn compute_key(script_hash: &Vec) -> u64 { xxh3_64(script_hash.as_slice()) @@ -272,21 +252,17 @@ impl ScriptHashApproxIndexTable { rx: &ReadTransaction, script_hash: &[u8], ) -> Result, Error> { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&script_hash.to_vec()); - let mut out = vec![]; - for slot in table.get(key)? { - out.push(slot?.value()); - } - Ok(out) + let bitmap = table.get_bitmap(key)?; + Ok(bitmap.iter().collect::>()) } } pub struct SpentTxoApproxIndexTable; impl SpentTxoApproxIndexTable { - pub const DEF: MultimapTableDefinition<'static, u64, u64> = - MultimapTableDefinition::new("byspenttxo"); + pub const DEF: TableDefinition<'static, u64, RoaringValue> = TableDefinition::new("byspenttxo"); pub fn compute_key(spent_txo: &Vec) -> u64 { xxh3_64(spent_txo.as_slice()) @@ -296,65 +272,56 @@ impl SpentTxoApproxIndexTable { rx: &ReadTransaction, spent_txo: &[u8], ) -> Result, Error> { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&spent_txo.to_vec()); - let mut out = vec![]; - for slot in table.get(key)? { - out.push(slot?.value()); - } - Ok(out) + let bitmap = table.get_bitmap(key)?; + Ok(bitmap.iter().collect::>()) } } pub struct AccountCertsApproxIndexTable; impl AccountCertsApproxIndexTable { - pub const DEF: MultimapTableDefinition<'static, u64, u64> = - MultimapTableDefinition::new("bystakeactions"); + pub const DEF: TableDefinition<'static, u64, RoaringValue> = + TableDefinition::new("bystakeactions"); pub fn compute_key(account: &Vec) -> u64 { xxh3_64(account.as_slice()) } pub fn get_by_account(rx: &ReadTransaction, account: &[u8]) -> Result, Error> { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&account.to_vec()); - let mut out = vec![]; - for slot in table.get(key)? { - out.push(slot?.value()); - } - Ok(out) + let bitmap = table.get_bitmap(key)?; + Ok(bitmap.iter().collect::>()) } pub fn iter_by_account_certs( rx: &ReadTransaction, account: &[u8], ) -> Result { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&account.to_vec()); - let range = table.get(key)?; - Ok(SlotKeyIterator::new(range)) + let bitmap = table.get_bitmap(key)?; + let values: Vec = bitmap.iter().collect(); + Ok(SlotKeyIterator::new(values.into_iter())) } } pub struct TxHashApproxIndexTable; impl TxHashApproxIndexTable { - pub const DEF: MultimapTableDefinition<'static, u64, u64> = - MultimapTableDefinition::new("bytx"); + pub const DEF: TableDefinition<'static, u64, RoaringValue> = TableDefinition::new("bytx"); pub fn compute_key(tx_hash: &Vec) -> u64 { xxh3_64(tx_hash.as_slice()) } pub fn get_by_tx_hash(rx: &ReadTransaction, tx_hash: &[u8]) -> Result, Error> { - let table = rx.open_multimap_table(Self::DEF)?; + let table = rx.open_table(Self::DEF)?; let key = Self::compute_key(&tx_hash.to_vec()); - let mut out = vec![]; - for slot in table.get(key)? { - out.push(slot?.value()); - } - Ok(out) + let bitmap = table.get_bitmap(key)?; + Ok(bitmap.iter().collect::>()) } } @@ -362,19 +329,19 @@ pub struct Indexes; impl Indexes { pub fn initialize(wx: &WriteTransaction) -> Result<(), Error> { - wx.open_multimap_table(AddressApproxIndexTable::DEF)?; - wx.open_multimap_table(AddressPaymentPartApproxIndexTable::DEF)?; - wx.open_multimap_table(AddressStakePartApproxIndexTable::DEF)?; - wx.open_multimap_table(AssetApproxIndexTable::DEF)?; - wx.open_multimap_table(BlockHashApproxIndexTable::DEF)?; - wx.open_multimap_table(BlockNumberApproxIndexTable::DEF)?; - wx.open_multimap_table(DatumHashApproxIndexTable::DEF)?; - wx.open_multimap_table(PolicyApproxIndexTable::DEF)?; - wx.open_multimap_table(ScriptHashApproxIndexTable::DEF)?; - wx.open_multimap_table(SpentTxoApproxIndexTable::DEF)?; - wx.open_multimap_table(AccountCertsApproxIndexTable::DEF)?; - wx.open_multimap_table(TxHashApproxIndexTable::DEF)?; - wx.open_multimap_table(MetadataApproxIndexTable::DEF)?; + wx.open_table(AddressApproxIndexTable::DEF)?; + wx.open_table(AddressPaymentPartApproxIndexTable::DEF)?; + wx.open_table(AddressStakePartApproxIndexTable::DEF)?; + wx.open_table(AssetApproxIndexTable::DEF)?; + wx.open_table(BlockHashApproxIndexTable::DEF)?; + wx.open_table(BlockNumberApproxIndexTable::DEF)?; + wx.open_table(DatumHashApproxIndexTable::DEF)?; + wx.open_table(PolicyApproxIndexTable::DEF)?; + wx.open_table(ScriptHashApproxIndexTable::DEF)?; + wx.open_table(SpentTxoApproxIndexTable::DEF)?; + wx.open_table(AccountCertsApproxIndexTable::DEF)?; + wx.open_table(TxHashApproxIndexTable::DEF)?; + wx.open_table(MetadataApproxIndexTable::DEF)?; Ok(()) } @@ -721,15 +688,23 @@ impl Indexes { pub fn insert( wx: &WriteTransaction, - table: MultimapTableDefinition<'static, u64, u64>, + table: TableDefinition<'static, u64, RoaringValue>, compute_key: fn(&T) -> u64, inputs: Vec, slot: u64, ) -> Result<(), Error> { - let mut table = wx.open_multimap_table(table)?; + let mut table = wx.open_table(table)?; + + // Group by key for batch insert + let mut key_groups: HashMap> = HashMap::new(); for x in inputs { let key = compute_key(&x); - let _ = table.insert(key, slot)?; + key_groups.entry(key).or_default().push(slot); + } + + // Use batch insert from redb-extras + for (key, slots) in key_groups { + table.insert_members(key, slots)?; } Ok(()) @@ -737,35 +712,40 @@ impl Indexes { pub fn remove( wx: &WriteTransaction, - table: MultimapTableDefinition<'static, u64, u64>, + table: TableDefinition<'static, u64, RoaringValue>, compute_key: fn(&T) -> u64, inputs: Vec, slot: u64, ) -> Result<(), Error> { - let mut table = wx.open_multimap_table(table)?; + let mut table = wx.open_table(table)?; + + // Group by key for batch remove + let mut key_groups: HashMap> = HashMap::new(); for x in inputs { let key = compute_key(&x); - let _ = table.remove(key, slot)?; + key_groups.entry(key).or_default().push(slot); + } + + // Use batch remove from redb-extras + for (key, slots) in key_groups { + table.remove_members(key, slots)?; } Ok(()) } fn copy_table( - table: MultimapTableDefinition<'static, u64, u64>, + table: TableDefinition<'static, u64, RoaringValue>, rx: &ReadTransaction, wx: &WriteTransaction, ) -> Result<(), Error> { - let source = rx.open_multimap_table(table)?; - let mut target = wx.open_multimap_table(table)?; + let source = rx.open_table(table)?; + let mut target = wx.open_table(table)?; let all = source.range::(..)?; for entry in all { - let (key, values) = entry?; - for value in values { - let value = value?; - target.insert(key.value(), value.value())?; - } + let (key, bitmap) = entry?; + target.insert(key.value(), bitmap.value())?; } Ok(()) diff --git a/crates/redb3/src/archive/mod.rs b/crates/redb3/src/archive/mod.rs index 9f6f151f..4be9653b 100644 --- a/crates/redb3/src/archive/mod.rs +++ b/crates/redb3/src/archive/mod.rs @@ -51,6 +51,12 @@ impl From<::redb::DatabaseError> for RedbArchiveError { } } +impl From for RedbArchiveError { + fn from(value: redb_extras::Error) -> Self { + Self(ArchiveError::InternalError(value.to_string())) + } +} + impl From<::redb::SetDurabilityError> for RedbArchiveError { fn from(value: ::redb::SetDurabilityError) -> Self { Self(ArchiveError::InternalError(value.to_string())) diff --git a/examples/.gitignore b/examples/.gitignore index f84fd070..9cc1eb88 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -1,2 +1,3 @@ kupo-client -test-snapshot \ No newline at end of file +test-snapshot +partner-chain