From 0c543860c5ea032dddb4b6308b50778b0ba34d5e Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Mon, 1 Dec 2025 12:22:29 -0500 Subject: [PATCH 01/19] Move TransactionManager to db4-storage --- db4-graph/src/lib.rs | 43 ++----------------- db4-storage/src/lib.rs | 3 ++ db4-storage/src/transaction/mod.rs | 40 +++++++++++++++++ raphtory-storage/src/mutation/addition_ops.rs | 4 +- .../src/mutation/addition_ops_ext.rs | 4 +- raphtory/src/db/api/storage/storage.rs | 4 +- 6 files changed, 52 insertions(+), 46 deletions(-) create mode 100644 db4-storage/src/transaction/mod.rs diff --git a/db4-graph/src/lib.rs b/db4-graph/src/lib.rs index 9250e8f485..40a97ea1b0 100644 --- a/db4-graph/src/lib.rs +++ b/db4-graph/src/lib.rs @@ -1,10 +1,7 @@ use std::{ io, path::{Path, PathBuf}, - sync::{ - atomic::{self, AtomicU64, AtomicUsize}, - Arc, - }, + sync::{atomic::AtomicUsize, Arc}, }; use raphtory_api::core::{ @@ -26,8 +23,8 @@ use storage::{ }, persist::strategy::{Config, PersistentStrategy}, resolver::GIDResolverOps, - wal::{GraphWal, TransactionID, Wal}, - Extension, GIDResolver, Layer, ReadLockedLayer, WalImpl, ES, NS, + wal::Wal, + Extension, GIDResolver, Layer, ReadLockedLayer, TransactionManager, WalImpl, ES, NS, }; use tempfile::TempDir; @@ -87,40 +84,6 @@ impl<'a> From<&'a Path> for GraphDir { } } -#[derive(Debug)] -pub struct TransactionManager { - last_transaction_id: AtomicU64, - wal: Arc, -} - -impl TransactionManager { - const STARTING_TRANSACTION_ID: TransactionID = 1; - - pub fn new(wal: Arc) -> Self { - Self { - last_transaction_id: AtomicU64::new(Self::STARTING_TRANSACTION_ID), - wal, - } - } - - pub fn load(self, last_transaction_id: TransactionID) { - self.last_transaction_id - .store(last_transaction_id, atomic::Ordering::SeqCst) - } - - pub fn begin_transaction(&self) -> TransactionID { - let transaction_id = self - .last_transaction_id - .fetch_add(1, atomic::Ordering::SeqCst); - self.wal.log_begin_transaction(transaction_id).unwrap(); - transaction_id - } - - pub fn end_transaction(&self, transaction_id: TransactionID) { - self.wal.log_end_transaction(transaction_id).unwrap(); - } -} - impl Default for TemporalGraph { fn default() -> Self { Self::new(Extension::default()).unwrap() diff --git a/db4-storage/src/lib.rs b/db4-storage/src/lib.rs index 28584817d7..e1c3b8559e 100644 --- a/db4-storage/src/lib.rs +++ b/db4-storage/src/lib.rs @@ -23,6 +23,7 @@ use crate::{ node_entry::{MemNodeEntry, MemNodeRef}, }, wal::no_wal::NoWal, + transaction::TransactionManager as GenericTransactionManager, }; use parking_lot::RwLock; use raphtory_api::core::entities::{EID, VID}; @@ -36,6 +37,7 @@ pub mod persist; pub mod properties; pub mod resolver; pub mod segments; +pub mod transaction; pub mod utils; pub mod wal; @@ -46,6 +48,7 @@ pub type Layer

= GraphStore, ES

, P>; pub type WalImpl = NoWal; pub type GIDResolver = MappingResolver; +pub type TransactionManager = GenericTransactionManager; pub type ReadLockedLayer

= ReadLockedGraphStore, ES

, P>; pub type ReadLockedNodes

= ReadLockedNodeStorage, P>; diff --git a/db4-storage/src/transaction/mod.rs b/db4-storage/src/transaction/mod.rs new file mode 100644 index 0000000000..a7b175af09 --- /dev/null +++ b/db4-storage/src/transaction/mod.rs @@ -0,0 +1,40 @@ +use std::sync::{ + Arc, + atomic::{self, AtomicU64}, +}; + +use crate::wal::{GraphWal, TransactionID}; + +#[derive(Debug)] +pub struct TransactionManager { + last_transaction_id: AtomicU64, + wal: Arc, +} + +impl TransactionManager { + const STARTING_TRANSACTION_ID: TransactionID = 1; + + pub fn new(wal: Arc) -> Self { + Self { + last_transaction_id: AtomicU64::new(Self::STARTING_TRANSACTION_ID), + wal, + } + } + + pub fn load(self, last_transaction_id: TransactionID) { + self.last_transaction_id + .store(last_transaction_id, atomic::Ordering::SeqCst) + } + + pub fn begin_transaction(&self) -> TransactionID { + let transaction_id = self + .last_transaction_id + .fetch_add(1, atomic::Ordering::SeqCst); + self.wal.log_begin_transaction(transaction_id).unwrap(); + transaction_id + } + + pub fn end_transaction(&self, transaction_id: TransactionID) { + self.wal.log_end_transaction(transaction_id).unwrap(); + } +} diff --git a/raphtory-storage/src/mutation/addition_ops.rs b/raphtory-storage/src/mutation/addition_ops.rs index 162eba66f9..a6beb2c03c 100644 --- a/raphtory-storage/src/mutation/addition_ops.rs +++ b/raphtory-storage/src/mutation/addition_ops.rs @@ -5,7 +5,7 @@ use crate::{ MutationError, }, }; -use db4_graph::{TransactionManager, WriteLockedGraph}; +use db4_graph::WriteLockedGraph; use raphtory_api::{ core::{ entities::{ @@ -20,7 +20,7 @@ use raphtory_api::{ inherit::Base, }; use raphtory_core::entities::{nodes::node_ref::NodeRef, ELID}; -use storage::{Extension, WalImpl}; +use storage::{Extension, TransactionManager, WalImpl}; pub trait InternalAdditionOps { type Error: From; diff --git a/raphtory-storage/src/mutation/addition_ops_ext.rs b/raphtory-storage/src/mutation/addition_ops_ext.rs index e770f8a537..e7eb5ee13b 100644 --- a/raphtory-storage/src/mutation/addition_ops_ext.rs +++ b/raphtory-storage/src/mutation/addition_ops_ext.rs @@ -2,7 +2,7 @@ use crate::mutation::{ addition_ops::{EdgeWriteLock, InternalAdditionOps, SessionAdditionOps}, MutationError, }; -use db4_graph::{TemporalGraph, TransactionManager, WriteLockedGraph}; +use db4_graph::{TemporalGraph, WriteLockedGraph}; use raphtory_api::core::{ entities::properties::{ meta::{Meta, NODE_ID_IDX, NODE_TYPE_IDX}, @@ -23,7 +23,7 @@ use storage::{ persist::strategy::PersistentStrategy, properties::props_meta_writer::PropsMetaWriter, resolver::GIDResolverOps, - Extension, WalImpl, ES, NS, + Extension, TransactionManager, WalImpl, ES, NS, }; pub struct WriteS<'a, EXT: PersistentStrategy, ES = ES>> { diff --git a/raphtory/src/db/api/storage/storage.rs b/raphtory/src/db/api/storage/storage.rs index adf3d02c24..efaf78f58c 100644 --- a/raphtory/src/db/api/storage/storage.rs +++ b/raphtory/src/db/api/storage/storage.rs @@ -6,7 +6,7 @@ use crate::{ }, errors::GraphError, }; -use db4_graph::{TemporalGraph, TransactionManager, WriteLockedGraph}; +use db4_graph::{TemporalGraph, WriteLockedGraph}; use raphtory_api::core::{ entities::{ properties::{ @@ -35,7 +35,7 @@ use std::{ path::Path, sync::Arc, }; -use storage::{Extension, WalImpl}; +use storage::{Extension, TransactionManager, WalImpl}; #[cfg(feature = "search")] use { From 92f279353261bf5bf466020b420c58a0e0f441c6 Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Mon, 1 Dec 2025 15:54:11 -0500 Subject: [PATCH 02/19] Create DurabilityOps --- raphtory-storage/src/mutation/addition_ops.rs | 26 +------------ .../src/mutation/addition_ops_ext.rs | 3 ++ .../src/mutation/durability_ops.rs | 37 +++++++++++++++++++ raphtory-storage/src/mutation/mod.rs | 3 ++ raphtory/src/db/api/mutation/addition_ops.rs | 7 ++-- raphtory/src/db/api/storage/storage.rs | 19 ++++++---- raphtory/src/db/graph/edge.rs | 4 +- 7 files changed, 62 insertions(+), 37 deletions(-) create mode 100644 raphtory-storage/src/mutation/durability_ops.rs diff --git a/raphtory-storage/src/mutation/addition_ops.rs b/raphtory-storage/src/mutation/addition_ops.rs index a6beb2c03c..3ac593e091 100644 --- a/raphtory-storage/src/mutation/addition_ops.rs +++ b/raphtory-storage/src/mutation/addition_ops.rs @@ -20,7 +20,7 @@ use raphtory_api::{ inherit::Base, }; use raphtory_core::entities::{nodes::node_ref::NodeRef, ELID}; -use storage::{Extension, TransactionManager, WalImpl}; +use storage::{Extension}; pub trait InternalAdditionOps { type Error: From; @@ -91,12 +91,6 @@ pub trait InternalAdditionOps { meta: &Meta, props: impl Iterator, ) -> Result>, Self::Error>; - - /// TODO: Not sure the below methods belong here... - - fn transaction_manager(&self) -> &TransactionManager; - - fn wal(&self) -> &WalImpl; } pub trait EdgeWriteLock: Send + Sync { @@ -294,14 +288,6 @@ impl InternalAdditionOps for GraphStorage { Ok(self.mutable()?.validate_gids(gids)?) } - fn transaction_manager(&self) -> &TransactionManager { - self.mutable().unwrap().transaction_manager.as_ref() - } - - fn wal(&self) -> &WalImpl { - self.mutable().unwrap().wal.as_ref() - } - fn resolve_node_and_type( &self, id: NodeRef, @@ -411,16 +397,6 @@ where self.base().validate_gids(gids) } - #[inline] - fn transaction_manager(&self) -> &TransactionManager { - self.base().transaction_manager() - } - - #[inline] - fn wal(&self) -> &WalImpl { - self.base().wal() - } - fn resolve_node_and_type( &self, id: NodeRef, diff --git a/raphtory-storage/src/mutation/addition_ops_ext.rs b/raphtory-storage/src/mutation/addition_ops_ext.rs index e7eb5ee13b..5b1cdcf5a1 100644 --- a/raphtory-storage/src/mutation/addition_ops_ext.rs +++ b/raphtory-storage/src/mutation/addition_ops_ext.rs @@ -1,5 +1,6 @@ use crate::mutation::{ addition_ops::{EdgeWriteLock, InternalAdditionOps, SessionAdditionOps}, + durability_ops::DurabilityOps, MutationError, }; use db4_graph::{TemporalGraph, WriteLockedGraph}; @@ -376,7 +377,9 @@ impl InternalAdditionOps for TemporalGraph { Ok(prop_ids) } } +} +impl DurabilityOps for TemporalGraph { fn transaction_manager(&self) -> &TransactionManager { &self.transaction_manager } diff --git a/raphtory-storage/src/mutation/durability_ops.rs b/raphtory-storage/src/mutation/durability_ops.rs new file mode 100644 index 0000000000..be6288a957 --- /dev/null +++ b/raphtory-storage/src/mutation/durability_ops.rs @@ -0,0 +1,37 @@ +use storage::{TransactionManager, WalImpl}; +use crate::graph::graph::GraphStorage; +use raphtory_api::inherit::Base; + +/// Accessor methods for transactions and write-ahead logging. +pub trait DurabilityOps { + fn transaction_manager(&self) -> &TransactionManager; + + fn wal(&self) -> &WalImpl; +} + +impl DurabilityOps for GraphStorage { + fn transaction_manager(&self) -> &TransactionManager { + self.mutable().unwrap().transaction_manager.as_ref() + } + + fn wal(&self) -> &WalImpl { + self.mutable().unwrap().wal.as_ref() + } +} + +pub trait InheritDurabilityOps: Base {} + +impl DurabilityOps for G +where + G::Base: DurabilityOps, +{ + #[inline] + fn transaction_manager(&self) -> &TransactionManager { + self.base().transaction_manager() + } + + #[inline] + fn wal(&self) -> &WalImpl { + self.base().wal() + } +} diff --git a/raphtory-storage/src/mutation/mod.rs b/raphtory-storage/src/mutation/mod.rs index 8dd4cf8157..835f84afca 100644 --- a/raphtory-storage/src/mutation/mod.rs +++ b/raphtory-storage/src/mutation/mod.rs @@ -4,6 +4,7 @@ use crate::{ mutation::{ addition_ops::InheritAdditionOps, deletion_ops::InheritDeletionOps, property_addition_ops::InheritPropertyAdditionOps, + durability_ops::InheritDurabilityOps, }, }; use parking_lot::RwLockWriteGuard; @@ -31,6 +32,7 @@ pub mod addition_ops; pub mod addition_ops_ext; pub mod deletion_ops; pub mod property_addition_ops; +pub mod durability_ops; pub type NodeWriterT<'a> = NodeWriter<'a, RwLockWriteGuard<'a, MemNodeSegment>, NS>; pub type EdgeWriterT<'a> = EdgeWriter<'a, RwLockWriteGuard<'a, MemEdgeSegment>, ES>; @@ -70,5 +72,6 @@ pub trait InheritMutationOps: Base {} impl InheritAdditionOps for G {} impl InheritPropertyAdditionOps for G {} impl InheritDeletionOps for G {} +impl InheritDurabilityOps for G {} impl InheritMutationOps for Arc {} diff --git a/raphtory/src/db/api/mutation/addition_ops.rs b/raphtory/src/db/api/mutation/addition_ops.rs index ed319d3d97..351627bfbb 100644 --- a/raphtory/src/db/api/mutation/addition_ops.rs +++ b/raphtory/src/db/api/mutation/addition_ops.rs @@ -15,9 +15,10 @@ use crate::{ }; use raphtory_api::core::entities::properties::prop::Prop; use raphtory_storage::mutation::addition_ops::{EdgeWriteLock, InternalAdditionOps}; +use raphtory_storage::mutation::durability_ops::DurabilityOps; use storage::wal::{GraphWal, Wal}; -pub trait AdditionOps: StaticGraphViewOps + InternalAdditionOps> { +pub trait AdditionOps: StaticGraphViewOps + InternalAdditionOps> + DurabilityOps { // TODO: Probably add vector reference here like add /// Add a node to the graph /// @@ -143,7 +144,7 @@ pub trait AdditionOps: StaticGraphViewOps + InternalAdditionOps> + StaticGraphViewOps> AdditionOps for G { +impl> + StaticGraphViewOps + DurabilityOps> AdditionOps for G { fn add_node< V: AsNodeRef, T: TryIntoInputTime, @@ -355,7 +356,7 @@ impl> + StaticGraphViewOps> Addit add_edge_op.store_src_node_info(src_id, src.as_node_ref().as_gid_ref().left()); add_edge_op.store_dst_node_info(dst_id, dst.as_node_ref().as_gid_ref().left()); - // Log transaction end + // Log transaction end. self.transaction_manager().end_transaction(transaction_id); // Flush all wal entries to disk. diff --git a/raphtory/src/db/api/storage/storage.rs b/raphtory/src/db/api/storage/storage.rs index efaf78f58c..3b1472d9e1 100644 --- a/raphtory/src/db/api/storage/storage.rs +++ b/raphtory/src/db/api/storage/storage.rs @@ -24,6 +24,7 @@ use raphtory_storage::{ layer_ops::InheritLayerOps, mutation::{ addition_ops::{EdgeWriteLock, InternalAdditionOps, SessionAdditionOps}, + durability_ops::DurabilityOps, addition_ops_ext::{UnlockedSession, WriteS}, deletion_ops::InternalDeletionOps, property_addition_ops::InternalPropertyAdditionOps, @@ -575,14 +576,6 @@ impl InternalAdditionOps for Storage { Ok(self.graph.validate_gids(gids)?) } - fn transaction_manager(&self) -> &TransactionManager { - self.graph.mutable().unwrap().transaction_manager.as_ref() - } - - fn wal(&self) -> &WalImpl { - self.graph.mutable().unwrap().wal.as_ref() - } - fn resolve_node_and_type( &self, id: NodeRef, @@ -592,6 +585,16 @@ impl InternalAdditionOps for Storage { } } +impl DurabilityOps for Storage { + fn transaction_manager(&self) -> &TransactionManager { + self.graph.mutable().unwrap().transaction_manager.as_ref() + } + + fn wal(&self) -> &WalImpl { + self.graph.mutable().unwrap().wal.as_ref() + } +} + impl InternalPropertyAdditionOps for Storage { type Error = GraphError; fn internal_add_properties( diff --git a/raphtory/src/db/graph/edge.rs b/raphtory/src/db/graph/edge.rs index 6e4add574f..c11781c251 100644 --- a/raphtory/src/db/graph/edge.rs +++ b/raphtory/src/db/graph/edge.rs @@ -40,6 +40,7 @@ use raphtory_storage::{ graph::edges::edge_storage_ops::EdgeStorageOps, mutation::{ addition_ops::{EdgeWriteLock, InternalAdditionOps}, + durability_ops::DurabilityOps, deletion_ops::InternalDeletionOps, property_addition_ops::InternalPropertyAdditionOps, }, @@ -176,7 +177,8 @@ impl< G: StaticGraphViewOps + InternalAdditionOps + InternalPropertyAdditionOps - + InternalDeletionOps, + + InternalDeletionOps + + DurabilityOps, > EdgeView { pub fn delete(&self, t: T, layer: Option<&str>) -> Result<(), GraphError> { From cff7a1e5634b420786098103b6670701ac4fdd87 Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Mon, 1 Dec 2025 19:27:24 -0500 Subject: [PATCH 03/19] Remove wal from transaction manager --- db4-graph/src/lib.rs | 6 ++-- db4-storage/src/lib.rs | 2 -- db4-storage/src/transaction/mod.rs | 36 +++++++++---------- .../src/mutation/addition_ops_ext.rs | 2 +- .../src/mutation/durability_ops.rs | 2 +- raphtory/src/db/api/storage/storage.rs | 2 +- raphtory/src/db/replay/mod.rs | 8 +++-- 7 files changed, 29 insertions(+), 29 deletions(-) diff --git a/db4-graph/src/lib.rs b/db4-graph/src/lib.rs index 40a97ea1b0..a6656d5d9d 100644 --- a/db4-graph/src/lib.rs +++ b/db4-graph/src/lib.rs @@ -24,7 +24,7 @@ use storage::{ persist::strategy::{Config, PersistentStrategy}, resolver::GIDResolverOps, wal::Wal, - Extension, GIDResolver, Layer, ReadLockedLayer, TransactionManager, WalImpl, ES, NS, + Extension, GIDResolver, Layer, ReadLockedLayer, transaction::TransactionManager, WalImpl, ES, NS, }; use tempfile::TempDir; @@ -119,7 +119,7 @@ impl, ES = ES>> TemporalGraph { node_count, storage: Arc::new(storage), graph_meta: Arc::new(GraphMeta::default()), - transaction_manager: Arc::new(TransactionManager::new(wal.clone())), + transaction_manager: Arc::new(TransactionManager::new()), wal, }) } @@ -164,7 +164,7 @@ impl, ES = ES>> TemporalGraph { node_count: AtomicUsize::new(0), storage: Arc::new(storage), graph_meta: Arc::new(GraphMeta::default()), - transaction_manager: Arc::new(TransactionManager::new(wal.clone())), + transaction_manager: Arc::new(TransactionManager::new()), wal, }) } diff --git a/db4-storage/src/lib.rs b/db4-storage/src/lib.rs index e1c3b8559e..9aa00c24ae 100644 --- a/db4-storage/src/lib.rs +++ b/db4-storage/src/lib.rs @@ -23,7 +23,6 @@ use crate::{ node_entry::{MemNodeEntry, MemNodeRef}, }, wal::no_wal::NoWal, - transaction::TransactionManager as GenericTransactionManager, }; use parking_lot::RwLock; use raphtory_api::core::entities::{EID, VID}; @@ -48,7 +47,6 @@ pub type Layer

= GraphStore, ES

, P>; pub type WalImpl = NoWal; pub type GIDResolver = MappingResolver; -pub type TransactionManager = GenericTransactionManager; pub type ReadLockedLayer

= ReadLockedGraphStore, ES

, P>; pub type ReadLockedNodes

= ReadLockedNodeStorage, P>; diff --git a/db4-storage/src/transaction/mod.rs b/db4-storage/src/transaction/mod.rs index a7b175af09..439e5b00de 100644 --- a/db4-storage/src/transaction/mod.rs +++ b/db4-storage/src/transaction/mod.rs @@ -1,40 +1,40 @@ -use std::sync::{ - Arc, - atomic::{self, AtomicU64}, -}; +use std::sync::atomic::{self, AtomicU64}; -use crate::wal::{GraphWal, TransactionID}; +use crate::wal::TransactionID; #[derive(Debug)] -pub struct TransactionManager { +pub struct TransactionManager { last_transaction_id: AtomicU64, - wal: Arc, } -impl TransactionManager { +impl TransactionManager { const STARTING_TRANSACTION_ID: TransactionID = 1; - pub fn new(wal: Arc) -> Self { + pub fn new() -> Self { Self { last_transaction_id: AtomicU64::new(Self::STARTING_TRANSACTION_ID), - wal, } } - pub fn load(self, last_transaction_id: TransactionID) { + /// Restores the last used transaction ID to the specified value. + /// Intended for using during recovery. + pub fn restore_transaction_id(&self, last_transaction_id: TransactionID) { self.last_transaction_id .store(last_transaction_id, atomic::Ordering::SeqCst) } pub fn begin_transaction(&self) -> TransactionID { - let transaction_id = self - .last_transaction_id - .fetch_add(1, atomic::Ordering::SeqCst); - self.wal.log_begin_transaction(transaction_id).unwrap(); - transaction_id + self.last_transaction_id + .fetch_add(1, atomic::Ordering::SeqCst) + } + + pub fn end_transaction(&self, _transaction_id: TransactionID) { + // No-op for now. } +} - pub fn end_transaction(&self, transaction_id: TransactionID) { - self.wal.log_end_transaction(transaction_id).unwrap(); +impl Default for TransactionManager { + fn default() -> Self { + Self::new() } } diff --git a/raphtory-storage/src/mutation/addition_ops_ext.rs b/raphtory-storage/src/mutation/addition_ops_ext.rs index 5b1cdcf5a1..875027319a 100644 --- a/raphtory-storage/src/mutation/addition_ops_ext.rs +++ b/raphtory-storage/src/mutation/addition_ops_ext.rs @@ -24,7 +24,7 @@ use storage::{ persist::strategy::PersistentStrategy, properties::props_meta_writer::PropsMetaWriter, resolver::GIDResolverOps, - Extension, TransactionManager, WalImpl, ES, NS, + Extension, transaction::TransactionManager, WalImpl, ES, NS, }; pub struct WriteS<'a, EXT: PersistentStrategy, ES = ES>> { diff --git a/raphtory-storage/src/mutation/durability_ops.rs b/raphtory-storage/src/mutation/durability_ops.rs index be6288a957..34713df7aa 100644 --- a/raphtory-storage/src/mutation/durability_ops.rs +++ b/raphtory-storage/src/mutation/durability_ops.rs @@ -1,4 +1,4 @@ -use storage::{TransactionManager, WalImpl}; +use storage::{transaction::TransactionManager, WalImpl}; use crate::graph::graph::GraphStorage; use raphtory_api::inherit::Base; diff --git a/raphtory/src/db/api/storage/storage.rs b/raphtory/src/db/api/storage/storage.rs index 3b1472d9e1..91702aa4a2 100644 --- a/raphtory/src/db/api/storage/storage.rs +++ b/raphtory/src/db/api/storage/storage.rs @@ -36,7 +36,7 @@ use std::{ path::Path, sync::Arc, }; -use storage::{Extension, TransactionManager, WalImpl}; +use storage::{Extension, transaction::TransactionManager, WalImpl}; #[cfg(feature = "search")] use { diff --git a/raphtory/src/db/replay/mod.rs b/raphtory/src/db/replay/mod.rs index 2c356faa3a..ab6a7e32ca 100644 --- a/raphtory/src/db/replay/mod.rs +++ b/raphtory/src/db/replay/mod.rs @@ -10,9 +10,9 @@ use storage::{ Extension, }; -/// Wrapper struct for implementing GraphReplayer for a TemporalGraph. -/// This is needed to workaround Rust's orphan rule since both ReplayGraph and TemporalGraph -/// are foreign to this crate. +/// Wrapper struct for implementing `GraphReplayer` for a `TemporalGraph`. +/// This is needed to workaround Rust's orphan rule since both `GraphReplayer` +/// and `TemporalGraph` are foreign to this crate. #[derive(Debug)] pub struct ReplayGraph { graph: TemporalGraph, @@ -72,6 +72,8 @@ impl GraphReplayer for ReplayGraph { _ => {} } + // TODO: Check max lsn on disk to see if replay is needed. + Ok(()) } From 6f24855296c118297de6b49e1803cc953aa3e6f2 Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Tue, 2 Dec 2025 15:12:12 -0500 Subject: [PATCH 04/19] Simplify wal log and replay methods --- db4-storage/src/wal/entry.rs | 77 ++------- db4-storage/src/wal/mod.rs | 147 ++---------------- raphtory-storage/src/mutation/addition_ops.rs | 1 + raphtory/src/db/replay/mod.rs | 76 +-------- 4 files changed, 30 insertions(+), 271 deletions(-) diff --git a/db4-storage/src/wal/entry.rs b/db4-storage/src/wal/entry.rs index 71ba54ce4a..d6cd68ebb4 100644 --- a/db4-storage/src/wal/entry.rs +++ b/db4-storage/src/wal/entry.rs @@ -1,6 +1,9 @@ use std::path::Path; -use raphtory_api::core::{entities::properties::prop::Prop, storage::dict_mapper::MaybeNew}; +use raphtory_api::core::{ + entities::properties::prop::Prop, + storage::dict_mapper::MaybeNew, +}; use raphtory_core::{ entities::{EID, GID, VID}, storage::timeindex::TimeIndexEntry, @@ -14,82 +17,22 @@ use crate::{ impl GraphWal for NoWal { type ReplayEntry = (); - fn log_begin_transaction(&self, _transaction_id: TransactionID) -> Result { - Ok(0) - } - - fn log_end_transaction(&self, _transaction_id: TransactionID) -> Result { - Ok(0) - } - - fn log_add_static_edge( - &self, - _transaction_id: TransactionID, - _t: TimeIndexEntry, - _src: VID, - _dst: VID, - ) -> Result { - Ok(0) - } - - fn log_add_edge( + fn log_add_edge>( &self, _transaction_id: TransactionID, _t: TimeIndexEntry, - _src: VID, - _dst: VID, + _src_name: GID, + _src_id: VID, + _dst_name: GID, + _dst_id: VID, _eid: EID, + _layer_name: Option<&str>, _layer_id: usize, - _props: &[(usize, Prop)], - ) -> Result { - Ok(0) - } - - fn log_node_id( - &self, - _transaction_id: TransactionID, - _gid: GID, - _vid: VID, - ) -> Result { - Ok(0) - } - - fn log_edge_id( - &self, - _transaction_id: TransactionID, - _src: VID, - _dst: VID, - _eid: EID, - _layer_id: usize, - ) -> Result { - Ok(0) - } - - fn log_const_prop_ids>( - &self, - _transaction_id: TransactionID, _props: &[MaybeNew<(PN, usize, Prop)>], ) -> Result { Ok(0) } - fn log_temporal_prop_ids>( - &self, - _transaction_id: TransactionID, - _props: &[MaybeNew<(PN, usize, Prop)>], - ) -> Result { - Ok(0) - } - - fn log_layer_id( - &self, - _transaction_id: TransactionID, - _name: &str, - _id: usize, - ) -> Result { - Ok(0) - } - fn log_checkpoint(&self, _lsn: LSN) -> Result { Ok(0) } diff --git a/db4-storage/src/wal/mod.rs b/db4-storage/src/wal/mod.rs index 7538781b16..912e7b2647 100644 --- a/db4-storage/src/wal/mod.rs +++ b/db4-storage/src/wal/mod.rs @@ -46,95 +46,20 @@ pub trait GraphWal { /// ReplayEntry represents the type of the wal entry returned during replay. type ReplayEntry; - fn log_begin_transaction(&self, transaction_id: TransactionID) -> Result; - - fn log_end_transaction(&self, transaction_id: TransactionID) -> Result; - - /// Log a static edge addition. - /// - /// # Arguments - /// - /// * `transaction_id` - The transaction ID - /// * `t` - The timestamp of the edge addition - /// * `src` - The source vertex ID - /// * `dst` - The destination vertex ID - fn log_add_static_edge( + fn log_add_edge>( &self, transaction_id: TransactionID, t: TimeIndexEntry, - src: VID, - dst: VID, - ) -> Result; - - /// Log an edge addition to a layer with temporal props. - /// - /// # Arguments - /// - /// * `transaction_id` - The transaction ID - /// * `t` - The timestamp of the edge addition - /// * `src` - The source vertex ID - /// * `dst` - The destination vertex ID - /// * `eid` - The edge ID - /// * `layer_id` - The layer ID - /// * `props` - The temporal properties of the edge - fn log_add_edge( - &self, - transaction_id: TransactionID, - t: TimeIndexEntry, - src: VID, - dst: VID, - eid: EID, - layer_id: usize, - props: &[(usize, Prop)], - ) -> Result; - - fn log_node_id( - &self, - transaction_id: TransactionID, - gid: GID, - vid: VID, - ) -> Result; - - fn log_edge_id( - &self, - transaction_id: TransactionID, - src: VID, - dst: VID, + src_name: GID, + src_id: VID, + dst_name: GID, + dst_id: VID, eid: EID, + layer_name: Option<&str>, layer_id: usize, - ) -> Result; - - /// Log constant prop name -> prop id mappings. - /// - /// # Arguments - /// - /// * `transaction_id` - The transaction ID - /// * `props` - A slice containing new or existing tuples of (prop name, id, value) - fn log_const_prop_ids>( - &self, - transaction_id: TransactionID, - props: &[MaybeNew<(PN, usize, Prop)>], - ) -> Result; - - /// Log temporal prop name -> prop id mappings. - /// - /// # Arguments - /// - /// * `transaction_id` - The transaction ID - /// * `props` - A slice containing new or existing tuples of (prop name, id, value). - fn log_temporal_prop_ids>( - &self, - transaction_id: TransactionID, props: &[MaybeNew<(PN, usize, Prop)>], ) -> Result; - fn log_layer_id( - &self, - transaction_id: TransactionID, - name: &str, - id: usize, - ) -> Result; - /// Logs a checkpoint record, indicating that all Wal operations upto and including /// `lsn` has been persisted to disk. fn log_checkpoint(&self, lsn: LSN) -> Result; @@ -153,66 +78,18 @@ pub trait GraphWal { /// Trait for defining callbacks for replaying from wal pub trait GraphReplayer { - fn replay_begin_transaction( - &self, - lsn: LSN, - transaction_id: TransactionID, - ) -> Result<(), StorageError>; - - fn replay_end_transaction( - &self, - lsn: LSN, - transaction_id: TransactionID, - ) -> Result<(), StorageError>; - - fn replay_add_static_edge( + fn replay_add_edge>( &self, lsn: LSN, transaction_id: TransactionID, t: TimeIndexEntry, - src: VID, - dst: VID, - ) -> Result<(), StorageError>; - - fn replay_add_edge( - &self, - lsn: LSN, - transaction_id: TransactionID, - t: TimeIndexEntry, - src: VID, - dst: VID, + src_name: GID, + src_id: VID, + dst_name: GID, + dst_id: VID, eid: EID, + layer_name: Option<&str>, layer_id: usize, - props: &[(usize, Prop)], - ) -> Result<(), StorageError>; - - fn replay_node_id( - &self, - lsn: LSN, - transaction_id: TransactionID, - gid: GID, - vid: VID, - ) -> Result<(), StorageError>; - - fn replay_const_prop_ids>( - &self, - lsn: LSN, - transaction_id: TransactionID, - props: &[MaybeNew<(PN, usize, Prop)>], - ) -> Result<(), StorageError>; - - fn replay_temporal_prop_ids>( - &self, - lsn: LSN, - transaction_id: TransactionID, props: &[MaybeNew<(PN, usize, Prop)>], ) -> Result<(), StorageError>; - - fn replay_layer_id( - &self, - lsn: LSN, - transaction_id: TransactionID, - name: &str, - id: usize, - ) -> Result<(), StorageError>; } diff --git a/raphtory-storage/src/mutation/addition_ops.rs b/raphtory-storage/src/mutation/addition_ops.rs index 3ac593e091..9fca8f2570 100644 --- a/raphtory-storage/src/mutation/addition_ops.rs +++ b/raphtory-storage/src/mutation/addition_ops.rs @@ -36,6 +36,7 @@ pub trait InternalAdditionOps { /// map layer name to id and allocate a new layer if needed fn resolve_layer(&self, layer: Option<&str>) -> Result, Self::Error>; + /// map external node id to internal id, allocating a new empty node if needed fn resolve_node(&self, id: NodeRef) -> Result, Self::Error>; diff --git a/raphtory/src/db/replay/mod.rs b/raphtory/src/db/replay/mod.rs index ab6a7e32ca..b733c1b46a 100644 --- a/raphtory/src/db/replay/mod.rs +++ b/raphtory/src/db/replay/mod.rs @@ -25,43 +25,19 @@ impl ReplayGraph { } impl GraphReplayer for ReplayGraph { - fn replay_begin_transaction( - &self, - lsn: LSN, - transaction_id: TransactionID, - ) -> Result<(), StorageError> { - Ok(()) - } - - fn replay_end_transaction( - &self, - lsn: LSN, - transaction_id: TransactionID, - ) -> Result<(), StorageError> { - Ok(()) - } - - fn replay_add_static_edge( + fn replay_add_edge>( &self, lsn: LSN, transaction_id: TransactionID, t: TimeIndexEntry, - src: VID, - dst: VID, - ) -> Result<(), StorageError> { - Ok(()) - } - - fn replay_add_edge( - &self, - lsn: LSN, - transaction_id: TransactionID, - t: TimeIndexEntry, - src: VID, - dst: VID, + src_name: GID, + src_id: VID, + dst_name: GID, + dst_id: VID, eid: EID, + layer_name: Option<&str>, layer_id: usize, - props: &[(usize, Prop)], + props: &[MaybeNew<(PN, usize, Prop)>], ) -> Result<(), StorageError> { let edge_segment = self.graph.storage().edges().get_edge_segment(eid); @@ -76,42 +52,4 @@ impl GraphReplayer for ReplayGraph { Ok(()) } - - fn replay_node_id( - &self, - lsn: LSN, - transaction_id: TransactionID, - gid: GID, - vid: VID, - ) -> Result<(), StorageError> { - Ok(()) - } - - fn replay_const_prop_ids>( - &self, - lsn: LSN, - transaction_id: TransactionID, - props: &[MaybeNew<(PN, usize, Prop)>], - ) -> Result<(), StorageError> { - Ok(()) - } - - fn replay_temporal_prop_ids>( - &self, - lsn: LSN, - transaction_id: TransactionID, - props: &[MaybeNew<(PN, usize, Prop)>], - ) -> Result<(), StorageError> { - Ok(()) - } - - fn replay_layer_id( - &self, - lsn: LSN, - transaction_id: TransactionID, - name: &str, - id: usize, - ) -> Result<(), StorageError> { - Ok(()) - } } From c192d648a22d68559f08ad3d99f6c803e72d3002 Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Tue, 2 Dec 2025 15:12:54 -0500 Subject: [PATCH 05/19] Add sketch of correct logging to add_edge --- db4-storage/src/wal/mod.rs | 2 +- raphtory/src/db/api/mutation/addition_ops.rs | 100 ++++++++----------- 2 files changed, 42 insertions(+), 60 deletions(-) diff --git a/db4-storage/src/wal/mod.rs b/db4-storage/src/wal/mod.rs index 912e7b2647..36772e704c 100644 --- a/db4-storage/src/wal/mod.rs +++ b/db4-storage/src/wal/mod.rs @@ -76,7 +76,7 @@ pub trait GraphWal { ) -> Result<(), StorageError>; } -/// Trait for defining callbacks for replaying from wal +/// Trait for defining callbacks for replaying from wal. pub trait GraphReplayer { fn replay_add_edge>( &self, diff --git a/raphtory/src/db/api/mutation/addition_ops.rs b/raphtory/src/db/api/mutation/addition_ops.rs index 351627bfbb..5d8024f2b0 100644 --- a/raphtory/src/db/api/mutation/addition_ops.rs +++ b/raphtory/src/db/api/mutation/addition_ops.rs @@ -14,6 +14,7 @@ use crate::{ prelude::{GraphViewOps, NodeViewOps}, }; use raphtory_api::core::entities::properties::prop::Prop; +use raphtory_core::entities::GID; use raphtory_storage::mutation::addition_ops::{EdgeWriteLock, InternalAdditionOps}; use raphtory_storage::mutation::durability_ops::DurabilityOps; use storage::wal::{GraphWal, Wal}; @@ -249,7 +250,6 @@ impl> + StaticGraphViewOps + Dura props: PII, layer: Option<&str>, ) -> Result, GraphError> { - // Log transaction start let transaction_id = self.transaction_manager().begin_transaction(); let session = self.write_session().map_err(|err| err.into())?; @@ -268,19 +268,6 @@ impl> + StaticGraphViewOps + Dura ) .map_err(into_graph_err)?; - // Log prop name -> prop id mappings - self.wal() - .log_temporal_prop_ids(transaction_id, &props_with_status) - .unwrap(); - - let props = props_with_status - .into_iter() - .map(|maybe_new| { - let (_, prop_id, prop) = maybe_new.inner(); - (prop_id, prop) - }) - .collect::>(); - let ti = time_from_input_session(&session, t)?; let src_id = self .resolve_node(src.as_node_ref()) @@ -290,76 +277,71 @@ impl> + StaticGraphViewOps + Dura .map_err(into_graph_err)?; let layer_id = self.resolve_layer(layer).map_err(into_graph_err)?; - // Log node -> node id mappings // FIXME: We are logging node -> node id mappings AFTER they are inserted into the // resolver. Make sure resolver mapping CANNOT get to disk before Wal. - if let Some(gid) = src.as_node_ref().as_gid_ref().left() { - self.wal() - .log_node_id(transaction_id, gid.into(), src_id.inner()) - .unwrap(); - } - - if let Some(gid) = dst.as_node_ref().as_gid_ref().left() { - self.wal() - .log_node_id(transaction_id, gid.into(), dst_id.inner()) - .unwrap(); - } + let src_gid = src.as_node_ref().as_gid_ref().left().map(|gid_ref| GID::from(gid_ref)).unwrap(); + let dst_gid = dst.as_node_ref().as_gid_ref().left().map(|gid_ref| GID::from(gid_ref)).unwrap(); let src_id = src_id.inner(); let dst_id = dst_id.inner(); - // Log layer -> layer id mappings - if let Some(layer) = layer { - self.wal() - .log_layer_id(transaction_id, layer, layer_id.inner()) - .unwrap(); - } - let layer_id = layer_id.inner(); - // Holds all locks for nodes and edge until add_edge_op goes out of scope + // Hold all locks for src node, dst node and edge until add_edge_op goes out of scope. let mut add_edge_op = self .atomic_add_edge(src_id, dst_id, None, layer_id) .map_err(into_graph_err)?; - // Log edge addition - let add_static_edge_lsn = self - .wal() - .log_add_static_edge(transaction_id, ti, src_id, dst_id) - .unwrap(); - let edge_id = add_edge_op.internal_add_static_edge(src_id, dst_id, add_static_edge_lsn); - - // Log edge -> edge id mappings - // NOTE: We log edge id mappings after they are inserted into edge segments. - // This is fine as long as we hold onto segment locks for the entire operation. - let add_edge_lsn = self - .wal() - .log_add_edge( - transaction_id, - ti, - src_id, - dst_id, - edge_id.inner(), - layer_id, - &props, - ) - .unwrap(); + // NOTE: We log edge id after it is inserted into the edge segment. + // This is fine as long as we hold onto the edge segment lock through add_edge_op + // for the entire operation. + let edge_id = add_edge_op.internal_add_static_edge(src_id, dst_id, 0); + + // All names, ids and values have been generated for this operation. + // Create a wal entry to mark it as durable. + let lsn = self.wal().log_add_edge( + transaction_id, + ti, + src_gid, + src_id, + dst_gid, + dst_id, + edge_id.inner(), + layer, + layer_id, + &props_with_status, + ).unwrap(); + + let props = props_with_status + .into_iter() + .map(|maybe_new| { + let (_, prop_id, prop) = maybe_new.inner(); + (prop_id, prop) + }) + .collect::>(); + let edge_id = add_edge_op.internal_add_edge( ti, src_id, dst_id, edge_id.map(|eid| eid.with_layer(layer_id)), - add_edge_lsn, + 0, props, ); add_edge_op.store_src_node_info(src_id, src.as_node_ref().as_gid_ref().left()); add_edge_op.store_dst_node_info(dst_id, dst.as_node_ref().as_gid_ref().left()); - // Log transaction end. + // Update the src, dst and edge segments with the lsn of the wal entry. + // add_edge_op.update_lsn(lsn); + self.transaction_manager().end_transaction(transaction_id); - // Flush all wal entries to disk. + // Drop to release all the segment locks. + // FIXME: Make sure segments cannot get to disk before wal entry is flushed. + // drop(add_edge_op); + + // Flush the wal entry to disk. self.wal().sync().unwrap(); Ok(EdgeView::new( From fd49e14571effb263e25608e16ecb419523e60f4 Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Tue, 2 Dec 2025 15:48:17 -0500 Subject: [PATCH 06/19] Add lsn to MemNodeSegment/MemEdgeSegment --- db4-storage/src/segments/edge.rs | 14 ++++++++++++-- db4-storage/src/segments/node.rs | 9 ++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/db4-storage/src/segments/edge.rs b/db4-storage/src/segments/edge.rs index 19a9045945..515f7cd829 100644 --- a/db4-storage/src/segments/edge.rs +++ b/db4-storage/src/segments/edge.rs @@ -51,6 +51,7 @@ impl HasRow for MemPageEntry { pub struct MemEdgeSegment { layers: Vec>, est_size: usize, + lsn: u64, } impl>> From for MemEdgeSegment { @@ -61,7 +62,11 @@ impl>> From for MemEdge !layers.is_empty(), "MemEdgeSegment must have at least one layer" ); - Self { layers, est_size } + Self { + layers, + est_size, + lsn: 0, + } } } @@ -82,6 +87,7 @@ impl MemEdgeSegment { Self { layers: vec![SegmentContainer::new(segment_id, max_page_len, meta)], est_size: 0, + lsn: 0, } } @@ -128,7 +134,11 @@ impl MemEdgeSegment { } pub fn lsn(&self) -> u64 { - self.layers.iter().map(|seg| seg.lsn()).min().unwrap_or(0) + self.lsn + } + + pub fn set_lsn(&mut self, lsn: u64) { + self.lsn = lsn; } pub fn max_page_len(&self) -> u32 { diff --git a/db4-storage/src/segments/node.rs b/db4-storage/src/segments/node.rs index 63b1d9f05d..e548d9e108 100644 --- a/db4-storage/src/segments/node.rs +++ b/db4-storage/src/segments/node.rs @@ -34,6 +34,7 @@ pub struct MemNodeSegment { segment_id: usize, max_page_len: u32, layers: Vec>, + lsn: u64, } impl>> From for MemNodeSegment { @@ -49,6 +50,7 @@ impl>> From for MemNodeSegm segment_id, max_page_len, layers, + lsn: 0, } } } @@ -140,7 +142,11 @@ impl MemNodeSegment { } pub fn lsn(&self) -> u64 { - self.layers.iter().map(|seg| seg.lsn()).min().unwrap_or(0) + self.lsn + } + + pub fn set_lsn(&mut self, lsn: u64) { + self.lsn = lsn; } pub fn to_vid(&self, pos: LocalPOS) -> VID { @@ -188,6 +194,7 @@ impl MemNodeSegment { segment_id, max_page_len, layers: vec![SegmentContainer::new(segment_id, max_page_len, meta)], + lsn: 0, } } From aa9f84b04805545dbb23ddf645c778b706e2b429 Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Tue, 2 Dec 2025 15:51:29 -0500 Subject: [PATCH 07/19] Add set_lsn method for AtomicAddEdge --- db4-storage/src/pages/session.rs | 19 +++++++++++++++++++ db4-storage/src/segments/node.rs | 4 +++- raphtory-storage/src/mutation/addition_ops.rs | 4 +++- .../src/mutation/addition_ops_ext.rs | 7 ++++++- raphtory/src/db/api/mutation/addition_ops.rs | 2 +- raphtory/src/db/api/storage/storage.rs | 6 +++++- 6 files changed, 37 insertions(+), 5 deletions(-) diff --git a/db4-storage/src/pages/session.rs b/db4-storage/src/pages/session.rs index 9d6c48eccb..2619f21a46 100644 --- a/db4-storage/src/pages/session.rs +++ b/db4-storage/src/pages/session.rs @@ -6,6 +6,7 @@ use crate::{ api::{edges::EdgeSegmentOps, nodes::NodeSegmentOps}, persist::strategy::{Config, PersistentStrategy}, segments::{edge::MemEdgeSegment, node::MemNodeSegment}, + wal::LSN, }; use parking_lot::RwLockWriteGuard; use raphtory_api::core::{entities::properties::prop::Prop, storage::dict_mapper::MaybeNew}; @@ -220,4 +221,22 @@ impl< ) -> &mut WriterPair<'a, RwLockWriteGuard<'a, MemNodeSegment>, NS> { &mut self.node_writers } + + pub fn set_lsn(&mut self, lsn: LSN) { + match &mut self.node_writers { + WriterPair::Same { writer } => { + writer.mut_segment.set_lsn(lsn); + } + WriterPair::Different { + src_writer, + dst_writer, + } => { + src_writer.mut_segment.set_lsn(lsn); + dst_writer.mut_segment.set_lsn(lsn); + } + } + if let Some(edge_writer) = &mut self.edge_writer { + edge_writer.writer.set_lsn(lsn); + } + } } diff --git a/db4-storage/src/segments/node.rs b/db4-storage/src/segments/node.rs index e548d9e108..2f4d52595e 100644 --- a/db4-storage/src/segments/node.rs +++ b/db4-storage/src/segments/node.rs @@ -146,7 +146,9 @@ impl MemNodeSegment { } pub fn set_lsn(&mut self, lsn: u64) { - self.lsn = lsn; + if lsn > self.lsn { + self.lsn = lsn; + } } pub fn to_vid(&self, pos: LocalPOS) -> VID { diff --git a/raphtory-storage/src/mutation/addition_ops.rs b/raphtory-storage/src/mutation/addition_ops.rs index 9fca8f2570..808f9ce04d 100644 --- a/raphtory-storage/src/mutation/addition_ops.rs +++ b/raphtory-storage/src/mutation/addition_ops.rs @@ -20,7 +20,7 @@ use raphtory_api::{ inherit::Base, }; use raphtory_core::entities::{nodes::node_ref::NodeRef, ELID}; -use storage::{Extension}; +use storage::{Extension, wal::LSN}; pub trait InternalAdditionOps { type Error: From; @@ -124,6 +124,8 @@ pub trait EdgeWriteLock: Send + Sync { fn store_src_node_info(&mut self, id: impl Into, node_id: Option); fn store_dst_node_info(&mut self, id: impl Into, node_id: Option); + + fn set_lsn(&mut self, lsn: LSN); } pub trait AtomicNodeAddition: Send + Sync { diff --git a/raphtory-storage/src/mutation/addition_ops_ext.rs b/raphtory-storage/src/mutation/addition_ops_ext.rs index 875027319a..0141fc6a53 100644 --- a/raphtory-storage/src/mutation/addition_ops_ext.rs +++ b/raphtory-storage/src/mutation/addition_ops_ext.rs @@ -17,7 +17,7 @@ use raphtory_core::{ nodes::node_ref::{AsNodeRef, NodeRef}, GidRef, EID, ELID, MAX_LAYER, VID, }, - storage::timeindex::TimeIndexEntry, + storage::{timeindex::TimeIndexEntry}, }; use storage::{ pages::{node_page::writer::node_info_as_props, session::WriteSession}, @@ -25,6 +25,7 @@ use storage::{ properties::props_meta_writer::PropsMetaWriter, resolver::GIDResolverOps, Extension, transaction::TransactionManager, WalImpl, ES, NS, + wal::LSN, }; pub struct WriteS<'a, EXT: PersistentStrategy, ES = ES>> { @@ -103,6 +104,10 @@ impl<'a, EXT: PersistentStrategy, ES = ES>> EdgeWriteLock for .update_c_props(pos, 0, [(NODE_ID_IDX, id.into())], 0); }; } + + fn set_lsn(&mut self, lsn: LSN) { + self.static_session.set_lsn(lsn); + } } impl<'a> SessionAdditionOps for UnlockedSession<'a> { diff --git a/raphtory/src/db/api/mutation/addition_ops.rs b/raphtory/src/db/api/mutation/addition_ops.rs index 5d8024f2b0..b10b0435f1 100644 --- a/raphtory/src/db/api/mutation/addition_ops.rs +++ b/raphtory/src/db/api/mutation/addition_ops.rs @@ -333,7 +333,7 @@ impl> + StaticGraphViewOps + Dura add_edge_op.store_dst_node_info(dst_id, dst.as_node_ref().as_gid_ref().left()); // Update the src, dst and edge segments with the lsn of the wal entry. - // add_edge_op.update_lsn(lsn); + add_edge_op.set_lsn(lsn); self.transaction_manager().end_transaction(transaction_id); diff --git a/raphtory/src/db/api/storage/storage.rs b/raphtory/src/db/api/storage/storage.rs index 91702aa4a2..d58db03725 100644 --- a/raphtory/src/db/api/storage/storage.rs +++ b/raphtory/src/db/api/storage/storage.rs @@ -36,7 +36,7 @@ use std::{ path::Path, sync::Arc, }; -use storage::{Extension, transaction::TransactionManager, WalImpl}; +use storage::{Extension, transaction::TransactionManager, WalImpl, wal::LSN}; #[cfg(feature = "search")] use { @@ -338,6 +338,10 @@ impl EdgeWriteLock for AtomicAddEdgeSession<'_> { fn store_dst_node_info(&mut self, id: impl Into, node_id: Option) { self.session.store_dst_node_info(id, node_id); } + + fn set_lsn(&mut self, lsn: LSN) { + self.session.set_lsn(lsn); + } } impl<'a> SessionAdditionOps for StorageWriteSession<'a> { From c2c77fab54d0c7e599cb3e6b74566ba688f706d1 Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Tue, 2 Dec 2025 16:27:57 -0500 Subject: [PATCH 08/19] Simplify WriterPair to NodeWriters --- db4-storage/src/pages/mod.rs | 34 ++++++++++------------- db4-storage/src/pages/node_page/writer.rs | 32 ++++++--------------- db4-storage/src/pages/session.rs | 24 ++++++---------- 3 files changed, 33 insertions(+), 57 deletions(-) diff --git a/db4-storage/src/pages/mod.rs b/db4-storage/src/pages/mod.rs index 58d7623f00..0420109d5c 100644 --- a/db4-storage/src/pages/mod.rs +++ b/db4-storage/src/pages/mod.rs @@ -9,7 +9,7 @@ use crate::{ }; use edge_page::writer::EdgeWriter; use edge_store::EdgeStorageInner; -use node_page::writer::{NodeWriter, WriterPair}; +use node_page::writer::{NodeWriter, NodeWriters}; use node_store::NodeStorageInner; use parking_lot::RwLockWriteGuard; use raphtory_api::core::{ @@ -315,23 +315,18 @@ impl< let (src_chunk, _) = self.nodes.resolve_pos(src); let (dst_chunk, _) = self.nodes.resolve_pos(dst); + // Acquire locks in consistent order (lower chunk ID first) to prevent deadlocks. let node_writers = if src_chunk < dst_chunk { - let src_writer = self.node_writer(src_chunk); - let dst_writer = self.node_writer(dst_chunk); - WriterPair::Different { - src_writer, - dst_writer, - } + let src = self.node_writer(src_chunk); + let dst = self.node_writer(dst_chunk); + NodeWriters { src, dst: Some(dst) } } else if src_chunk > dst_chunk { - let dst_writer = self.node_writer(dst_chunk); - let src_writer = self.node_writer(src_chunk); - WriterPair::Different { - src_writer, - dst_writer, - } + let dst = self.node_writer(dst_chunk); + let src = self.node_writer(src_chunk); + NodeWriters { src, dst: Some(dst) } } else { - let writer = self.node_writer(src_chunk); - WriterPair::Same { writer } + let src = self.node_writer(src_chunk); + NodeWriters { src, dst: None } }; let edge_writer = e_id.map(|e_id| self.edge_writer(e_id)); @@ -352,19 +347,20 @@ impl< self.nodes().get_or_create_segment(src_chunk); self.nodes().get_or_create_segment(dst_chunk); + // FIXME: This can livelock due to inconsistent lock acquisition order. loop { if let Some(src_writer) = self.nodes().try_writer(src_chunk) { if let Some(dst_writer) = self.nodes().try_writer(dst_chunk) { - break WriterPair::Different { - src_writer, - dst_writer, + break NodeWriters { + src: src_writer, + dst: Some(dst_writer), }; } } } } else { let writer = self.node_writer(src_chunk); - WriterPair::Same { writer } + NodeWriters { src: writer, dst: None } }; let edge_writer = e_id.map(|e_id| self.edge_writer(e_id)); diff --git a/db4-storage/src/pages/node_page/writer.rs b/db4-storage/src/pages/node_page/writer.rs index f11e6ce498..4973943281 100644 --- a/db4-storage/src/pages/node_page/writer.rs +++ b/db4-storage/src/pages/node_page/writer.rs @@ -231,34 +231,20 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> Drop } } -pub enum WriterPair<'a, MP: DerefMut, NS: NodeSegmentOps> { - Same { - writer: NodeWriter<'a, MP, NS>, - }, - Different { - src_writer: NodeWriter<'a, MP, NS>, - dst_writer: NodeWriter<'a, MP, NS>, - }, + +/// Holds writers for src and dst node segments when adding an edge. +/// If both nodes are in the same segment, `dst` is `None` and `src` is used for both. +pub struct NodeWriters<'a, MP: DerefMut, NS: NodeSegmentOps> { + pub src: NodeWriter<'a, MP, NS>, + pub dst: Option>, } -impl<'a, MP: DerefMut, NS: NodeSegmentOps> WriterPair<'a, MP, NS> { +impl<'a, MP: DerefMut, NS: NodeSegmentOps> NodeWriters<'a, MP, NS> { pub fn get_mut_src(&mut self) -> &mut NodeWriter<'a, MP, NS> { - match self { - WriterPair::Same { writer, .. } => writer, - WriterPair::Different { - src_writer: writer_i, - .. - } => writer_i, - } + &mut self.src } pub fn get_mut_dst(&mut self) -> &mut NodeWriter<'a, MP, NS> { - match self { - WriterPair::Same { writer, .. } => writer, - WriterPair::Different { - dst_writer: writer_j, - .. - } => writer_j, - } + self.dst.as_mut().unwrap_or(&mut self.src) } } diff --git a/db4-storage/src/pages/session.rs b/db4-storage/src/pages/session.rs index 2619f21a46..e2093f627b 100644 --- a/db4-storage/src/pages/session.rs +++ b/db4-storage/src/pages/session.rs @@ -1,5 +1,5 @@ use super::{ - GraphStore, edge_page::writer::EdgeWriter, node_page::writer::WriterPair, resolve_pos, + GraphStore, edge_page::writer::EdgeWriter, node_page::writer::NodeWriters, resolve_pos, }; use crate::{ LocalPOS, @@ -16,7 +16,7 @@ use raphtory_core::{ }; pub struct WriteSession<'a, NS: NodeSegmentOps, ES: EdgeSegmentOps, EXT: Config> { - node_writers: WriterPair<'a, RwLockWriteGuard<'a, MemNodeSegment>, NS>, + node_writers: NodeWriters<'a, RwLockWriteGuard<'a, MemNodeSegment>, NS>, edge_writer: Option, ES>>, graph: &'a GraphStore, } @@ -29,7 +29,7 @@ impl< > WriteSession<'a, NS, ES, EXT> { pub fn new( - node_writers: WriterPair<'a, RwLockWriteGuard<'a, MemNodeSegment>, NS>, + node_writers: NodeWriters<'a, RwLockWriteGuard<'a, MemNodeSegment>, NS>, edge_writer: Option, ES>>, graph: &'a GraphStore, ) -> Self { @@ -218,23 +218,17 @@ impl< pub fn node_writers( &mut self, - ) -> &mut WriterPair<'a, RwLockWriteGuard<'a, MemNodeSegment>, NS> { + ) -> &mut NodeWriters<'a, RwLockWriteGuard<'a, MemNodeSegment>, NS> { &mut self.node_writers } pub fn set_lsn(&mut self, lsn: LSN) { - match &mut self.node_writers { - WriterPair::Same { writer } => { - writer.mut_segment.set_lsn(lsn); - } - WriterPair::Different { - src_writer, - dst_writer, - } => { - src_writer.mut_segment.set_lsn(lsn); - dst_writer.mut_segment.set_lsn(lsn); - } + self.node_writers.src.mut_segment.set_lsn(lsn); + + if let Some(dst) = &mut self.node_writers.dst { + dst.mut_segment.set_lsn(lsn); } + if let Some(edge_writer) = &mut self.edge_writer { edge_writer.writer.set_lsn(lsn); } From f263616ad88f77e9aa9274e6558103fd7e6472a9 Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Wed, 3 Dec 2025 15:01:34 -0500 Subject: [PATCH 09/19] Remove lsn args --- db4-storage/src/pages/edge_page/writer.rs | 9 ++--- db4-storage/src/pages/mod.rs | 9 +++-- db4-storage/src/pages/node_page/writer.rs | 37 +++++++------------ db4-storage/src/pages/session.rs | 33 +++++++---------- db4-storage/src/pages/test_utils/checkers.rs | 10 +++-- db4-storage/src/segments/edge.rs | 20 +--------- db4-storage/src/segments/mod.rs | 12 ------ db4-storage/src/segments/node.rs | 20 ++++------ raphtory-storage/src/mutation/addition_ops.rs | 3 -- .../src/mutation/addition_ops_ext.rs | 19 ++++------ raphtory-storage/src/mutation/deletion_ops.rs | 7 ++-- .../src/mutation/property_addition_ops.rs | 4 +- raphtory/src/db/api/mutation/addition_ops.rs | 3 +- raphtory/src/db/api/mutation/deletion_ops.rs | 2 +- raphtory/src/db/api/storage/storage.rs | 9 ++--- raphtory/src/db/api/view/graph.rs | 28 ++++++-------- raphtory/src/db/graph/edge.rs | 1 - raphtory/src/io/arrow/df_loaders.rs | 30 +++++++-------- 18 files changed, 94 insertions(+), 162 deletions(-) diff --git a/db4-storage/src/pages/edge_page/writer.rs b/db4-storage/src/pages/edge_page/writer.rs index 3348ba2510..9eb43eab37 100644 --- a/db4-storage/src/pages/edge_page/writer.rs +++ b/db4-storage/src/pages/edge_page/writer.rs @@ -42,7 +42,6 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen dst: VID, props: impl IntoIterator, layer_id: usize, - lsn: u64, ) -> LocalPOS { let existing_edge = self .page @@ -52,7 +51,7 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen } self.graph_stats.update_time(t.t()); self.writer - .insert_edge_internal(t, edge_pos, src, dst, layer_id, props, lsn); + .insert_edge_internal(t, edge_pos, src, dst, layer_id, props); edge_pos } @@ -88,7 +87,6 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen src: VID, dst: VID, layer_id: usize, - lsn: u64, ) { let existing_edge = self .page @@ -98,7 +96,7 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen } self.graph_stats.update_time(t.t()); self.writer - .delete_edge_internal(t, edge_pos, src, dst, layer_id, lsn); + .delete_edge_internal(t, edge_pos, src, dst, layer_id); } pub fn add_static_edge( @@ -106,7 +104,6 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen edge_pos: Option, src: impl Into, dst: impl Into, - lsn: u64, exists_hint: Option, // used when edge_pos is Some but the is not counted, this is used in the bulk loader ) -> LocalPOS { let layer_id = 0; // assuming layer_id 0 for static edges, adjust as needed @@ -117,7 +114,7 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen let edge_pos = edge_pos.unwrap_or_else(|| self.new_local_pos(layer_id)); self.writer - .insert_static_edge_internal(edge_pos, src, dst, layer_id, lsn); + .insert_static_edge_internal(edge_pos, src, dst, layer_id); edge_pos } diff --git a/db4-storage/src/pages/mod.rs b/db4-storage/src/pages/mod.rs index 0420109d5c..c029abf75f 100644 --- a/db4-storage/src/pages/mod.rs +++ b/db4-storage/src/pages/mod.rs @@ -213,10 +213,11 @@ impl< let src = src.into(); let dst = dst.into(); let mut session = self.write_session(src, dst, None); + session.set_lsn(lsn); let elid = session - .add_static_edge(src, dst, lsn) + .add_static_edge(src, dst) .map(|eid| eid.with_layer(0)); - session.add_edge_into_layer(t, src, dst, elid, lsn, props); + session.add_edge_into_layer(t, src, dst, elid, props); Ok(elid) } @@ -284,7 +285,7 @@ impl< let (segment, node_pos) = self.nodes.resolve_pos(node); let mut node_writer = self.nodes.writer(segment); let prop_writer = PropsMetaWriter::constant(self.node_meta(), props.into_iter())?; - node_writer.update_c_props(node_pos, layer_id, prop_writer.into_props_const()?, 0); // TODO: LSN + node_writer.update_c_props(node_pos, layer_id, prop_writer.into_props_const()?); Ok(()) } @@ -302,7 +303,7 @@ impl< let mut node_writer = self.nodes.writer(segment); let prop_writer = PropsMetaWriter::temporal(self.node_meta(), props.into_iter())?; - node_writer.add_props(t, node_pos, layer_id, prop_writer.into_props_temporal()?, 0); // TODO: LSN + node_writer.add_props(t, node_pos, layer_id, prop_writer.into_props_temporal()?); Ok(()) } diff --git a/db4-storage/src/pages/node_page/writer.rs b/db4-storage/src/pages/node_page/writer.rs index 4973943281..82a3eaf0ca 100644 --- a/db4-storage/src/pages/node_page/writer.rs +++ b/db4-storage/src/pages/node_page/writer.rs @@ -37,9 +37,8 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri src_pos: impl Into, dst: impl Into, e_id: impl Into, - lsn: u64, ) { - self.add_outbound_edge_inner(t, src_pos, dst, e_id, lsn); + self.add_outbound_edge_inner(t, src_pos, dst, e_id); } pub fn add_static_outbound_edge( @@ -47,10 +46,9 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri src_pos: LocalPOS, dst: impl Into, e_id: impl Into, - lsn: u64, ) { let e_id = e_id.into(); - self.add_outbound_edge_inner::(None, src_pos, dst, e_id.with_layer(0), lsn); + self.add_outbound_edge_inner::(None, src_pos, dst, e_id.with_layer(0)); } fn add_outbound_edge_inner( @@ -59,7 +57,6 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri src_pos: impl Into, dst: impl Into, e_id: impl Into, - lsn: u64, ) { let src_pos = src_pos.into(); let dst = dst.into(); @@ -71,7 +68,7 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri let layer_id = e_id.layer(); let (is_new_node, add) = self .mut_segment - .add_outbound_edge(t, src_pos, dst, e_id, lsn); + .add_outbound_edge(t, src_pos, dst, e_id); self.page.increment_est_size(add); if is_new_node && !self.page.check_node(src_pos, layer_id) { @@ -85,9 +82,8 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri dst_pos: impl Into, src: impl Into, e_id: impl Into, - lsn: u64, ) { - self.add_inbound_edge_inner(t, dst_pos, src, e_id, lsn); + self.add_inbound_edge_inner(t, dst_pos, src, e_id); } pub fn add_static_inbound_edge( @@ -95,10 +91,9 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri dst_pos: LocalPOS, src: impl Into, e_id: impl Into, - lsn: u64, ) { let e_id = e_id.into(); - self.add_inbound_edge_inner::(None, dst_pos, src, e_id.with_layer(0), lsn); + self.add_inbound_edge_inner::(None, dst_pos, src, e_id.with_layer(0)); } fn add_inbound_edge_inner( @@ -107,7 +102,6 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri dst_pos: impl Into, src: impl Into, e_id: impl Into, - lsn: u64, ) { let e_id = e_id.into(); let src = src.into(); @@ -118,7 +112,7 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri let dst_pos = dst_pos.into(); let (is_new_node, add) = self .mut_segment - .add_inbound_edge(t, dst_pos, src, e_id, lsn); + .add_inbound_edge(t, dst_pos, src, e_id); self.page.increment_est_size(add); @@ -133,11 +127,9 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri pos: LocalPOS, layer_id: usize, props: impl IntoIterator, - lsn: u64, ) { self.l_counter.update_time(t.t()); let (is_new_node, add) = self.mut_segment.add_props(t, pos, layer_id, props); - self.mut_segment.as_mut()[layer_id].set_lsn(lsn); self.page.increment_est_size(add); if is_new_node && !self.page.check_node(pos, layer_id) { self.l_counter.increment(layer_id); @@ -158,10 +150,8 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri pos: LocalPOS, layer_id: usize, props: impl IntoIterator, - lsn: u64, ) { let (is_new_node, add) = self.mut_segment.update_c_props(pos, layer_id, props); - self.mut_segment.as_mut()[layer_id].set_lsn(lsn); self.page.increment_est_size(add); if is_new_node && !self.page.check_node(pos, layer_id) { self.l_counter.increment(layer_id); @@ -172,9 +162,9 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri self.mut_segment.get_metadata(pos, layer_id, prop_id) } - pub fn update_timestamp(&mut self, t: T, pos: LocalPOS, e_id: ELID, lsn: u64) { + pub fn update_timestamp(&mut self, t: T, pos: LocalPOS, e_id: ELID) { self.l_counter.update_time(t.t()); - let add = self.mut_segment.update_timestamp(t, pos, e_id, lsn); + let add = self.mut_segment.update_timestamp(t, pos, e_id); self.page.increment_est_size(add); } @@ -194,18 +184,17 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri layer_id: usize, gid: GidRef<'_>, node_type: usize, - lsn: u64, ) { let node_type = (node_type != 0).then_some(node_type); - self.update_c_props(pos, layer_id, node_info_as_props(Some(gid), node_type), lsn); + self.update_c_props(pos, layer_id, node_info_as_props(Some(gid), node_type)); } - pub fn store_node_id(&mut self, pos: LocalPOS, layer_id: usize, gid: GidRef<'_>, lsn: u64) { - self.update_c_props(pos, layer_id, node_info_as_props(Some(gid), None), lsn); + pub fn store_node_id(&mut self, pos: LocalPOS, layer_id: usize, gid: GidRef<'_>) { + self.update_c_props(pos, layer_id, node_info_as_props(Some(gid), None)); } - pub fn update_deletion_time(&mut self, t: T, node: LocalPOS, e_id: ELID, lsn: u64) { - self.update_timestamp(t, node, e_id, lsn); + pub fn update_deletion_time(&mut self, t: T, node: LocalPOS, e_id: ELID) { + self.update_timestamp(t, node, e_id); } } diff --git a/db4-storage/src/pages/session.rs b/db4-storage/src/pages/session.rs index e2093f627b..a997343859 100644 --- a/db4-storage/src/pages/session.rs +++ b/db4-storage/src/pages/session.rs @@ -50,7 +50,6 @@ impl< src: impl Into, dst: impl Into, edge: MaybeNew, - lsn: u64, props: impl IntoIterator, ) { let src = src.into(); @@ -67,13 +66,13 @@ impl< let edge_max_page_len = writer.writer.get_or_create_layer(layer).max_page_len(); let (_, edge_pos) = resolve_pos(e_id.edge, edge_max_page_len); - writer.add_edge(t, edge_pos, src, dst, props, layer, lsn); + writer.add_edge(t, edge_pos, src, dst, props, layer); } else { let mut writer = self.graph.edge_writer(e_id.edge); let edge_max_page_len = writer.writer.get_or_create_layer(layer).max_page_len(); let (_, edge_pos) = resolve_pos(e_id.edge, edge_max_page_len); - writer.add_edge(t, edge_pos, src, dst, props, layer, lsn); + writer.add_edge(t, edge_pos, src, dst, props, layer); self.edge_writer = Some(writer); // Attach edge_writer to hold onto locks } @@ -88,18 +87,18 @@ impl< { self.node_writers .get_mut_src() - .add_outbound_edge(Some(t), src_pos, dst, edge_id, lsn); + .add_outbound_edge(Some(t), src_pos, dst, edge_id); self.node_writers .get_mut_dst() - .add_inbound_edge(Some(t), dst_pos, src, edge_id, lsn); + .add_inbound_edge(Some(t), dst_pos, src, edge_id); } self.node_writers .get_mut_src() - .update_timestamp(t, src_pos, e_id, lsn); + .update_timestamp(t, src_pos, e_id); self.node_writers .get_mut_dst() - .update_timestamp(t, dst_pos, e_id, lsn); + .update_timestamp(t, dst_pos, e_id); } pub fn delete_edge_from_layer( @@ -108,7 +107,6 @@ impl< src: impl Into, dst: impl Into, edge: MaybeNew, - lsn: u64, ) { let src = src.into(); let dst = dst.into(); @@ -124,13 +122,13 @@ impl< let edge_max_page_len = writer.writer.get_or_create_layer(layer).max_page_len(); let (_, edge_pos) = resolve_pos(e_id.edge, edge_max_page_len); - writer.delete_edge(t, edge_pos, src, dst, layer, lsn); + writer.delete_edge(t, edge_pos, src, dst, layer); } else { let mut writer = self.graph.edge_writer(e_id.edge); let edge_max_page_len = writer.writer.get_or_create_layer(layer).max_page_len(); let (_, edge_pos) = resolve_pos(e_id.edge, edge_max_page_len); - writer.delete_edge(t, edge_pos, src, dst, layer, lsn); + writer.delete_edge(t, edge_pos, src, dst, layer); self.edge_writer = Some(writer); // Attach edge_writer to hold onto locks } @@ -149,23 +147,21 @@ impl< src_pos, dst, edge_id, - lsn, ); self.node_writers.get_mut_dst().add_inbound_edge( Some(t), dst_pos, src, edge_id, - lsn, ); } self.node_writers .get_mut_src() - .update_deletion_time(t, src_pos, e_id, lsn); + .update_deletion_time(t, src_pos, e_id); self.node_writers .get_mut_dst() - .update_deletion_time(t, dst_pos, e_id, lsn); + .update_deletion_time(t, dst_pos, e_id); } } @@ -173,7 +169,6 @@ impl< &mut self, src: impl Into, dst: impl Into, - lsn: u64, ) -> MaybeNew { let src = src.into(); let dst = dst.into(); @@ -194,12 +189,12 @@ impl< let edge_writer = self.edge_writer.as_mut().unwrap(); let (_, edge_pos) = self.graph.edges().resolve_pos(e_id); - edge_writer.add_static_edge(Some(edge_pos), src, dst, lsn, Some(true)); + edge_writer.add_static_edge(Some(edge_pos), src, dst, Some(true)); MaybeNew::Existing(e_id) } else { let mut edge_writer = self.graph.get_free_writer(); - let edge_id = edge_writer.add_static_edge(None, src, dst, lsn, Some(false)); + let edge_id = edge_writer.add_static_edge(None, src, dst, Some(false)); let edge_id = edge_id.as_eid(edge_writer.segment_id(), self.graph.edges().max_page_len()); @@ -207,10 +202,10 @@ impl< self.node_writers .get_mut_src() - .add_static_outbound_edge(src_pos, dst, edge_id, lsn); + .add_static_outbound_edge(src_pos, dst, edge_id); self.node_writers .get_mut_dst() - .add_static_inbound_edge(dst_pos, src, edge_id, lsn); + .add_static_inbound_edge(dst_pos, src, edge_id); MaybeNew::New(edge_id) } diff --git a/db4-storage/src/pages/test_utils/checkers.rs b/db4-storage/src/pages/test_utils/checkers.rs index 0f701c88c3..ec3f9e521e 100644 --- a/db4-storage/src/pages/test_utils/checkers.rs +++ b/db4-storage/src/pages/test_utils/checkers.rs @@ -56,9 +56,10 @@ pub fn make_graph_from_edges< let layer_id = layer_id.unwrap_or(0); let mut session = graph.write_session(*src, *dst, None); - let eid = session.add_static_edge(*src, *dst, lsn); + session.set_lsn(lsn); + let eid = session.add_static_edge(*src, *dst); let elid = eid.map(|eid| eid.with_layer(layer_id)); - session.add_edge_into_layer(timestamp, *src, *dst, elid, lsn, []); + session.add_edge_into_layer(timestamp, *src, *dst, elid, []); Ok::<_, StorageError>(()) }) @@ -73,9 +74,10 @@ pub fn make_graph_from_edges< let layer_id = layer_id.unwrap_or(0); let mut session = graph.write_session(*src, *dst, None); - let eid = session.add_static_edge(*src, *dst, lsn); + session.set_lsn(lsn); + let eid = session.add_static_edge(*src, *dst); let elid = eid.map(|e| e.with_layer(layer_id)); - session.add_edge_into_layer(timestamp, *src, *dst, elid, lsn, []); + session.add_edge_into_layer(timestamp, *src, *dst, elid, []); Ok::<_, StorageError>(()) }) diff --git a/db4-storage/src/segments/edge.rs b/db4-storage/src/segments/edge.rs index 515f7cd829..aa32b0cbb3 100644 --- a/db4-storage/src/segments/edge.rs +++ b/db4-storage/src/segments/edge.rs @@ -215,12 +215,10 @@ impl MemEdgeSegment { dst: VID, layer_id: usize, props: impl IntoIterator, - lsn: u64, ) { // Ensure we have enough layers self.ensure_layer(layer_id); let est_size = self.layers[layer_id].est_size(); - self.layers[layer_id].set_lsn(lsn); let local_row = self.reserve_local_row(edge_pos, src, dst, layer_id); @@ -240,14 +238,12 @@ impl MemEdgeSegment { src: VID, dst: VID, layer_id: usize, - lsn: u64, ) { let t = TimeIndexEntry::new(t.t(), t.i()); // Ensure we have enough layers self.ensure_layer(layer_id); let est_size = self.layers[layer_id].est_size(); - self.layers[layer_id].set_lsn(lsn); let local_row = self.reserve_local_row(edge_pos, src, dst, layer_id); let props = self.layers[layer_id].properties_mut(); @@ -262,14 +258,12 @@ impl MemEdgeSegment { src: impl Into, dst: impl Into, layer_id: usize, - lsn: u64, ) { let src = src.into(); let dst = dst.into(); // Ensure we have enough layers self.ensure_layer(layer_id); - self.layers[layer_id].set_lsn(lsn); let est_size = self.layers[layer_id].est_size(); self.reserve_local_row(edge_pos, src, dst, layer_id); @@ -613,7 +607,6 @@ mod test { VID(2), 0, vec![(0, Prop::from("test1"))], - 1, ); segment.insert_edge_internal( @@ -623,7 +616,6 @@ mod test { VID(4), 0, vec![(0, Prop::from("test2"))], - 2, ); segment.insert_edge_internal( @@ -633,7 +625,6 @@ mod test { VID(6), 0, vec![(0, Prop::from("test3"))], - 3, ); // Verify edges exist @@ -763,7 +754,6 @@ mod test { VID(2), 0, vec![(0, Prop::from("test1"))], - 1, ); segment1.insert_edge_internal( TimeIndexEntry::new(2, 1), @@ -772,7 +762,6 @@ mod test { VID(4), 0, vec![(0, Prop::from("test2"))], - 1, ); segment1.insert_edge_internal( TimeIndexEntry::new(3, 2), @@ -781,7 +770,6 @@ mod test { VID(6), 0, vec![(0, Prop::from("test3"))], - 1, ); // Equivalent bulk insertion @@ -831,7 +819,6 @@ mod test { VID(2), 0, vec![(0, Prop::from("individual1"))], - 1, ); // Bulk insert some edges @@ -863,7 +850,6 @@ mod test { VID(8), 0, vec![(0, Prop::from("individual2"))], - 1, ); // Another bulk insert @@ -983,14 +969,13 @@ mod test { VID(2), 0, vec![(0, Prop::from("test"))], - 1, ); let est_size1 = segment.est_size(); assert!(est_size1 > 0); - segment.delete_edge_internal(TimeIndexEntry::new(2, 3), LocalPOS(0), VID(5), VID(3), 0, 0); + segment.delete_edge_internal(TimeIndexEntry::new(2, 3), LocalPOS(0), VID(5), VID(3), 0); let est_size2 = segment.est_size(); @@ -1007,7 +992,6 @@ mod test { VID(6), 0, vec![(0, Prop::from("test2"))], - 1, ); let est_size3 = segment.est_size(); @@ -1018,7 +1002,7 @@ mod test { // Insert a static edge - segment.insert_static_edge_internal(LocalPOS(1), 4, 6, 0, 1); + segment.insert_static_edge_internal(LocalPOS(1), 4, 6, 0); let est_size4 = segment.est_size(); assert_eq!( diff --git a/db4-storage/src/segments/mod.rs b/db4-storage/src/segments/mod.rs index 7f9b8688ef..70eed8cbc0 100644 --- a/db4-storage/src/segments/mod.rs +++ b/db4-storage/src/segments/mod.rs @@ -158,7 +158,6 @@ pub struct SegmentContainer { max_page_len: u32, properties: Properties, meta: Arc, - lsn: u64, } pub trait HasRow: Default + Send + Sync + Sized { @@ -175,7 +174,6 @@ impl SegmentContainer { max_page_len, properties: Default::default(), meta, - lsn: 0, } } @@ -275,16 +273,6 @@ impl SegmentContainer { self.segment_id } - #[inline(always)] - pub fn lsn(&self) -> u64 { - self.lsn - } - - #[inline(always)] - pub fn set_lsn(&mut self, lsn: u64) { - self.lsn = lsn; - } - pub fn len(&self) -> u32 { self.data.data.len() as u32 } diff --git a/db4-storage/src/segments/node.rs b/db4-storage/src/segments/node.rs index 2f4d52595e..1385bff972 100644 --- a/db4-storage/src/segments/node.rs +++ b/db4-storage/src/segments/node.rs @@ -206,14 +206,12 @@ impl MemNodeSegment { src_pos: LocalPOS, dst: impl Into, e_id: impl Into, - lsn: u64, ) -> (bool, usize) { let dst = dst.into(); let e_id = e_id.into(); let layer_id = e_id.layer(); let layer = self.get_or_create_layer(layer_id); let est_size = layer.est_size(); - layer.set_lsn(lsn); let add_out = layer.reserve_local_row(src_pos); let new_entry = add_out.is_new(); @@ -235,7 +233,6 @@ impl MemNodeSegment { dst_pos: impl Into, src: impl Into, e_id: impl Into, - lsn: u64, ) -> (bool, usize) { let src = src.into(); let e_id = e_id.into(); @@ -244,7 +241,6 @@ impl MemNodeSegment { let layer = self.get_or_create_layer(layer_id); let est_size = layer.est_size(); - layer.set_lsn(lsn); let add_in = layer.reserve_local_row(dst_pos); let new_entry = add_in.is_new(); @@ -275,12 +271,10 @@ impl MemNodeSegment { t: T, node_pos: LocalPOS, e_id: ELID, - lsn: u64, ) -> usize { let layer_id = e_id.layer(); let (est_size, row) = { let segment_container = self.get_or_create_layer(layer_id); //&mut self.layers[e_id.layer()]; - segment_container.set_lsn(lsn); let est_size = segment_container.est_size(); let row = segment_container.reserve_local_row(node_pos).inner().row(); (est_size, row) @@ -592,7 +586,7 @@ mod test { let est_size1 = segment.est_size(); assert_eq!(est_size1, 0); - writer.add_outbound_edge(Some(1), LocalPOS(1), VID(3), EID(7).with_layer(0), 0); + writer.add_outbound_edge(Some(1), LocalPOS(1), VID(3), EID(7).with_layer(0)); let est_size2 = segment.est_size(); assert!( @@ -600,7 +594,7 @@ mod test { "Estimated size should be greater than 0 after adding an edge" ); - writer.add_inbound_edge(Some(1), LocalPOS(2), VID(4), EID(8).with_layer(0), 0); + writer.add_inbound_edge(Some(1), LocalPOS(2), VID(4), EID(8).with_layer(0)); let est_size3 = segment.est_size(); assert!( @@ -610,7 +604,7 @@ mod test { // no change when adding the same edge again - writer.add_outbound_edge::(None, LocalPOS(1), VID(3), EID(7).with_layer(0), 0); + writer.add_outbound_edge::(None, LocalPOS(1), VID(3), EID(7).with_layer(0)); let est_size4 = segment.est_size(); assert_eq!( est_size4, est_size3, @@ -625,7 +619,7 @@ mod test { .unwrap() .inner(); - writer.update_c_props(LocalPOS(1), 0, [(prop_id, Prop::U64(73))], 0); + writer.update_c_props(LocalPOS(1), 0, [(prop_id, Prop::U64(73))]); let est_size5 = segment.est_size(); assert!( @@ -633,7 +627,7 @@ mod test { "Estimated size should increase after adding constant properties" ); - writer.update_timestamp(17, LocalPOS(1), ELID::new(EID(0), 0), 0); + writer.update_timestamp(17, LocalPOS(1), ELID::new(EID(0), 0)); let est_size6 = segment.est_size(); assert!( @@ -648,7 +642,7 @@ mod test { .unwrap() .inner(); - writer.add_props(42, LocalPOS(1), 0, [(prop_id, Prop::F64(4.13))], 0); + writer.add_props(42, LocalPOS(1), 0, [(prop_id, Prop::F64(4.13))]); let est_size7 = segment.est_size(); assert!( @@ -656,7 +650,7 @@ mod test { "Estimated size should increase after adding temporal properties" ); - writer.add_props(72, LocalPOS(1), 0, [(prop_id, Prop::F64(5.41))], 0); + writer.add_props(72, LocalPOS(1), 0, [(prop_id, Prop::F64(5.41))]); let est_size8 = segment.est_size(); assert!( est_size8 > est_size7, diff --git a/raphtory-storage/src/mutation/addition_ops.rs b/raphtory-storage/src/mutation/addition_ops.rs index 808f9ce04d..3264ed18e4 100644 --- a/raphtory-storage/src/mutation/addition_ops.rs +++ b/raphtory-storage/src/mutation/addition_ops.rs @@ -99,7 +99,6 @@ pub trait EdgeWriteLock: Send + Sync { &mut self, src: impl Into, dst: impl Into, - lsn: u64, ) -> MaybeNew; /// add edge update @@ -109,7 +108,6 @@ pub trait EdgeWriteLock: Send + Sync { src: impl Into, dst: impl Into, eid: MaybeNew, - lsn: u64, props: impl IntoIterator, ) -> MaybeNew; @@ -118,7 +116,6 @@ pub trait EdgeWriteLock: Send + Sync { t: TimeIndexEntry, src: impl Into, dst: impl Into, - lsn: u64, layer: usize, ) -> MaybeNew; diff --git a/raphtory-storage/src/mutation/addition_ops_ext.rs b/raphtory-storage/src/mutation/addition_ops_ext.rs index 0141fc6a53..c04ee79a31 100644 --- a/raphtory-storage/src/mutation/addition_ops_ext.rs +++ b/raphtory-storage/src/mutation/addition_ops_ext.rs @@ -42,9 +42,8 @@ impl<'a, EXT: PersistentStrategy, ES = ES>> EdgeWriteLock for &mut self, src: impl Into, dst: impl Into, - lsn: u64, ) -> MaybeNew { - self.static_session.add_static_edge(src, dst, lsn) + self.static_session.add_static_edge(src, dst) } fn internal_add_edge( @@ -53,11 +52,10 @@ impl<'a, EXT: PersistentStrategy, ES = ES>> EdgeWriteLock for src: impl Into, dst: impl Into, eid: MaybeNew, - lsn: u64, props: impl IntoIterator, ) -> MaybeNew { self.static_session - .add_edge_into_layer(t, src, dst, eid, lsn, props); + .add_edge_into_layer(t, src, dst, eid, props); eid } @@ -67,18 +65,17 @@ impl<'a, EXT: PersistentStrategy, ES = ES>> EdgeWriteLock for t: TimeIndexEntry, src: impl Into, dst: impl Into, - lsn: u64, layer: usize, ) -> MaybeNew { let src = src.into(); let dst = dst.into(); let eid = self .static_session - .add_static_edge(src, dst, lsn) + .add_static_edge(src, dst) .map(|eid| eid.with_layer_deletion(layer)); self.static_session - .delete_edge_from_layer(t, src, dst, eid, lsn); + .delete_edge_from_layer(t, src, dst, eid); eid } @@ -90,7 +87,7 @@ impl<'a, EXT: PersistentStrategy, ES = ES>> EdgeWriteLock for self.static_session .node_writers() .get_mut_src() - .update_c_props(pos, 0, [(NODE_ID_IDX, id.into())], 0); + .update_c_props(pos, 0, [(NODE_ID_IDX, id.into())]); }; } @@ -101,7 +98,7 @@ impl<'a, EXT: PersistentStrategy, ES = ES>> EdgeWriteLock for self.static_session .node_writers() .get_mut_dst() - .update_c_props(pos, 0, [(NODE_ID_IDX, id.into())], 0); + .update_c_props(pos, 0, [(NODE_ID_IDX, id.into())]); }; } @@ -261,7 +258,6 @@ impl InternalAdditionOps for TemporalGraph { local_pos, 0, node_info_as_props(id.as_gid_ref().left(), None), - 0, ); MaybeNew::Existing(0) } @@ -277,7 +273,6 @@ impl InternalAdditionOps for TemporalGraph { id.as_gid_ref().left(), Some(node_type_id.inner()).filter(|&id| id != 0), ), - 0, ); node_type_id } @@ -341,7 +336,7 @@ impl InternalAdditionOps for TemporalGraph { ) -> Result<(), Self::Error> { let (segment, node_pos) = self.storage().nodes().resolve_pos(v); let mut node_writer = self.storage().node_writer(segment); - node_writer.add_props(t, node_pos, 0, props, 0); + node_writer.add_props(t, node_pos, 0, props); Ok(()) } diff --git a/raphtory-storage/src/mutation/deletion_ops.rs b/raphtory-storage/src/mutation/deletion_ops.rs index 06b934cc3c..0a7b0a4b12 100644 --- a/raphtory-storage/src/mutation/deletion_ops.rs +++ b/raphtory-storage/src/mutation/deletion_ops.rs @@ -36,8 +36,9 @@ impl InternalDeletionOps for db4_graph::TemporalGraph { layer: usize, ) -> Result, Self::Error> { let mut session = self.storage().write_session(src, dst, None); - let edge = session.add_static_edge(src, dst, 0); - session.delete_edge_from_layer(t, src, dst, edge.map(|eid| eid.with_layer(layer)), 0); + session.set_lsn(0); + let edge = session.add_static_edge(src, dst); + session.delete_edge_from_layer(t, src, dst, edge.map(|eid| eid.with_layer(layer))); Ok(edge) } @@ -52,7 +53,7 @@ impl InternalDeletionOps for db4_graph::TemporalGraph { let (src, dst) = writer.get_edge(0, edge_pos).unwrap_or_else(|| { panic!("Internal Error: Edge {eid:?} not found in storage"); }); - writer.delete_edge(t, edge_pos, src, dst, layer, 0); + writer.delete_edge(t, edge_pos, src, dst, layer); Ok(()) } } diff --git a/raphtory-storage/src/mutation/property_addition_ops.rs b/raphtory-storage/src/mutation/property_addition_ops.rs index a10b04eac8..5c16f51bc0 100644 --- a/raphtory-storage/src/mutation/property_addition_ops.rs +++ b/raphtory-storage/src/mutation/property_addition_ops.rs @@ -83,7 +83,7 @@ impl InternalPropertyAdditionOps for db4_graph::TemporalGraph { let (segment_id, node_pos) = self.storage().nodes().resolve_pos(vid); let mut writer = self.storage().nodes().writer(segment_id); writer.check_metadata(node_pos, 0, &props)?; - writer.update_c_props(node_pos, 0, props, 0); + writer.update_c_props(node_pos, 0, props); Ok(writer) } @@ -94,7 +94,7 @@ impl InternalPropertyAdditionOps for db4_graph::TemporalGraph { ) -> Result, Self::Error> { let (segment_id, node_pos) = self.storage().nodes().resolve_pos(vid); let mut writer = self.storage().nodes().writer(segment_id); - writer.update_c_props(node_pos, 0, props, 0); + writer.update_c_props(node_pos, 0, props); Ok(writer) } diff --git a/raphtory/src/db/api/mutation/addition_ops.rs b/raphtory/src/db/api/mutation/addition_ops.rs index b10b0435f1..2dce0acd74 100644 --- a/raphtory/src/db/api/mutation/addition_ops.rs +++ b/raphtory/src/db/api/mutation/addition_ops.rs @@ -295,7 +295,7 @@ impl> + StaticGraphViewOps + Dura // NOTE: We log edge id after it is inserted into the edge segment. // This is fine as long as we hold onto the edge segment lock through add_edge_op // for the entire operation. - let edge_id = add_edge_op.internal_add_static_edge(src_id, dst_id, 0); + let edge_id = add_edge_op.internal_add_static_edge(src_id, dst_id); // All names, ids and values have been generated for this operation. // Create a wal entry to mark it as durable. @@ -325,7 +325,6 @@ impl> + StaticGraphViewOps + Dura src_id, dst_id, edge_id.map(|eid| eid.with_layer(layer_id)), - 0, props, ); diff --git a/raphtory/src/db/api/mutation/deletion_ops.rs b/raphtory/src/db/api/mutation/deletion_ops.rs index e25b1ca190..8157040213 100644 --- a/raphtory/src/db/api/mutation/deletion_ops.rs +++ b/raphtory/src/db/api/mutation/deletion_ops.rs @@ -51,7 +51,7 @@ pub trait DeletionOps: .atomic_add_edge(src_id, dst_id, None, layer_id) .map_err(into_graph_err)?; - let edge_id = add_edge_op.internal_delete_edge(ti, src_id, dst_id, 0, layer_id); + let edge_id = add_edge_op.internal_delete_edge(ti, src_id, dst_id, layer_id); add_edge_op.store_src_node_info(src_id, src.as_node_ref().as_gid_ref().left()); add_edge_op.store_dst_node_info(dst_id, dst.as_node_ref().as_gid_ref().left()); diff --git a/raphtory/src/db/api/storage/storage.rs b/raphtory/src/db/api/storage/storage.rs index d58db03725..72c31d5d80 100644 --- a/raphtory/src/db/api/storage/storage.rs +++ b/raphtory/src/db/api/storage/storage.rs @@ -302,9 +302,8 @@ impl EdgeWriteLock for AtomicAddEdgeSession<'_> { &mut self, src: impl Into, dst: impl Into, - lsn: u64, ) -> MaybeNew { - self.session.internal_add_static_edge(src, dst, lsn) + self.session.internal_add_static_edge(src, dst) } fn internal_add_edge( @@ -313,11 +312,10 @@ impl EdgeWriteLock for AtomicAddEdgeSession<'_> { src: impl Into, dst: impl Into, e_id: MaybeNew, - lsn: u64, props: impl IntoIterator, ) -> MaybeNew { self.session - .internal_add_edge(t, src, dst, e_id, lsn, props) + .internal_add_edge(t, src, dst, e_id, props) } fn internal_delete_edge( @@ -325,10 +323,9 @@ impl EdgeWriteLock for AtomicAddEdgeSession<'_> { t: TimeIndexEntry, src: impl Into, dst: impl Into, - lsn: u64, layer: usize, ) -> MaybeNew { - self.session.internal_delete_edge(t, src, dst, lsn, layer) + self.session.internal_delete_edge(t, src, dst, layer) } fn store_src_node_info(&mut self, id: impl Into, node_id: Option) { diff --git a/raphtory/src/db/api/view/graph.rs b/raphtory/src/db/api/view/graph.rs index 3d519ad20f..54a4ce452a 100644 --- a/raphtory/src/db/api/view/graph.rs +++ b/raphtory/src/db/api/view/graph.rs @@ -351,17 +351,16 @@ impl<'graph, G: GraphView + 'graph> GraphViewOps<'graph> for G { 0, gid.as_ref(), new_type_id, - 0, ); } else { - writer.store_node_id(node_pos, 0, gid.as_ref(), 0); + writer.store_node_id(node_pos, 0, gid.as_ref()); } graph_storage .write_session()? .set_node(gid.as_ref(), new_id)?; for (t, row) in node.rows() { - writer.add_props(t, node_pos, 0, row, 0); + writer.add_props(t, node_pos, 0, row); } writer.update_c_props( @@ -369,7 +368,6 @@ impl<'graph, G: GraphView + 'graph> GraphViewOps<'graph> for G { 0, node.metadata_ids() .filter_map(|id| node.get_metadata(id).map(|prop| (id, prop))), - 0, ); } } @@ -390,13 +388,13 @@ impl<'graph, G: GraphView + 'graph> GraphViewOps<'graph> for G { if let Some(edge_pos) = shard.resolve_pos(eid) { let mut writer = shard.writer(); // make the edge for the first time - writer.add_static_edge(Some(edge_pos), src, dst, 0, Some(false)); + writer.add_static_edge(Some(edge_pos), src, dst, Some(false)); for edge in edge.explode_layers() { let layer = layer_map[edge.edge.layer().unwrap()]; for edge in edge.explode() { let t = edge.edge.time().unwrap(); - writer.add_edge(t, edge_pos, src, dst, [], layer, 0); + writer.add_edge(t, edge_pos, src, dst, [], layer); } //TODO: move this in edge.row() for (t, t_props) in edge @@ -416,7 +414,7 @@ impl<'graph, G: GraphView + 'graph> GraphViewOps<'graph> for G { let props = t_props .map(|(_, prop_id, prop)| (prop_id, prop)) .collect::>(); - writer.add_edge(t, edge_pos, src, dst, props, layer, 0); + writer.add_edge(t, edge_pos, src, dst, props, layer); } writer.update_c_props( edge_pos, @@ -437,7 +435,7 @@ impl<'graph, G: GraphView + 'graph> GraphViewOps<'graph> for G { self.layer_ids(), ) { let layer = layer_map[layer]; - writer.delete_edge(t, edge_pos, src, dst, layer, 0); + writer.delete_edge(t, edge_pos, src, dst, layer); } } } @@ -454,12 +452,12 @@ impl<'graph, G: GraphView + 'graph> GraphViewOps<'graph> for G { if let Some(node_pos) = maybe_src_pos { let mut writer = shard.writer(); - writer.add_static_outbound_edge(node_pos, dst_id, eid, 0); + writer.add_static_outbound_edge(node_pos, dst_id, eid); } if let Some(node_pos) = maybe_dst_pos { let mut writer = shard.writer(); - writer.add_static_inbound_edge(node_pos, src_id, eid, 0); + writer.add_static_inbound_edge(node_pos, src_id, eid); } for e in edge.explode_layers() { @@ -471,7 +469,6 @@ impl<'graph, G: GraphView + 'graph> GraphViewOps<'graph> for G { node_pos, dst_id, eid.with_layer(layer), - 0, ); } if let Some(node_pos) = maybe_dst_pos { @@ -481,7 +478,6 @@ impl<'graph, G: GraphView + 'graph> GraphViewOps<'graph> for G { node_pos, src_id, eid.with_layer(layer), - 0, ); } } @@ -492,14 +488,14 @@ impl<'graph, G: GraphView + 'graph> GraphViewOps<'graph> for G { let t = e.time_and_index().expect("exploded edge should have time"); let l = layer_map[e.edge.layer().unwrap()]; - writer.update_timestamp(t, node_pos, eid.with_layer(l), 0); + writer.update_timestamp(t, node_pos, eid.with_layer(l)); } if let Some(node_pos) = maybe_dst_pos { let mut writer = shard.writer(); let t = e.time_and_index().expect("exploded edge should have time"); let l = layer_map[e.edge.layer().unwrap()]; - writer.update_timestamp(t, node_pos, eid.with_layer(l), 0); + writer.update_timestamp(t, node_pos, eid.with_layer(l)); } } @@ -513,11 +509,11 @@ impl<'graph, G: GraphView + 'graph> GraphViewOps<'graph> for G { let layer = layer_map[layer]; if let Some(node_pos) = maybe_src_pos { let mut writer = shard.writer(); - writer.update_timestamp(t, node_pos, eid.with_layer_deletion(layer), 0); + writer.update_timestamp(t, node_pos, eid.with_layer_deletion(layer)); } if let Some(node_pos) = maybe_dst_pos { let mut writer = shard.writer(); - writer.update_timestamp(t, node_pos, eid.with_layer_deletion(layer), 0); + writer.update_timestamp(t, node_pos, eid.with_layer_deletion(layer)); } } } diff --git a/raphtory/src/db/graph/edge.rs b/raphtory/src/db/graph/edge.rs index c11781c251..1fa9de0f73 100644 --- a/raphtory/src/db/graph/edge.rs +++ b/raphtory/src/db/graph/edge.rs @@ -446,7 +446,6 @@ impl EdgeView { src, dst, MaybeNew::New(e_id.with_layer(layer_id)), - 0, props, ); diff --git a/raphtory/src/io/arrow/df_loaders.rs b/raphtory/src/io/arrow/df_loaders.rs index 2f8e5e0594..bcdb01334a 100644 --- a/raphtory/src/io/arrow/df_loaders.rs +++ b/raphtory/src/io/arrow/df_loaders.rs @@ -204,19 +204,18 @@ pub fn load_nodes_from_df< let mut writer = shard.writer(); let t = TimeIndexEntry(time, secondary_index); let layer_id = STATIC_GRAPH_LAYER_ID; - let lsn = 0; update_time(t); writer - .store_node_id_and_node_type(mut_node, layer_id, gid, *node_type, lsn); + .store_node_id_and_node_type(mut_node, layer_id, gid, *node_type); let t_props = prop_cols.iter_row(row); let c_props = metadata_cols .iter_row(row) .chain(shared_metadata.iter().cloned()); - writer.add_props(t, mut_node, layer_id, t_props, lsn); - writer.update_c_props(mut_node, layer_id, c_props, lsn); + writer.add_props(t, mut_node, layer_id, t_props); + writer.update_c_props(mut_node, layer_id, c_props); }; } @@ -469,7 +468,7 @@ pub fn load_edges_from_df Date: Thu, 4 Dec 2025 18:19:10 -0500 Subject: [PATCH 10/19] Implement basic add_edge replay --- db4-storage/src/wal/mod.rs | 2 +- .../src/mutation/property_addition_ops.rs | 7 +++ raphtory/src/db/replay/mod.rs | 63 ++++++++++++++----- 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/db4-storage/src/wal/mod.rs b/db4-storage/src/wal/mod.rs index 36772e704c..d84e506972 100644 --- a/db4-storage/src/wal/mod.rs +++ b/db4-storage/src/wal/mod.rs @@ -90,6 +90,6 @@ pub trait GraphReplayer { eid: EID, layer_name: Option<&str>, layer_id: usize, - props: &[MaybeNew<(PN, usize, Prop)>], + props: Vec>, ) -> Result<(), StorageError>; } diff --git a/raphtory-storage/src/mutation/property_addition_ops.rs b/raphtory-storage/src/mutation/property_addition_ops.rs index 5c16f51bc0..741be2e6b1 100644 --- a/raphtory-storage/src/mutation/property_addition_ops.rs +++ b/raphtory-storage/src/mutation/property_addition_ops.rs @@ -13,29 +13,36 @@ use storage::Extension; pub trait InternalPropertyAdditionOps { type Error: From; + fn internal_add_properties( &self, t: TimeIndexEntry, props: &[(usize, Prop)], ) -> Result<(), Self::Error>; + fn internal_add_metadata(&self, props: &[(usize, Prop)]) -> Result<(), Self::Error>; + fn internal_update_metadata(&self, props: &[(usize, Prop)]) -> Result<(), Self::Error>; + fn internal_add_node_metadata( &self, vid: VID, props: Vec<(usize, Prop)>, ) -> Result, Self::Error>; + fn internal_update_node_metadata( &self, vid: VID, props: Vec<(usize, Prop)>, ) -> Result, Self::Error>; + fn internal_add_edge_metadata( &self, eid: EID, layer: usize, props: Vec<(usize, Prop)>, ) -> Result, Self::Error>; + fn internal_update_edge_metadata( &self, eid: EID, diff --git a/raphtory/src/db/replay/mod.rs b/raphtory/src/db/replay/mod.rs index b733c1b46a..7e9f501c5d 100644 --- a/raphtory/src/db/replay/mod.rs +++ b/raphtory/src/db/replay/mod.rs @@ -1,26 +1,31 @@ -use db4_graph::TemporalGraph; +use crate::db::api::{ + storage::{graph, storage::Storage}, + view::internal::{Base, InternalStorageOps}, + }; use raphtory_api::core::{ entities::{properties::prop::Prop, EID, GID, VID}, storage::{dict_mapper::MaybeNew, timeindex::TimeIndexEntry}, }; +use raphtory_core::entities::GidRef; +use raphtory_storage::{core_ops::CoreGraphOps, mutation::addition_ops::{EdgeWriteLock, InternalAdditionOps}}; use storage::{ api::edges::EdgeSegmentOps, error::StorageError, wal::{GraphReplayer, TransactionID, LSN}, - Extension, }; +use storage::resolver::GIDResolverOps; -/// Wrapper struct for implementing `GraphReplayer` for a `TemporalGraph`. +/// Wrapper struct for implementing `GraphReplayer` for a `Storage`. /// This is needed to workaround Rust's orphan rule since both `GraphReplayer` -/// and `TemporalGraph` are foreign to this crate. +/// and `Storage` are foreign to this crate. #[derive(Debug)] pub struct ReplayGraph { - graph: TemporalGraph, + storage: Storage, } impl ReplayGraph { - pub fn new(graph: TemporalGraph) -> Self { - Self { graph } + pub fn new(graph: Storage) -> Self { + Self { storage: graph } } } @@ -37,18 +42,46 @@ impl GraphReplayer for ReplayGraph { eid: EID, layer_name: Option<&str>, layer_id: usize, - props: &[MaybeNew<(PN, usize, Prop)>], + props_with_status: Vec>, ) -> Result<(), StorageError> { - let edge_segment = self.graph.storage().edges().get_edge_segment(eid); + // TODO: Check max lsn on disk to see if this record should be replayed. - match edge_segment { - Some(edge_segment) => { - edge_segment.head().lsn(); - } - _ => {} + let storage = self.storage.get_storage() + .ok_or_else(|| StorageError::GenericFailure("Storage not available during replay".to_string()))?; + + let temporal_graph = storage.core_graph().mutable().unwrap(); + + // 1. Insert prop ids into edge meta. + // No need to validate props again since they are already validated before + // being logged to the WAL. + let edge_meta = temporal_graph.edge_meta(); + let mut prop_ids = Vec::new(); + + for prop in props_with_status.into_iter() { + let (prop_name, prop_id, prop_value) = prop.inner(); + let prop_mapper = edge_meta.temporal_prop_mapper(); + + prop_mapper.set_id_and_dtype(prop_name.as_ref(), prop_id, prop_value.dtype()); + prop_ids.push((prop_id, prop_value)); } - // TODO: Check max lsn on disk to see if replay is needed. + // 2. Insert node ids into resolver. + temporal_graph.logical_to_physical.set(GidRef::from(&src_name), src_id)?; + temporal_graph.logical_to_physical.set(GidRef::from(&dst_name), dst_id)?; + + // 3. Insert layer id into the layer meta of both edge and node. + let node_meta = temporal_graph.node_meta(); + + edge_meta.layer_meta().set_id(layer_name.unwrap_or("_default"), layer_id); + node_meta.layer_meta().set_id(layer_name.unwrap_or("_default"), layer_id); + + // 4. Grab src, dst and edge segment locks and add the edge. + let mut add_edge_op = temporal_graph.atomic_add_edge(src_id, dst_id, Some(eid), layer_id).unwrap(); + + let edge_id = add_edge_op.internal_add_static_edge(src_id, dst_id); + let edge_id_with_layer = edge_id.map(|eid| eid.with_layer(layer_id)); + + add_edge_op.internal_add_edge(t, src_id, dst_id, edge_id_with_layer, prop_ids); Ok(()) } From fa11c7ad55de1d7c3a63759ce65f9d556c3cbe69 Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Fri, 5 Dec 2025 18:10:58 -0500 Subject: [PATCH 11/19] Simplify GraphWal --- db4-storage/src/wal/entry.rs | 9 +++------ db4-storage/src/wal/mod.rs | 12 ++++++------ raphtory/src/db/api/mutation/addition_ops.rs | 11 ++++++++++- raphtory/src/db/replay/mod.rs | 17 ++++++++--------- 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/db4-storage/src/wal/entry.rs b/db4-storage/src/wal/entry.rs index d6cd68ebb4..def51d8bf4 100644 --- a/db4-storage/src/wal/entry.rs +++ b/db4-storage/src/wal/entry.rs @@ -1,9 +1,6 @@ use std::path::Path; -use raphtory_api::core::{ - entities::properties::prop::Prop, - storage::dict_mapper::MaybeNew, -}; +use raphtory_api::core::entities::properties::prop::Prop; use raphtory_core::{ entities::{EID, GID, VID}, storage::timeindex::TimeIndexEntry, @@ -17,7 +14,7 @@ use crate::{ impl GraphWal for NoWal { type ReplayEntry = (); - fn log_add_edge>( + fn log_add_edge( &self, _transaction_id: TransactionID, _t: TimeIndexEntry, @@ -28,7 +25,7 @@ impl GraphWal for NoWal { _eid: EID, _layer_name: Option<&str>, _layer_id: usize, - _props: &[MaybeNew<(PN, usize, Prop)>], + _props: Vec<(&str, usize, Prop)>, ) -> Result { Ok(0) } diff --git a/db4-storage/src/wal/mod.rs b/db4-storage/src/wal/mod.rs index d84e506972..37c86f2425 100644 --- a/db4-storage/src/wal/mod.rs +++ b/db4-storage/src/wal/mod.rs @@ -1,5 +1,5 @@ use crate::error::StorageError; -use raphtory_api::core::{entities::properties::prop::Prop, storage::dict_mapper::MaybeNew}; +use raphtory_api::core::entities::properties::prop::Prop; use raphtory_core::{ entities::{EID, GID, VID}, storage::timeindex::TimeIndexEntry, @@ -46,7 +46,7 @@ pub trait GraphWal { /// ReplayEntry represents the type of the wal entry returned during replay. type ReplayEntry; - fn log_add_edge>( + fn log_add_edge( &self, transaction_id: TransactionID, t: TimeIndexEntry, @@ -57,7 +57,7 @@ pub trait GraphWal { eid: EID, layer_name: Option<&str>, layer_id: usize, - props: &[MaybeNew<(PN, usize, Prop)>], + props: Vec<(&str, usize, Prop)>, ) -> Result; /// Logs a checkpoint record, indicating that all Wal operations upto and including @@ -78,7 +78,7 @@ pub trait GraphWal { /// Trait for defining callbacks for replaying from wal. pub trait GraphReplayer { - fn replay_add_edge>( + fn replay_add_edge( &self, lsn: LSN, transaction_id: TransactionID, @@ -88,8 +88,8 @@ pub trait GraphReplayer { dst_name: GID, dst_id: VID, eid: EID, - layer_name: Option<&str>, + layer_name: Option, layer_id: usize, - props: Vec>, + props: Vec<(String, usize, Prop)>, ) -> Result<(), StorageError>; } diff --git a/raphtory/src/db/api/mutation/addition_ops.rs b/raphtory/src/db/api/mutation/addition_ops.rs index 2dce0acd74..769eb14e32 100644 --- a/raphtory/src/db/api/mutation/addition_ops.rs +++ b/raphtory/src/db/api/mutation/addition_ops.rs @@ -299,6 +299,15 @@ impl> + StaticGraphViewOps + Dura // All names, ids and values have been generated for this operation. // Create a wal entry to mark it as durable. + + let props_for_wal = props_with_status + .iter() + .map(|maybe_new| { + let (prop_name, prop_id, prop) = maybe_new.as_ref().inner(); + (prop_name.as_ref(), *prop_id, prop.clone()) + }) + .collect::>(); + let lsn = self.wal().log_add_edge( transaction_id, ti, @@ -309,7 +318,7 @@ impl> + StaticGraphViewOps + Dura edge_id.inner(), layer, layer_id, - &props_with_status, + props_for_wal, ).unwrap(); let props = props_with_status diff --git a/raphtory/src/db/replay/mod.rs b/raphtory/src/db/replay/mod.rs index 7e9f501c5d..4d3d8a6601 100644 --- a/raphtory/src/db/replay/mod.rs +++ b/raphtory/src/db/replay/mod.rs @@ -4,7 +4,7 @@ use crate::db::api::{ }; use raphtory_api::core::{ entities::{properties::prop::Prop, EID, GID, VID}, - storage::{dict_mapper::MaybeNew, timeindex::TimeIndexEntry}, + storage::timeindex::TimeIndexEntry, }; use raphtory_core::entities::GidRef; use raphtory_storage::{core_ops::CoreGraphOps, mutation::addition_ops::{EdgeWriteLock, InternalAdditionOps}}; @@ -30,7 +30,7 @@ impl ReplayGraph { } impl GraphReplayer for ReplayGraph { - fn replay_add_edge>( + fn replay_add_edge( &self, lsn: LSN, transaction_id: TransactionID, @@ -40,9 +40,9 @@ impl GraphReplayer for ReplayGraph { dst_name: GID, dst_id: VID, eid: EID, - layer_name: Option<&str>, + layer_name: Option, layer_id: usize, - props_with_status: Vec>, + props: Vec<(String, usize, Prop)>, ) -> Result<(), StorageError> { // TODO: Check max lsn on disk to see if this record should be replayed. @@ -57,11 +57,10 @@ impl GraphReplayer for ReplayGraph { let edge_meta = temporal_graph.edge_meta(); let mut prop_ids = Vec::new(); - for prop in props_with_status.into_iter() { - let (prop_name, prop_id, prop_value) = prop.inner(); + for (prop_name, prop_id, prop_value) in props.into_iter() { let prop_mapper = edge_meta.temporal_prop_mapper(); - prop_mapper.set_id_and_dtype(prop_name.as_ref(), prop_id, prop_value.dtype()); + prop_mapper.set_id_and_dtype(prop_name, prop_id, prop_value.dtype()); prop_ids.push((prop_id, prop_value)); } @@ -72,8 +71,8 @@ impl GraphReplayer for ReplayGraph { // 3. Insert layer id into the layer meta of both edge and node. let node_meta = temporal_graph.node_meta(); - edge_meta.layer_meta().set_id(layer_name.unwrap_or("_default"), layer_id); - node_meta.layer_meta().set_id(layer_name.unwrap_or("_default"), layer_id); + edge_meta.layer_meta().set_id(layer_name.as_deref().unwrap_or("_default"), layer_id); + node_meta.layer_meta().set_id(layer_name.as_deref().unwrap_or("_default"), layer_id); // 4. Grab src, dst and edge segment locks and add the edge. let mut add_edge_op = temporal_graph.atomic_add_edge(src_id, dst_id, Some(eid), layer_id).unwrap(); From aa73139a7479d19e4d7a44762290cac5ac4dc827 Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Wed, 10 Dec 2025 13:51:46 +0400 Subject: [PATCH 12/19] Remove wrapper for graph replay --- db4-storage/src/wal/entry.rs | 4 ++-- db4-storage/src/wal/mod.rs | 4 ++-- raphtory/src/db/replay/mod.rs | 20 +++----------------- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/db4-storage/src/wal/entry.rs b/db4-storage/src/wal/entry.rs index def51d8bf4..7b0b0e6745 100644 --- a/db4-storage/src/wal/entry.rs +++ b/db4-storage/src/wal/entry.rs @@ -8,7 +8,7 @@ use raphtory_core::{ use crate::{ error::StorageError, - wal::{GraphReplayer, GraphWal, LSN, TransactionID, no_wal::NoWal}, + wal::{GraphReplay, GraphWal, LSN, TransactionID, no_wal::NoWal}, }; impl GraphWal for NoWal { @@ -40,7 +40,7 @@ impl GraphWal for NoWal { std::iter::once(Ok((0, ()))) } - fn replay_to_graph( + fn replay_to_graph( _dir: impl AsRef, _graph: &mut G, ) -> Result<(), StorageError> { diff --git a/db4-storage/src/wal/mod.rs b/db4-storage/src/wal/mod.rs index 37c86f2425..5677961089 100644 --- a/db4-storage/src/wal/mod.rs +++ b/db4-storage/src/wal/mod.rs @@ -70,14 +70,14 @@ pub trait GraphWal { ) -> impl Iterator>; /// Replays and applies all the wal entries in the given directory to the given graph. - fn replay_to_graph( + fn replay_to_graph( dir: impl AsRef, graph: &mut G, ) -> Result<(), StorageError>; } /// Trait for defining callbacks for replaying from wal. -pub trait GraphReplayer { +pub trait GraphReplay { fn replay_add_edge( &self, lsn: LSN, diff --git a/raphtory/src/db/replay/mod.rs b/raphtory/src/db/replay/mod.rs index 4d3d8a6601..1ad99a8ca7 100644 --- a/raphtory/src/db/replay/mod.rs +++ b/raphtory/src/db/replay/mod.rs @@ -11,25 +11,11 @@ use raphtory_storage::{core_ops::CoreGraphOps, mutation::addition_ops::{EdgeWrit use storage::{ api::edges::EdgeSegmentOps, error::StorageError, - wal::{GraphReplayer, TransactionID, LSN}, + wal::{GraphReplay, TransactionID, LSN}, }; use storage::resolver::GIDResolverOps; -/// Wrapper struct for implementing `GraphReplayer` for a `Storage`. -/// This is needed to workaround Rust's orphan rule since both `GraphReplayer` -/// and `Storage` are foreign to this crate. -#[derive(Debug)] -pub struct ReplayGraph { - storage: Storage, -} - -impl ReplayGraph { - pub fn new(graph: Storage) -> Self { - Self { storage: graph } - } -} - -impl GraphReplayer for ReplayGraph { +impl GraphReplay for Storage { fn replay_add_edge( &self, lsn: LSN, @@ -46,7 +32,7 @@ impl GraphReplayer for ReplayGraph { ) -> Result<(), StorageError> { // TODO: Check max lsn on disk to see if this record should be replayed. - let storage = self.storage.get_storage() + let storage = self.get_storage() .ok_or_else(|| StorageError::GenericFailure("Storage not available during replay".to_string()))?; let temporal_graph = storage.core_graph().mutable().unwrap(); From 6944a6fc0c357b061181df46bc6dc3e1770854f3 Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Wed, 10 Dec 2025 14:17:34 +0400 Subject: [PATCH 13/19] Fix leftover merge issues --- db4-graph/src/lib.rs | 3 ++- db4-storage/src/segments/edge/segment.rs | 3 ++- db4-storage/src/segments/graph_prop/segment.rs | 16 ++++++++++++---- db4-storage/src/segments/node/segment.rs | 18 ++++++------------ .../src/mutation/addition_ops_ext.rs | 3 ++- 5 files changed, 24 insertions(+), 19 deletions(-) diff --git a/db4-graph/src/lib.rs b/db4-graph/src/lib.rs index 4dd2352b4e..babf9343a1 100644 --- a/db4-graph/src/lib.rs +++ b/db4-graph/src/lib.rs @@ -23,7 +23,8 @@ use storage::{ }, persist::strategy::{Config, PersistentStrategy}, resolver::GIDResolverOps, - Extension, GIDResolver, Layer, ReadLockedLayer, transaction::TransactionManager, WalImpl, ES, NS, + Extension, GIDResolver, Layer, ReadLockedLayer, transaction::TransactionManager, + WalImpl, ES, NS, GS, wal::Wal, }; use tempfile::TempDir; diff --git a/db4-storage/src/segments/edge/segment.rs b/db4-storage/src/segments/edge/segment.rs index 9ab1814c35..f3dbf64a4d 100644 --- a/db4-storage/src/segments/edge/segment.rs +++ b/db4-storage/src/segments/edge/segment.rs @@ -10,6 +10,7 @@ use crate::{ edge::entry::{MemEdgeEntry, MemEdgeRef}, }, utils::Iter4, + wal::LSN, }; use arrow_array::{ArrayRef, BooleanArray}; use parking_lot::lock_api::ArcRwLockReadGuard; @@ -53,7 +54,7 @@ impl HasRow for EdgeEntry { pub struct MemEdgeSegment { layers: Vec>, est_size: usize, - lsn: u64, + lsn: LSN, } impl>> From for MemEdgeSegment { diff --git a/db4-storage/src/segments/graph_prop/segment.rs b/db4-storage/src/segments/graph_prop/segment.rs index 3c17fa7fa9..2636ce8d9a 100644 --- a/db4-storage/src/segments/graph_prop/segment.rs +++ b/db4-storage/src/segments/graph_prop/segment.rs @@ -1,7 +1,5 @@ use crate::{ - LocalPOS, - error::StorageError, - segments::{HasRow, SegmentContainer}, + error::StorageError, segments::{HasRow, SegmentContainer}, wal::LSN, LocalPOS }; use raphtory_api::core::entities::properties::{meta::Meta, prop::Prop}; use raphtory_core::{ @@ -15,6 +13,7 @@ use std::sync::Arc; pub struct MemGraphPropSegment { /// Layers containing graph properties and metadata. layers: Vec>, + lsn: LSN, } /// A unit-like struct for use with `SegmentContainer`. @@ -49,6 +48,7 @@ impl MemGraphPropSegment { Self { layers: vec![SegmentContainer::new(segment_id, max_page_len, meta)], + lsn: 0, } } @@ -83,7 +83,15 @@ impl MemGraphPropSegment { pub fn take(&mut self) -> Self { let layers = self.layers.iter_mut().map(|layer| layer.take()).collect(); - Self { layers } + Self { layers, lsn: self.lsn } + } + + pub fn lsn(&self) -> LSN { + self.lsn + } + + pub fn set_lsn(&mut self, lsn: LSN) { + self.lsn = lsn; } pub fn add_properties( diff --git a/db4-storage/src/segments/node/segment.rs b/db4-storage/src/segments/node/segment.rs index b2836ccb61..c6dabc24f4 100644 --- a/db4-storage/src/segments/node/segment.rs +++ b/db4-storage/src/segments/node/segment.rs @@ -1,13 +1,7 @@ use crate::{ - LocalPOS, - api::nodes::{LockedNSSegment, NodeSegmentOps}, - error::StorageError, - loop_lock_write, - persist::strategy::PersistentStrategy, - segments::{ - HasRow, SegmentContainer, - node::entry::{MemNodeEntry, MemNodeRef}, - }, + api::nodes::{LockedNSSegment, NodeSegmentOps}, error::StorageError, loop_lock_write, persist::strategy::PersistentStrategy, segments::{ + node::entry::{MemNodeEntry, MemNodeRef}, HasRow, SegmentContainer + }, wal::LSN, LocalPOS }; use either::Either; use parking_lot::lock_api::ArcRwLockReadGuard; @@ -36,7 +30,7 @@ pub struct MemNodeSegment { segment_id: usize, max_page_len: u32, layers: Vec>, - lsn: u64, + lsn: LSN, } impl>> From for MemNodeSegment { @@ -143,11 +137,11 @@ impl MemNodeSegment { self.get_adj(n, layer_id).map_or(0, |adj| adj.degree(dir)) } - pub fn lsn(&self) -> u64 { + pub fn lsn(&self) -> LSN { self.lsn } - pub fn set_lsn(&mut self, lsn: u64) { + pub fn set_lsn(&mut self, lsn: LSN) { if lsn > self.lsn { self.lsn = lsn; } diff --git a/raphtory-storage/src/mutation/addition_ops_ext.rs b/raphtory-storage/src/mutation/addition_ops_ext.rs index 13ca4f7e57..f9d1cee68b 100644 --- a/raphtory-storage/src/mutation/addition_ops_ext.rs +++ b/raphtory-storage/src/mutation/addition_ops_ext.rs @@ -24,7 +24,8 @@ use storage::{ persist::strategy::PersistentStrategy, properties::props_meta_writer::PropsMetaWriter, resolver::GIDResolverOps, - Extension, transaction::TransactionManager, WalImpl, ES, NS, + Extension, transaction::TransactionManager, WalImpl, ES, NS, GS, + wal::LSN, }; pub struct WriteS<'a, EXT: PersistentStrategy, ES = ES, GS = GS>> { From 65be87aca1a79dc002b2d08b8e8d8554f31cf1e4 Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Wed, 10 Dec 2025 17:31:36 +0400 Subject: [PATCH 14/19] Change mark_dirty to set_dirty --- db4-storage/src/api/edges.rs | 3 +-- db4-storage/src/api/graph_props.rs | 2 +- db4-storage/src/api/nodes.rs | 2 +- db4-storage/src/pages/edge_store.rs | 2 +- db4-storage/src/pages/graph_prop_page/writer.rs | 4 ++-- db4-storage/src/pages/locked/graph_props.rs | 4 ++-- db4-storage/src/pages/node_store.rs | 2 +- db4-storage/src/segments/edge/segment.rs | 2 +- db4-storage/src/segments/graph_prop/mod.rs | 4 ++-- db4-storage/src/segments/node/segment.rs | 2 +- 10 files changed, 13 insertions(+), 14 deletions(-) diff --git a/db4-storage/src/api/edges.rs b/db4-storage/src/api/edges.rs index 61136444cd..905f6ed64b 100644 --- a/db4-storage/src/api/edges.rs +++ b/db4-storage/src/api/edges.rs @@ -58,8 +58,7 @@ pub trait EdgeSegmentOps: Send + Sync + std::fmt::Debug + 'static { fn try_head_mut(&self) -> Option>; - /// mark segment as dirty without triggering a write - fn mark_dirty(&self); + fn set_dirty(&self, dirty: bool); /// notify that an edge was added (might need to write to disk) fn notify_write( diff --git a/db4-storage/src/api/graph_props.rs b/db4-storage/src/api/graph_props.rs index 768aa8b123..a06ab76acc 100644 --- a/db4-storage/src/api/graph_props.rs +++ b/db4-storage/src/api/graph_props.rs @@ -29,7 +29,7 @@ where fn est_size(&self) -> usize; - fn mark_dirty(&self); + fn set_dirty(&self, dirty: bool); fn notify_write( &self, diff --git a/db4-storage/src/api/nodes.rs b/db4-storage/src/api/nodes.rs index ebea776c8a..9f9b2c2283 100644 --- a/db4-storage/src/api/nodes.rs +++ b/db4-storage/src/api/nodes.rs @@ -94,7 +94,7 @@ pub trait NodeSegmentOps: Send + Sync + std::fmt::Debug + 'static { head_lock: impl DerefMut, ) -> Result<(), StorageError>; - fn mark_dirty(&self); + fn set_dirty(&self, dirty: bool); fn check_node(&self, pos: LocalPOS, layer_id: usize) -> bool; diff --git a/db4-storage/src/pages/edge_store.rs b/db4-storage/src/pages/edge_store.rs index 71ed0d1be3..eaff61c5d2 100644 --- a/db4-storage/src/pages/edge_store.rs +++ b/db4-storage/src/pages/edge_store.rs @@ -144,7 +144,7 @@ impl, EXT: Config> EdgeStorageInner .properties_mut() .set_has_properties() } - segment.mark_dirty(); + segment.set_dirty(true); } empty } diff --git a/db4-storage/src/pages/graph_prop_page/writer.rs b/db4-storage/src/pages/graph_prop_page/writer.rs index f2c89064b3..612a1be9cc 100644 --- a/db4-storage/src/pages/graph_prop_page/writer.rs +++ b/db4-storage/src/pages/graph_prop_page/writer.rs @@ -33,7 +33,7 @@ impl<'a, GS: GraphPropSegmentOps> GraphPropWriter<'a, GS> { let add = self.mem_segment.add_properties(t, props); self.graph_props.increment_est_size(add); - self.graph_props.mark_dirty(); + self.graph_props.set_dirty(true); } pub fn check_metadata(&self, props: &[(usize, Prop)]) -> Result<(), StorageError> { @@ -44,7 +44,7 @@ impl<'a, GS: GraphPropSegmentOps> GraphPropWriter<'a, GS> { let add = self.mem_segment.update_metadata(props); self.graph_props.increment_est_size(add); - self.graph_props.mark_dirty(); + self.graph_props.set_dirty(true); } } diff --git a/db4-storage/src/pages/locked/graph_props.rs b/db4-storage/src/pages/locked/graph_props.rs index 9aa19dbd91..87d41dc222 100644 --- a/db4-storage/src/pages/locked/graph_props.rs +++ b/db4-storage/src/pages/locked/graph_props.rs @@ -29,7 +29,7 @@ impl<'a, GS: GraphPropSegmentOps> LockedGraphPropPage<'a, GS> { let add = self.lock.add_properties(t, props); self.page.increment_est_size(add); - self.page.mark_dirty(); + self.page.set_dirty(true); } /// Add metadata (constant properties) to the graph @@ -42,7 +42,7 @@ impl<'a, GS: GraphPropSegmentOps> LockedGraphPropPage<'a, GS> { let add = self.lock.update_metadata(props); self.page.increment_est_size(add); - self.page.mark_dirty(); + self.page.set_dirty(true); } } diff --git a/db4-storage/src/pages/node_store.rs b/db4-storage/src/pages/node_store.rs index 113112a77a..44fccfbfd2 100644 --- a/db4-storage/src/pages/node_store.rs +++ b/db4-storage/src/pages/node_store.rs @@ -159,7 +159,7 @@ impl, EXT: Config> NodeStorageInner .properties_mut() .set_has_properties() } - segment.mark_dirty(); + segment.set_dirty(true); } empty } diff --git a/db4-storage/src/segments/edge/segment.rs b/db4-storage/src/segments/edge/segment.rs index f3dbf64a4d..eb946ef868 100644 --- a/db4-storage/src/segments/edge/segment.rs +++ b/db4-storage/src/segments/edge/segment.rs @@ -585,7 +585,7 @@ impl>> EdgeSegmentOps for EdgeSegm .map_or(0, |layer| layer.len()) } - fn mark_dirty(&self) {} + fn set_dirty(&self, _dirty: bool) {} } #[cfg(test)] diff --git a/db4-storage/src/segments/graph_prop/mod.rs b/db4-storage/src/segments/graph_prop/mod.rs index 7d20c0624d..d6f98c9038 100644 --- a/db4-storage/src/segments/graph_prop/mod.rs +++ b/db4-storage/src/segments/graph_prop/mod.rs @@ -79,8 +79,8 @@ impl GraphPropSegmentOps for GraphPropSegmentView

{ self.est_size.load(Ordering::Relaxed) } - fn mark_dirty(&self) { - self.is_dirty.store(true, Ordering::Relaxed); + fn set_dirty(&self, dirty: bool) { + self.is_dirty.store(dirty, Ordering::Release); } fn notify_write( diff --git a/db4-storage/src/segments/node/segment.rs b/db4-storage/src/segments/node/segment.rs index c6dabc24f4..832828b990 100644 --- a/db4-storage/src/segments/node/segment.rs +++ b/db4-storage/src/segments/node/segment.rs @@ -480,7 +480,7 @@ impl>> NodeSegmentOps for NodeSegm Ok(()) } - fn mark_dirty(&self) {} + fn set_dirty(&self, _dirty: bool) {} fn check_node(&self, _pos: LocalPOS, _layer_id: usize) -> bool { false From 9a80df792f692a8b346f551464b4d7e5acc53efb Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Thu, 11 Dec 2025 11:13:24 +0400 Subject: [PATCH 15/19] Add replay tests --- db4-storage/src/pages/edge_page/writer.rs | 1 + db4-storage/src/pages/edge_store.rs | 4 ++++ db4-storage/src/pages/graph_prop_store.rs | 4 ++++ db4-storage/src/pages/session.rs | 1 + db4-storage/src/wal/entry.rs | 2 +- db4-storage/src/wal/mod.rs | 2 +- raphtory-storage/src/mutation/addition_ops_ext.rs | 3 +-- raphtory/src/db/replay/mod.rs | 11 +++++++---- 8 files changed, 20 insertions(+), 8 deletions(-) diff --git a/db4-storage/src/pages/edge_page/writer.rs b/db4-storage/src/pages/edge_page/writer.rs index 9d7ff4fdd0..55babb1405 100644 --- a/db4-storage/src/pages/edge_page/writer.rs +++ b/db4-storage/src/pages/edge_page/writer.rs @@ -119,6 +119,7 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen let edge_pos = edge_pos.unwrap_or_else(|| self.new_local_pos(layer_id)); self.writer .insert_static_edge_internal(edge_pos, src, dst, layer_id); + edge_pos } diff --git a/db4-storage/src/pages/edge_store.rs b/db4-storage/src/pages/edge_store.rs index eaff61c5d2..d50f87cc1d 100644 --- a/db4-storage/src/pages/edge_store.rs +++ b/db4-storage/src/pages/edge_store.rs @@ -117,6 +117,10 @@ impl, EXT: Config> EdgeStorageInner &self.layer_counter } + pub fn segments(&self) -> &boxcar::Vec> { + &self.segments + } + pub fn new_with_meta(edges_path: Option, edge_meta: Arc, ext: EXT) -> Self { let free_pages = (0..N).map(RwLock::new).collect::>(); let empty = Self { diff --git a/db4-storage/src/pages/graph_prop_store.rs b/db4-storage/src/pages/graph_prop_store.rs index 6e958182c3..0f1aaff698 100644 --- a/db4-storage/src/pages/graph_prop_store.rs +++ b/db4-storage/src/pages/graph_prop_store.rs @@ -66,6 +66,10 @@ impl, EXT: Config> GraphPropStorageInne self.page.entry() } + pub fn segment(&self) -> &Arc { + &self.page + } + pub fn writer(&self) -> GraphPropWriter<'_, GS> { let head = self.page.head_mut(); let graph_props = &self.page; diff --git a/db4-storage/src/pages/session.rs b/db4-storage/src/pages/session.rs index e43730142e..e99a496455 100644 --- a/db4-storage/src/pages/session.rs +++ b/db4-storage/src/pages/session.rs @@ -193,6 +193,7 @@ impl< if self.edge_writer.is_none() { self.edge_writer = Some(self.graph.edge_writer(e_id)); } + let edge_writer = self.edge_writer.as_mut().unwrap(); let (_, edge_pos) = self.graph.edges().resolve_pos(e_id); diff --git a/db4-storage/src/wal/entry.rs b/db4-storage/src/wal/entry.rs index 7b0b0e6745..dc2680580f 100644 --- a/db4-storage/src/wal/entry.rs +++ b/db4-storage/src/wal/entry.rs @@ -42,7 +42,7 @@ impl GraphWal for NoWal { fn replay_to_graph( _dir: impl AsRef, - _graph: &mut G, + _graph: &G, ) -> Result<(), StorageError> { todo!() } diff --git a/db4-storage/src/wal/mod.rs b/db4-storage/src/wal/mod.rs index 5677961089..72b26c0d41 100644 --- a/db4-storage/src/wal/mod.rs +++ b/db4-storage/src/wal/mod.rs @@ -72,7 +72,7 @@ pub trait GraphWal { /// Replays and applies all the wal entries in the given directory to the given graph. fn replay_to_graph( dir: impl AsRef, - graph: &mut G, + graph: &G, ) -> Result<(), StorageError>; } diff --git a/raphtory-storage/src/mutation/addition_ops_ext.rs b/raphtory-storage/src/mutation/addition_ops_ext.rs index f9d1cee68b..612caef67d 100644 --- a/raphtory-storage/src/mutation/addition_ops_ext.rs +++ b/raphtory-storage/src/mutation/addition_ops_ext.rs @@ -56,8 +56,7 @@ impl<'a, EXT: PersistentStrategy, ES = ES, GS = GS>> Edge eid: MaybeNew, props: impl IntoIterator, ) -> MaybeNew { - self.static_session - .add_edge_into_layer(t, src, dst, eid, props); + self.static_session.add_edge_into_layer(t, src, dst, eid, props); eid } diff --git a/raphtory/src/db/replay/mod.rs b/raphtory/src/db/replay/mod.rs index 1ad99a8ca7..e332a0bfbf 100644 --- a/raphtory/src/db/replay/mod.rs +++ b/raphtory/src/db/replay/mod.rs @@ -32,10 +32,7 @@ impl GraphReplay for Storage { ) -> Result<(), StorageError> { // TODO: Check max lsn on disk to see if this record should be replayed. - let storage = self.get_storage() - .ok_or_else(|| StorageError::GenericFailure("Storage not available during replay".to_string()))?; - - let temporal_graph = storage.core_graph().mutable().unwrap(); + let temporal_graph = self.core_graph().mutable().unwrap(); // 1. Insert prop ids into edge meta. // No need to validate props again since they are already validated before @@ -61,13 +58,19 @@ impl GraphReplay for Storage { node_meta.layer_meta().set_id(layer_name.as_deref().unwrap_or("_default"), layer_id); // 4. Grab src, dst and edge segment locks and add the edge. + println!("Grabbing add_edge_op lock"); let mut add_edge_op = temporal_graph.atomic_add_edge(src_id, dst_id, Some(eid), layer_id).unwrap(); + println!("Added edge to atomic_add_edge"); + let edge_id = add_edge_op.internal_add_static_edge(src_id, dst_id); let edge_id_with_layer = edge_id.map(|eid| eid.with_layer(layer_id)); + println!("Adding edge to internal_add_edge"); add_edge_op.internal_add_edge(t, src_id, dst_id, edge_id_with_layer, prop_ids); + println!("Added edge to internal_add_edge"); + Ok(()) } } From c3f3352e3ef98e8a5813afb63d590701657b3377 Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Thu, 11 Dec 2025 12:57:16 +0400 Subject: [PATCH 16/19] Always set edge_writer in init in WriteSession --- db4-storage/src/pages/mod.rs | 19 ++++++++- db4-storage/src/pages/session.rs | 66 ++++++++++++-------------------- 2 files changed, 42 insertions(+), 43 deletions(-) diff --git a/db4-storage/src/pages/mod.rs b/db4-storage/src/pages/mod.rs index 9c90934e94..d0b6da23e1 100644 --- a/db4-storage/src/pages/mod.rs +++ b/db4-storage/src/pages/mod.rs @@ -354,17 +354,26 @@ impl< let node_writers = if src_chunk < dst_chunk { let src = self.node_writer(src_chunk); let dst = self.node_writer(dst_chunk); + NodeWriters { src, dst: Some(dst) } } else if src_chunk > dst_chunk { let dst = self.node_writer(dst_chunk); let src = self.node_writer(src_chunk); + NodeWriters { src, dst: Some(dst) } } else { let src = self.node_writer(src_chunk); + NodeWriters { src, dst: None } }; - let edge_writer = e_id.map(|e_id| self.edge_writer(e_id)); + let (_, src_pos) = self.nodes.resolve_pos(src); + let existing_eid = node_writers.src.get_out_edge(src_pos, dst, 0); + + let edge_writer = match e_id.or(existing_eid) { + Some(e_id) => self.edge_writer(e_id), + None => self.get_free_writer(), + }; WriteSession::new(node_writers, edge_writer, self) } @@ -398,7 +407,13 @@ impl< NodeWriters { src: writer, dst: None } }; - let edge_writer = e_id.map(|e_id| self.edge_writer(e_id)); + let (_, src_pos) = self.nodes.resolve_pos(src); + let existing_eid = node_writers.src.get_out_edge(src_pos, dst, 0); + + let edge_writer = match e_id.or(existing_eid) { + Some(e_id) => self.edge_writer(e_id), + None => self.get_free_writer(), + }; WriteSession::new(node_writers, edge_writer, self) } diff --git a/db4-storage/src/pages/session.rs b/db4-storage/src/pages/session.rs index e99a496455..df45afc6a1 100644 --- a/db4-storage/src/pages/session.rs +++ b/db4-storage/src/pages/session.rs @@ -23,7 +23,7 @@ pub struct WriteSession< EXT: Config, > { node_writers: NodeWriters<'a, RwLockWriteGuard<'a, MemNodeSegment>, NS>, - edge_writer: Option, ES>>, + edge_writer: EdgeWriter<'a, RwLockWriteGuard<'a, MemEdgeSegment>, ES>, graph: &'a GraphStore, } @@ -37,7 +37,7 @@ impl< { pub fn new( node_writers: NodeWriters<'a, RwLockWriteGuard<'a, MemNodeSegment>, NS>, - edge_writer: Option, ES>>, + edge_writer: EdgeWriter<'a, RwLockWriteGuard<'a, MemEdgeSegment>, ES>, graph: &'a GraphStore, ) -> Self { Self { @@ -69,19 +69,15 @@ impl< let (_, src_pos) = self.graph.nodes().resolve_pos(src); let (_, dst_pos) = self.graph.nodes().resolve_pos(dst); - if let Some(writer) = self.edge_writer.as_mut() { - let edge_max_page_len = writer.writer.get_or_create_layer(layer).max_page_len(); - let (_, edge_pos) = resolve_pos(e_id.edge, edge_max_page_len); + let edge_max_page_len = self + .edge_writer + .writer + .get_or_create_layer(layer) + .max_page_len(); + let (_, edge_pos) = resolve_pos(e_id.edge, edge_max_page_len); - writer.add_edge(t, edge_pos, src, dst, props, layer); - } else { - let mut writer = self.graph.edge_writer(e_id.edge); - let edge_max_page_len = writer.writer.get_or_create_layer(layer).max_page_len(); - let (_, edge_pos) = resolve_pos(e_id.edge, edge_max_page_len); - - writer.add_edge(t, edge_pos, src, dst, props, layer); - self.edge_writer = Some(writer); // Attach edge_writer to hold onto locks - } + self.edge_writer + .add_edge(t, edge_pos, src, dst, props, layer); let edge_id = edge.inner(); @@ -125,19 +121,15 @@ impl< let (_, src_pos) = self.graph.nodes().resolve_pos(src); let (_, dst_pos) = self.graph.nodes().resolve_pos(dst); - if let Some(writer) = self.edge_writer.as_mut() { - let edge_max_page_len = writer.writer.get_or_create_layer(layer).max_page_len(); - let (_, edge_pos) = resolve_pos(e_id.edge, edge_max_page_len); - - writer.delete_edge(t, edge_pos, src, dst, layer); - } else { - let mut writer = self.graph.edge_writer(e_id.edge); - let edge_max_page_len = writer.writer.get_or_create_layer(layer).max_page_len(); - let (_, edge_pos) = resolve_pos(e_id.edge, edge_max_page_len); + let edge_max_page_len = self + .edge_writer + .writer + .get_or_create_layer(layer) + .max_page_len(); + let (_, edge_pos) = resolve_pos(e_id.edge, edge_max_page_len); - writer.delete_edge(t, edge_pos, src, dst, layer); - self.edge_writer = Some(writer); // Attach edge_writer to hold onto locks - } + self.edge_writer + .delete_edge(t, edge_pos, src, dst, layer); let edge_id = edge.inner(); @@ -189,24 +181,18 @@ impl< .get_mut_src() .get_out_edge(src_pos, dst, layer_id) { - // If edge_writer is not set, we need to create a new one - if self.edge_writer.is_none() { - self.edge_writer = Some(self.graph.edge_writer(e_id)); - } - - let edge_writer = self.edge_writer.as_mut().unwrap(); let (_, edge_pos) = self.graph.edges().resolve_pos(e_id); - edge_writer.add_static_edge(Some(edge_pos), src, dst, true); + self.edge_writer + .add_static_edge(Some(edge_pos), src, dst, true); MaybeNew::Existing(e_id) } else { - let mut edge_writer = self.graph.get_free_writer(); - let edge_id = edge_writer.add_static_edge(None, src, dst, false); + let edge_id = self + .edge_writer + .add_static_edge(None, src, dst, false); let edge_id = - edge_id.as_eid(edge_writer.segment_id(), self.graph.edges().max_page_len()); - - self.edge_writer = Some(edge_writer); // Attach edge_writer to hold onto locks + edge_id.as_eid(self.edge_writer.segment_id(), self.graph.edges().max_page_len()); self.node_writers .get_mut_src() @@ -232,8 +218,6 @@ impl< dst.mut_segment.set_lsn(lsn); } - if let Some(edge_writer) = &mut self.edge_writer { - edge_writer.writer.set_lsn(lsn); - } + self.edge_writer.writer.set_lsn(lsn); } } From bc5410383534e71bfd1145eaa00a585c1b34cea7 Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Fri, 12 Dec 2025 11:41:12 +0400 Subject: [PATCH 17/19] Return early from add_static_edge if edge exists --- db4-storage/src/pages/session.rs | 40 +++++++++++++++----------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/db4-storage/src/pages/session.rs b/db4-storage/src/pages/session.rs index df45afc6a1..6df54943af 100644 --- a/db4-storage/src/pages/session.rs +++ b/db4-storage/src/pages/session.rs @@ -176,33 +176,31 @@ impl< let (_, src_pos) = self.graph.nodes().resolve_pos(src); let (_, dst_pos) = self.graph.nodes().resolve_pos(dst); - if let Some(e_id) = self + let existing_eid = self .node_writers .get_mut_src() - .get_out_edge(src_pos, dst, layer_id) - { - let (_, edge_pos) = self.graph.edges().resolve_pos(e_id); + .get_out_edge(src_pos, dst, layer_id); - self.edge_writer - .add_static_edge(Some(edge_pos), src, dst, true); + // Edge already exists, so no need to add it again. + if let Some(eid) = existing_eid { + return MaybeNew::Existing(eid) + } - MaybeNew::Existing(e_id) - } else { - let edge_id = self - .edge_writer - .add_static_edge(None, src, dst, false); - let edge_id = - edge_id.as_eid(self.edge_writer.segment_id(), self.graph.edges().max_page_len()); + let edge_pos = None; + let edge_exists_hint = false; + let edge_pos = + self.edge_writer.add_static_edge(edge_pos, src, dst, edge_exists_hint); + let edge_id = + edge_pos.as_eid(self.edge_writer.segment_id(), self.graph.edges().max_page_len()); - self.node_writers - .get_mut_src() - .add_static_outbound_edge(src_pos, dst, edge_id); - self.node_writers - .get_mut_dst() - .add_static_inbound_edge(dst_pos, src, edge_id); + self.node_writers + .get_mut_src() + .add_static_outbound_edge(src_pos, dst, edge_id); + self.node_writers + .get_mut_dst() + .add_static_inbound_edge(dst_pos, src, edge_id); - MaybeNew::New(edge_id) - } + MaybeNew::New(edge_id) } pub fn node_writers( From f7ac76a5c3db28fd444348bbc0231be04dbdeeaf Mon Sep 17 00:00:00 2001 From: Fadhil Abubaker Date: Fri, 12 Dec 2025 14:34:51 +0400 Subject: [PATCH 18/19] Move GraphReplay to WriteLockedGraph --- db4-graph/src/lib.rs | 3 ++ .../replay/mod.rs => db4-graph/src/replay.rs | 36 +++++++++---------- db4-storage/src/pages/locked/edges.rs | 5 +++ db4-storage/src/pages/locked/nodes.rs | 29 +++++++++------ raphtory/src/io/arrow/df_loaders.rs | 2 +- 5 files changed, 44 insertions(+), 31 deletions(-) rename raphtory/src/db/replay/mod.rs => db4-graph/src/replay.rs (65%) diff --git a/db4-graph/src/lib.rs b/db4-graph/src/lib.rs index babf9343a1..2b21a7d232 100644 --- a/db4-graph/src/lib.rs +++ b/db4-graph/src/lib.rs @@ -28,6 +28,8 @@ use storage::{ }; use tempfile::TempDir; +mod replay; + #[derive(Debug)] pub struct TemporalGraph { // mapping between logical and physical ids @@ -334,6 +336,7 @@ impl, ES = ES, GS = GS>> Temporal } } +/// Holds write locks across all segments in the graph for fast bulk ingestion. pub struct WriteLockedGraph<'a, EXT> where EXT: PersistentStrategy, ES = ES, GS = GS>, diff --git a/raphtory/src/db/replay/mod.rs b/db4-graph/src/replay.rs similarity index 65% rename from raphtory/src/db/replay/mod.rs rename to db4-graph/src/replay.rs index e332a0bfbf..203829a862 100644 --- a/raphtory/src/db/replay/mod.rs +++ b/db4-graph/src/replay.rs @@ -1,21 +1,28 @@ +//! Implement WAL replay for a `WriteLockedGraph`. +//! + use crate::db::api::{ - storage::{graph, storage::Storage}, - view::internal::{Base, InternalStorageOps}, - }; + storage::{graph, storage::Storage}, + view::internal::{Base, InternalStorageOps}, +}; +use crate::WriteLockedGraph; use raphtory_api::core::{ entities::{properties::prop::Prop, EID, GID, VID}, storage::timeindex::TimeIndexEntry, }; use raphtory_core::entities::GidRef; -use raphtory_storage::{core_ops::CoreGraphOps, mutation::addition_ops::{EdgeWriteLock, InternalAdditionOps}}; use storage::{ - api::edges::EdgeSegmentOps, + persist::strategy::PersistentStrategy, + NS, ES, GS, error::StorageError, wal::{GraphReplay, TransactionID, LSN}, }; use storage::resolver::GIDResolverOps; -impl GraphReplay for Storage { +impl GraphReplay for WriteLockedGraph<'_, EXT> +where + EXT: PersistentStrategy, ES = ES, GS = GS>, +{ fn replay_add_edge( &self, lsn: LSN, @@ -31,8 +38,7 @@ impl GraphReplay for Storage { props: Vec<(String, usize, Prop)>, ) -> Result<(), StorageError> { // TODO: Check max lsn on disk to see if this record should be replayed. - - let temporal_graph = self.core_graph().mutable().unwrap(); + let temporal_graph = self.graph(); // 1. Insert prop ids into edge meta. // No need to validate props again since they are already validated before @@ -57,19 +63,9 @@ impl GraphReplay for Storage { edge_meta.layer_meta().set_id(layer_name.as_deref().unwrap_or("_default"), layer_id); node_meta.layer_meta().set_id(layer_name.as_deref().unwrap_or("_default"), layer_id); - // 4. Grab src, dst and edge segment locks and add the edge. - println!("Grabbing add_edge_op lock"); - let mut add_edge_op = temporal_graph.atomic_add_edge(src_id, dst_id, Some(eid), layer_id).unwrap(); - - println!("Added edge to atomic_add_edge"); - - let edge_id = add_edge_op.internal_add_static_edge(src_id, dst_id); - let edge_id_with_layer = edge_id.map(|eid| eid.with_layer(layer_id)); - - println!("Adding edge to internal_add_edge"); - add_edge_op.internal_add_edge(t, src_id, dst_id, edge_id_with_layer, prop_ids); + // 4. Grab src, dst and edge segment writers and add the edge. + let src_writer = self.nodes().get_writer(src_id); - println!("Added edge to internal_add_edge"); Ok(()) } diff --git a/db4-storage/src/pages/locked/edges.rs b/db4-storage/src/pages/locked/edges.rs index a07f03147b..cee46e8981 100644 --- a/db4-storage/src/pages/locked/edges.rs +++ b/db4-storage/src/pages/locked/edges.rs @@ -79,6 +79,11 @@ impl<'a, ES: EdgeSegmentOps> WriteLockedEdgePages<'a, ES> { Self { writers } } + #[inline] + pub fn get(&mut self, segment_id: usize) -> Option<&mut LockedEdgePage<'a, ES>> { + self.writers.get_mut(segment_id) + } + pub fn par_iter_mut(&mut self) -> rayon::slice::IterMut<'_, LockedEdgePage<'a, ES>> { self.writers.par_iter_mut() } diff --git a/db4-storage/src/pages/locked/nodes.rs b/db4-storage/src/pages/locked/nodes.rs index 48b4fd7f10..d715cd7af2 100644 --- a/db4-storage/src/pages/locked/nodes.rs +++ b/db4-storage/src/pages/locked/nodes.rs @@ -11,7 +11,7 @@ use rayon::prelude::*; use std::ops::DerefMut; pub struct LockedNodePage<'a, NS> { - page_id: usize, + segment_id: usize, max_page_len: u32, layer_counter: &'a GraphStats, page: &'a NS, @@ -20,14 +20,14 @@ pub struct LockedNodePage<'a, NS> { impl<'a, NS: NodeSegmentOps> LockedNodePage<'a, NS> { pub fn new( - page_id: usize, + segment_id: usize, layer_counter: &'a GraphStats, max_page_len: u32, page: &'a NS, lock: RwLockWriteGuard<'a, MemNodeSegment>, ) -> Self { Self { - page_id, + segment_id, layer_counter, max_page_len, page, @@ -49,14 +49,15 @@ impl<'a, NS: NodeSegmentOps> LockedNodePage<'a, NS> { } #[inline(always)] - pub fn page_id(&self) -> usize { - self.page_id + pub fn segment_id(&self) -> usize { + self.segment_id } #[inline(always)] pub fn resolve_pos(&self, node_id: VID) -> Option { let (page, pos) = resolve_pos(node_id, self.max_page_len); - if page == self.page_id { + + if page == self.segment_id { Some(pos) } else { None @@ -86,6 +87,18 @@ impl<'a, NS: NodeSegmentOps> WriteLockedNodePages<'a, NS> { Self { writers } } + #[inline] + pub fn get( + &mut self, + segment_id: usize, + ) -> Option<&mut LockedNodePage<'a, NS>> { + self.writers.get_mut(segment_id) + } + + pub fn len(&self) -> usize { + self.writers.len() + } + pub fn par_iter_mut(&mut self) -> rayon::slice::IterMut<'_, LockedNodePage<'a, NS>> { self.writers.par_iter_mut() } @@ -104,10 +117,6 @@ impl<'a, NS: NodeSegmentOps> WriteLockedNodePages<'a, NS> { } } - pub fn len(&self) -> usize { - self.writers.len() - } - pub fn vacuum(&mut self) -> Result<(), StorageError> { for LockedNodePage { page, lock, .. } in &mut self.writers { page.vacuum(lock.deref_mut())?; diff --git a/raphtory/src/io/arrow/df_loaders.rs b/raphtory/src/io/arrow/df_loaders.rs index 6d9e2c1beb..5cfba8776d 100644 --- a/raphtory/src/io/arrow/df_loaders.rs +++ b/raphtory/src/io/arrow/df_loaders.rs @@ -415,9 +415,9 @@ pub fn load_edges_from_df Date: Tue, 16 Dec 2025 13:01:52 +0400 Subject: [PATCH 19/19] Implement add_edge replay for WriteLockedGraph --- db4-graph/src/replay.rs | 83 +++++++++++++++++--- db4-storage/src/pages/edge_page/writer.rs | 40 ++++++---- db4-storage/src/pages/edge_store.rs | 16 ++-- db4-storage/src/pages/locked/edges.rs | 2 +- db4-storage/src/pages/locked/nodes.rs | 13 ++- db4-storage/src/pages/node_store.rs | 10 ++- db4-storage/src/pages/session.rs | 9 +-- db4-storage/src/segments/edge/segment.rs | 4 +- db4-storage/src/wal/entry.rs | 2 +- db4-storage/src/wal/mod.rs | 4 +- raphtory-api/src/core/entities/mod.rs | 4 + raphtory/src/db/api/mutation/addition_ops.rs | 1 - raphtory/src/db/mod.rs | 1 - raphtory/src/io/arrow/df_loaders.rs | 12 +-- 14 files changed, 138 insertions(+), 63 deletions(-) diff --git a/db4-graph/src/replay.rs b/db4-graph/src/replay.rs index 203829a862..b620c4fe83 100644 --- a/db4-graph/src/replay.rs +++ b/db4-graph/src/replay.rs @@ -1,14 +1,13 @@ -//! Implement WAL replay for a `WriteLockedGraph`. -//! +//! Implements WAL replay for a `WriteLockedGraph`. +//! Allows for fast replay by making use of one-time lock acquisition for +//! all the segments in the graph. -use crate::db::api::{ - storage::{graph, storage::Storage}, - view::internal::{Base, InternalStorageOps}, -}; -use crate::WriteLockedGraph; +use storage::pages::resolve_pos; +use crate::{WriteLockedGraph}; use raphtory_api::core::{ entities::{properties::prop::Prop, EID, GID, VID}, storage::timeindex::TimeIndexEntry, + entities::properties::meta::STATIC_GRAPH_LAYER_ID, }; use raphtory_core::entities::GidRef; use storage::{ @@ -24,7 +23,7 @@ where EXT: PersistentStrategy, ES = ES, GS = GS>, { fn replay_add_edge( - &self, + &mut self, lsn: LSN, transaction_id: TransactionID, t: TimeIndexEntry, @@ -38,19 +37,22 @@ where props: Vec<(String, usize, Prop)>, ) -> Result<(), StorageError> { // TODO: Check max lsn on disk to see if this record should be replayed. + let temporal_graph = self.graph(); + let node_max_page_len = temporal_graph.storage().nodes().max_page_len(); + let edge_max_page_len = temporal_graph.storage().edges().max_page_len(); // 1. Insert prop ids into edge meta. // No need to validate props again since they are already validated before // being logged to the WAL. let edge_meta = temporal_graph.edge_meta(); - let mut prop_ids = Vec::new(); + let mut prop_ids_and_values = Vec::new(); for (prop_name, prop_id, prop_value) in props.into_iter() { let prop_mapper = edge_meta.temporal_prop_mapper(); prop_mapper.set_id_and_dtype(prop_name, prop_id, prop_value.dtype()); - prop_ids.push((prop_id, prop_value)); + prop_ids_and_values.push((prop_id, prop_value)); } // 2. Insert node ids into resolver. @@ -63,9 +65,66 @@ where edge_meta.layer_meta().set_id(layer_name.as_deref().unwrap_or("_default"), layer_id); node_meta.layer_meta().set_id(layer_name.as_deref().unwrap_or("_default"), layer_id); - // 4. Grab src, dst and edge segment writers and add the edge. - let src_writer = self.nodes().get_writer(src_id); + // 4. Grab src writer and add edge data. + let (src_segment_id, src_pos) = resolve_pos(src_id, node_max_page_len); + let num_nodes = src_id.index() + 1; + self.resize_chunks_to_num_nodes(num_nodes); // Create enough segments. + + let mut src_writer = self.nodes.get_mut(src_segment_id).unwrap().writer(); + src_writer.store_node_id(src_pos, STATIC_GRAPH_LAYER_ID, GidRef::from(&src_name)); + + let is_new_edge_static = src_writer.get_out_edge(src_pos, dst_id, STATIC_GRAPH_LAYER_ID).is_none(); + let is_new_edge_layer = src_writer.get_out_edge(src_pos, dst_id, layer_id).is_none(); + + // Add the edge to the static graph if it doesn't already exist. + if is_new_edge_static { + src_writer.add_static_outbound_edge(src_pos, dst_id, eid); + } + + // Add the edge to the layer if it doesn't already exist, else just record the timestamp. + if is_new_edge_layer { + src_writer.add_outbound_edge(Some(t), src_pos, dst_id, eid.with_layer(layer_id)); + } else { + src_writer.update_timestamp(t, src_pos, eid.with_layer(layer_id)); + } + + // Release the writer for mutable access to dst_writer. + drop(src_writer); + + // 5. Grab dst writer and add edge data. + let (dst_segment_id, dst_pos) = resolve_pos(dst_id, node_max_page_len); + let num_nodes = dst_id.index() + 1; + self.resize_chunks_to_num_nodes(num_nodes); + + let mut dst_writer = self.nodes.get_mut(dst_segment_id).unwrap().writer(); + dst_writer.store_node_id(dst_pos, STATIC_GRAPH_LAYER_ID, GidRef::from(&dst_name)); + + if is_new_edge_static { + dst_writer.add_static_inbound_edge(dst_pos, src_id, eid); + } + + if is_new_edge_layer { + dst_writer.add_inbound_edge(Some(t), dst_pos, src_id, eid.with_layer(layer_id)); + } else { + dst_writer.update_timestamp(t, dst_pos, eid.with_layer(layer_id)); + } + + drop(dst_writer); + + // 6. Grab edge writer and add temporal props & metadata. + let (edge_segment_id, edge_pos) = resolve_pos(eid, edge_max_page_len); + let num_edges = eid.index() + 1; + self.resize_chunks_to_num_edges(num_edges); + let mut edge_writer = self.edges.get_mut(edge_segment_id).unwrap().writer(); + + // Add edge into the static graph if it doesn't already exist. + if is_new_edge_static { + let already_counted = false; + edge_writer.add_static_edge(Some(edge_pos), src_id, dst_id, already_counted); + } + // Add edge into the specified layer with timestamp and props. + edge_writer.add_edge(t, edge_pos, src_id, dst_id, prop_ids_and_values, layer_id); Ok(()) } diff --git a/db4-storage/src/pages/edge_page/writer.rs b/db4-storage/src/pages/edge_page/writer.rs index 55babb1405..c8ad00db36 100644 --- a/db4-storage/src/pages/edge_page/writer.rs +++ b/db4-storage/src/pages/edge_page/writer.rs @@ -3,7 +3,7 @@ use crate::{ segments::edge::segment::MemEdgeSegment, }; use arrow_array::{ArrayRef, BooleanArray}; -use raphtory_api::core::entities::{VID, properties::prop::Prop}; +use raphtory_api::core::entities::{properties::{meta::STATIC_GRAPH_LAYER_ID, prop::Prop}, VID}; use raphtory_core::{ entities::EID, storage::timeindex::{AsTime, TimeIndexEntry}, @@ -46,15 +46,19 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen props: impl IntoIterator, layer_id: usize, ) -> LocalPOS { - let existing_edge = self + let is_new_edge = !self .page .contains_edge(edge_pos, layer_id, self.writer.deref()); - if !existing_edge { + + if is_new_edge { self.increment_layer_num_edges(layer_id); } + self.graph_stats.update_time(t.t()); + self.writer .insert_edge_internal(t, edge_pos, src, dst, layer_id, props); + edge_pos } @@ -102,23 +106,26 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen .delete_edge_internal(t, edge_pos, src, dst, layer_id); } + /// Adds a static edge to the graph. + /// + /// If `edge_pos` is `None`, a new position is allocated. If `Some`, the provided position + /// is used. + /// Set `already_counted` to `true` when bulk loading to avoid double-counting statistics. pub fn add_static_edge( &mut self, edge_pos: Option, src: impl Into, dst: impl Into, - exists_hint: bool, // used when edge_pos is Some but the is not counted, this is used in the bulk loader + already_counted: bool, ) -> LocalPOS { - let layer_id = 0; // assuming layer_id 0 for static edges, adjust as needed - - if edge_pos.is_some() && !exists_hint { + if edge_pos.is_some() && !already_counted { self.page.increment_num_edges(); - self.increment_layer_num_edges(layer_id); + self.increment_layer_num_edges(STATIC_GRAPH_LAYER_ID); } - let edge_pos = edge_pos.unwrap_or_else(|| self.new_local_pos(layer_id)); + let edge_pos = edge_pos.unwrap_or_else(|| self.new_local_pos(STATIC_GRAPH_LAYER_ID)); self.writer - .insert_static_edge_internal(edge_pos, src, dst, layer_id); + .insert_static_edge_internal(edge_pos, src, dst, STATIC_GRAPH_LAYER_ID); edge_pos } @@ -129,23 +136,24 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen edge_pos: LocalPOS, src: VID, dst: VID, - exists: bool, + edge_exists: bool, layer_id: usize, c_props: impl IntoIterator, t_props: impl IntoIterator, ) { - if !exists { - self.increment_layer_num_edges(0); + if !edge_exists { + self.increment_layer_num_edges(STATIC_GRAPH_LAYER_ID); self.increment_layer_num_edges(layer_id); + + self.writer + .insert_static_edge_internal(edge_pos, src, dst, STATIC_GRAPH_LAYER_ID); } - self.writer - .insert_static_edge_internal(edge_pos, src, dst, 0); + self.graph_stats.update_time(t.t()); self.writer .update_const_properties(edge_pos, src, dst, layer_id, c_props); - self.graph_stats.update_time(t.t()); self.writer .insert_edge_internal(t, edge_pos, src, dst, layer_id, t_props); } diff --git a/db4-storage/src/pages/edge_store.rs b/db4-storage/src/pages/edge_store.rs index d50f87cc1d..2f676e5246 100644 --- a/db4-storage/src/pages/edge_store.rs +++ b/db4-storage/src/pages/edge_store.rs @@ -17,7 +17,7 @@ use crate::{ segments::edge::segment::MemEdgeSegment, }; use parking_lot::{RwLock, RwLockWriteGuard}; -use raphtory_api::core::entities::{EID, VID, properties::meta::Meta}; +use raphtory_api::core::entities::{properties::meta::{Meta, STATIC_GRAPH_LAYER_ID}, EID, VID}; use raphtory_core::{ entities::{ELID, LayerIds}, storage::timeindex::{AsTime, TimeIndexEntry}, @@ -134,20 +134,24 @@ impl, EXT: Config> EdgeStorageInner let layer_mapper = empty.edge_meta().layer_meta(); let prop_mapper = empty.edge_meta().temporal_prop_mapper(); let metadata_mapper = empty.edge_meta().metadata_mapper(); + if layer_mapper.num_fields() > 0 || prop_mapper.num_fields() > 0 || metadata_mapper.num_fields() > 0 { - let segment = empty.get_or_create_segment(0); + let segment = empty.get_or_create_segment(STATIC_GRAPH_LAYER_ID); let mut head = segment.head_mut(); + for layer in layer_mapper.ids() { head.get_or_create_layer(layer); } + if prop_mapper.num_fields() > 0 { head.get_or_create_layer(0) .properties_mut() .set_has_properties() } + segment.set_dirty(true); } empty @@ -334,9 +338,11 @@ impl, EXT: Config> EdgeStorageInner if let Some(segment) = self.segments.get(segment_id) { return segment; } + let count = self.segments.count(); + if count > segment_id { - // something has allocated the segment, wait for it to be added + // Something has allocated the segment, wait for it to be added. loop { if let Some(segment) = self.segments.get(segment_id) { return segment; @@ -346,7 +352,7 @@ impl, EXT: Config> EdgeStorageInner } } } else { - // we need to create the segment + // We need to create the segment. self.segments.reserve(segment_id + 1 - count); loop { @@ -364,7 +370,7 @@ impl, EXT: Config> EdgeStorageInner if let Some(segment) = self.segments.get(segment_id) { return segment; } else { - // wait for the segment to be created + // Wait for the segment to be created. std::thread::yield_now(); } } diff --git a/db4-storage/src/pages/locked/edges.rs b/db4-storage/src/pages/locked/edges.rs index cee46e8981..ff01546c1d 100644 --- a/db4-storage/src/pages/locked/edges.rs +++ b/db4-storage/src/pages/locked/edges.rs @@ -80,7 +80,7 @@ impl<'a, ES: EdgeSegmentOps> WriteLockedEdgePages<'a, ES> { } #[inline] - pub fn get(&mut self, segment_id: usize) -> Option<&mut LockedEdgePage<'a, ES>> { + pub fn get_mut(&mut self, segment_id: usize) -> Option<&mut LockedEdgePage<'a, ES>> { self.writers.get_mut(segment_id) } diff --git a/db4-storage/src/pages/locked/nodes.rs b/db4-storage/src/pages/locked/nodes.rs index d715cd7af2..78aed9dbd5 100644 --- a/db4-storage/src/pages/locked/nodes.rs +++ b/db4-storage/src/pages/locked/nodes.rs @@ -87,18 +87,15 @@ impl<'a, NS: NodeSegmentOps> WriteLockedNodePages<'a, NS> { Self { writers } } - #[inline] - pub fn get( - &mut self, - segment_id: usize, - ) -> Option<&mut LockedNodePage<'a, NS>> { - self.writers.get_mut(segment_id) - } - pub fn len(&self) -> usize { self.writers.len() } + #[inline] + pub fn get_mut(&mut self, segment_id: usize) -> Option<&mut LockedNodePage<'a, NS>> { + self.writers.get_mut(segment_id) + } + pub fn par_iter_mut(&mut self) -> rayon::slice::IterMut<'_, LockedNodePage<'a, NS>> { self.writers.par_iter_mut() } diff --git a/db4-storage/src/pages/node_store.rs b/db4-storage/src/pages/node_store.rs index 44fccfbfd2..b94924f1a0 100644 --- a/db4-storage/src/pages/node_store.rs +++ b/db4-storage/src/pages/node_store.rs @@ -354,19 +354,21 @@ impl, EXT: Config> NodeStorageInner if let Some(segment) = self.pages.get(segment_id) { return segment; } + let count = self.pages.count(); + if count > segment_id { - // something has allocated the segment, wait for it to be added + // Something has allocated the segment, wait for it to be added. loop { if let Some(segment) = self.pages.get(segment_id) { return segment; } else { - // wait for the segment to be created + // Wait for the segment to be created. std::thread::yield_now(); } } } else { - // we need to create the segment + // We need to create the segment. self.pages.reserve(segment_id + 1 - count); loop { @@ -385,7 +387,7 @@ impl, EXT: Config> NodeStorageInner if let Some(segment) = self.pages.get(segment_id) { return segment; } else { - // wait for the segment to be created + // Wait for the segment to be created. std::thread::yield_now(); } } diff --git a/db4-storage/src/pages/session.rs b/db4-storage/src/pages/session.rs index 6df54943af..a83bb9f899 100644 --- a/db4-storage/src/pages/session.rs +++ b/db4-storage/src/pages/session.rs @@ -9,7 +9,7 @@ use crate::{ wal::LSN, }; use parking_lot::RwLockWriteGuard; -use raphtory_api::core::{entities::properties::prop::Prop, storage::dict_mapper::MaybeNew}; +use raphtory_api::core::{entities::properties::{meta::STATIC_GRAPH_LAYER_ID, prop::Prop}, storage::dict_mapper::MaybeNew}; use raphtory_core::{ entities::{EID, ELID, VID}, storage::timeindex::AsTime, @@ -171,7 +171,6 @@ impl< ) -> MaybeNew { let src = src.into(); let dst = dst.into(); - let layer_id = 0; // static graph goes to layer 0 let (_, src_pos) = self.graph.nodes().resolve_pos(src); let (_, dst_pos) = self.graph.nodes().resolve_pos(dst); @@ -179,7 +178,7 @@ impl< let existing_eid = self .node_writers .get_mut_src() - .get_out_edge(src_pos, dst, layer_id); + .get_out_edge(src_pos, dst, STATIC_GRAPH_LAYER_ID); // Edge already exists, so no need to add it again. if let Some(eid) = existing_eid { @@ -187,9 +186,9 @@ impl< } let edge_pos = None; - let edge_exists_hint = false; + let already_counted = false; let edge_pos = - self.edge_writer.add_static_edge(edge_pos, src, dst, edge_exists_hint); + self.edge_writer.add_static_edge(edge_pos, src, dst, already_counted); let edge_id = edge_pos.as_eid(self.edge_writer.segment_id(), self.graph.edges().max_page_len()); diff --git a/db4-storage/src/segments/edge/segment.rs b/db4-storage/src/segments/edge/segment.rs index eb946ef868..f7e5a72923 100644 --- a/db4-storage/src/segments/edge/segment.rs +++ b/db4-storage/src/segments/edge/segment.rs @@ -228,8 +228,10 @@ impl MemEdgeSegment { let mut prop_entry: PropMutEntry<'_> = self.layers[layer_id] .properties_mut() .get_mut_entry(local_row); + let ts = TimeIndexEntry::new(t.t(), t.i()); prop_entry.append_t_props(ts, props); + let layer_est_size = self.layers[layer_id].est_size(); self.est_size += layer_est_size.saturating_sub(est_size); } @@ -276,7 +278,7 @@ impl MemEdgeSegment { fn ensure_layer(&mut self, layer_id: usize) { if layer_id >= self.layers.len() { - // Get details from first layer to create consistent new layers + // Get details from first layer to create consistent new layers. if let Some(first_layer) = self.layers.first() { let segment_id = first_layer.segment_id(); let max_page_len = first_layer.max_page_len(); diff --git a/db4-storage/src/wal/entry.rs b/db4-storage/src/wal/entry.rs index dc2680580f..7b0b0e6745 100644 --- a/db4-storage/src/wal/entry.rs +++ b/db4-storage/src/wal/entry.rs @@ -42,7 +42,7 @@ impl GraphWal for NoWal { fn replay_to_graph( _dir: impl AsRef, - _graph: &G, + _graph: &mut G, ) -> Result<(), StorageError> { todo!() } diff --git a/db4-storage/src/wal/mod.rs b/db4-storage/src/wal/mod.rs index 72b26c0d41..9752c6ef4f 100644 --- a/db4-storage/src/wal/mod.rs +++ b/db4-storage/src/wal/mod.rs @@ -72,14 +72,14 @@ pub trait GraphWal { /// Replays and applies all the wal entries in the given directory to the given graph. fn replay_to_graph( dir: impl AsRef, - graph: &G, + graph: &mut G, ) -> Result<(), StorageError>; } /// Trait for defining callbacks for replaying from wal. pub trait GraphReplay { fn replay_add_edge( - &self, + &mut self, lsn: LSN, transaction_id: TransactionID, t: TimeIndexEntry, diff --git a/raphtory-api/src/core/entities/mod.rs b/raphtory-api/src/core/entities/mod.rs index cce5d1c80a..2256f86c6d 100644 --- a/raphtory-api/src/core/entities/mod.rs +++ b/raphtory-api/src/core/entities/mod.rs @@ -64,6 +64,10 @@ impl Default for EID { } impl EID { + pub fn index(&self) -> usize { + self.0 + } + pub fn as_u64(self) -> u64 { self.0 as u64 } diff --git a/raphtory/src/db/api/mutation/addition_ops.rs b/raphtory/src/db/api/mutation/addition_ops.rs index 769eb14e32..9d62c395d8 100644 --- a/raphtory/src/db/api/mutation/addition_ops.rs +++ b/raphtory/src/db/api/mutation/addition_ops.rs @@ -299,7 +299,6 @@ impl> + StaticGraphViewOps + Dura // All names, ids and values have been generated for this operation. // Create a wal entry to mark it as durable. - let props_for_wal = props_with_status .iter() .map(|maybe_new| { diff --git a/raphtory/src/db/mod.rs b/raphtory/src/db/mod.rs index 54e9c74f6c..63e711afda 100644 --- a/raphtory/src/db/mod.rs +++ b/raphtory/src/db/mod.rs @@ -1,4 +1,3 @@ pub mod api; pub mod graph; -pub mod replay; pub mod task; diff --git a/raphtory/src/io/arrow/df_loaders.rs b/raphtory/src/io/arrow/df_loaders.rs index 5cfba8776d..5b5ec145c6 100644 --- a/raphtory/src/io/arrow/df_loaders.rs +++ b/raphtory/src/io/arrow/df_loaders.rs @@ -459,7 +459,7 @@ pub fn load_edges_from_df