From 200608690d1f983a67addf4145b69d6997610722 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Thu, 4 Jul 2024 23:50:01 +0200 Subject: [PATCH 01/28] WIP --- Cargo.toml | 12 +-- src/datastore.rs | 207 ++++++++++++++++++++++++++++++++++------------- src/errors.rs | 2 +- src/managers.rs | 73 ++++++++++------- 4 files changed, 198 insertions(+), 96 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index cc9ed03..15808c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,9 +19,9 @@ test-suite = ["indradb-lib/test-suite", "tempfile"] bench-suite = ["indradb-lib/bench-suite", "tempfile"] [dependencies] -chrono = { version = "0.4.19", features = ["serde"] } -indradb-lib = "^2.2.0" -serde_json = "^1.0.57" -sled = { version = "0.34.6", features = ["compression", "no_metrics"] } -tempfile = { version = "^3.2.0", optional = true} -uuid = { version = "~0.8.2", features = ["v1", "serde"] } +chrono = { version = "0.4", features = ["serde"] } +indradb-lib = "4.0" +serde_json = "1.0" +sled = { version = "0.34" } +tempfile = { version = "3.10", optional = true } +uuid = { version = "1.9", features = ["v1", "serde"] } diff --git a/src/datastore.rs b/src/datastore.rs index 1d3f8c3..10a53e8 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -1,21 +1,17 @@ +use std::{u64, usize}; use std::path::Path; use std::sync::Arc; -use std::{u64, usize}; - -use super::errors::map_err; -use super::managers::*; use chrono::offset::Utc; +use indradb::{Datastore, DynIter, Edge, EdgeDirection, EdgeProperties, EdgeProperty, Identifier, Json, NamedProperty, Result, Transaction, Vertex, VertexProperties, VertexProperty}; use indradb::util::next_uuid; -use indradb::{ - BulkInsertItem, Datastore, Edge, EdgeDirection, EdgeKey, EdgeProperties, EdgeProperty, EdgePropertyQuery, - EdgeQuery, NamedProperty, Result, Transaction, Type, Vertex, VertexProperties, VertexProperty, VertexPropertyQuery, - VertexQuery, -}; use serde_json::Value as JsonValue; use sled::{Config, Db, Tree}; use uuid::Uuid; +use super::errors::map_err; +use super::managers::*; + #[derive(Copy, Clone, Default, Debug)] pub struct SledConfig { use_compression: bool, @@ -101,48 +97,12 @@ impl<'ds> SledDatastore { } impl Datastore for SledDatastore { - type Trans = SledTransaction; + type Transaction = SledTransaction; - fn sync(&self) -> Result<()> { - let holder = self.holder.clone(); - let db = holder.db.clone(); - map_err(db.flush())?; - Ok(()) - } fn transaction(&self) -> Result { Ok(SledTransaction::new(self.holder.clone())) } - - fn bulk_insert(&self, items: I) -> Result<()> - where - I: Iterator, - { - let vertex_manager = VertexManager::new(&self.holder); - let edge_manager = EdgeManager::new(&self.holder); - let vertex_property_manager = VertexPropertyManager::new(&self.holder.vertex_properties); - let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); - - for item in items { - match item { - BulkInsertItem::Vertex(ref vertex) => { - vertex_manager.create(vertex)?; - } - BulkInsertItem::Edge(ref key) => { - edge_manager.set(key.outbound_id, &key.t, key.inbound_id, Utc::now())?; - } - BulkInsertItem::VertexProperty(id, ref name, ref value) => { - vertex_property_manager.set(id, name, value)?; - } - BulkInsertItem::EdgeProperty(ref key, ref name, ref value) => { - edge_property_manager.set(key.outbound_id, &key.t, key.inbound_id, name, value)?; - } - } - } - - map_err(self.holder.db.flush())?; - Ok(()) - } } /// A transaction that is backed by Sled. @@ -150,6 +110,10 @@ pub struct SledTransaction { holder: Arc, } +pub struct SledTransactionFour { + holder: Arc, +} + impl SledTransaction { fn new(holder: Arc) -> Self { SledTransaction { holder } @@ -159,7 +123,7 @@ impl SledTransaction { fn vertex_query_to_iterator<'iter, 'trans: 'iter>( &'trans self, q: VertexQuery, - ) -> Result> + 'iter>> { + ) -> Result> + 'iter>> { match q { VertexQuery::Range(q) => { let vertex_manager = VertexManager::new(&self.holder); @@ -179,7 +143,7 @@ impl SledTransaction { None => Uuid::default(), }; - let mut iter: Box>> = + let mut iter: Box>> = Box::new(vertex_manager.iterate_for_range(next_uuid)); if let Some(ref t) = q.t { @@ -221,7 +185,7 @@ impl SledTransaction { } }); - let mut iter: Box>> = Box::new(remove_nones_from_iterator(iter)); + let mut iter: Box>> = Box::new(remove_nones_from_iterator(iter)); if let Some(ref t) = q.t { iter = Box::new(iter.filter(move |item| match item { @@ -239,7 +203,7 @@ impl SledTransaction { fn edge_query_to_iterator<'iter, 'trans: 'iter>( &'trans self, q: EdgeQuery, - ) -> Result> + 'iter>> { + ) -> Result> + 'iter>> { match q { EdgeQuery::Specific(q) => { let edge_manager = EdgeManager::new(&self.holder); @@ -278,11 +242,11 @@ impl SledTransaction { for item in edge_iterator { match item { Ok(( - edge_range_first_id, - edge_range_t, - edge_range_update_datetime, - edge_range_second_id, - )) => { + edge_range_first_id, + edge_range_t, + edge_range_update_datetime, + edge_range_second_id, + )) => { if let Some(low) = q.low { if edge_range_update_datetime < low { break; @@ -319,6 +283,133 @@ impl SledTransaction { } } +impl<'a> Transaction<'a> for SledTransactionFour { + fn vertex_count(&self) -> u64 { + let vertex_manager = VertexManager::new(&self.holder); + let iterator = vertex_manager.iterate_for_range(Uuid::default()); + iterator.count() as u64 + } + + fn all_vertices(&'a self) -> Result> { + let vertex_manager = VertexManager::new(&self.holder); + let iterator = vertex_manager.iterate_for_range(Uuid::default()); + let mapped = iterator.map(move |item| { + let (id, t) = item?; + let vertex = Vertex::with_id(id, t); + Ok(vertex) + }); + + Ok(mapped.into()) + } + + fn range_vertices(&'a self, offset: Uuid) -> Result> { + let vertex_manager = VertexManager::new(&self.holder); + let iter = vertex_manager.iterate_for_range(offset); + Ok(iter.into()) + } + + fn specific_vertices(&'a self, ids: Vec) -> Result> { + let vertex_manager = VertexManager::new(&self.holder); + let iter = ids.into_iter().filter_map(move |id| vertex_manager.get(id)?) + .map(|a|); + Ok(iter) + } + + fn vertex_ids_with_property(&'a self, name: Identifier) -> Result>> { + let vertex_manager = VertexManager::new(&self.holder); + let iter = ids.into_iter().map(move |id| match vertex_manager.get(id)? { + Some(value) => Ok(Some((id, value))), + None => Ok(None), + }); + + Ok(Box::new(remove_nones_from_iterator(iter))) + } + + fn vertex_ids_with_property_value(&'a self, name: Identifier, value: &Json) -> Result>> { + todo!() + } + + fn edge_count(&self) -> u64 { + todo!() + } + + fn all_edges(&'a self) -> Result> { + todo!() + } + + fn range_edges(&'a self, offset: Edge) -> Result> { + todo!() + } + + fn range_reversed_edges(&'a self, offset: Edge) -> Result> { + todo!() + } + + fn specific_edges(&'a self, edges: Vec) -> Result> { + todo!() + } + + fn edges_with_property(&'a self, name: Identifier) -> Result>> { + todo!() + } + + fn edges_with_property_value(&'a self, name: Identifier, value: &Json) -> Result>> { + todo!() + } + + fn vertex_property(&self, vertex: &Vertex, name: Identifier) -> Result> { + todo!() + } + + fn all_vertex_properties_for_vertex(&'a self, vertex: &Vertex) -> Result> { + todo!() + } + + fn edge_property(&self, edge: &Edge, name: Identifier) -> Result> { + todo!() + } + + fn all_edge_properties_for_edge(&'a self, edge: &Edge) -> Result> { + todo!() + } + + fn delete_vertices(&mut self, vertices: Vec) -> Result<()> { + todo!() + } + + fn delete_edges(&mut self, edges: Vec) -> Result<()> { + todo!() + } + + fn delete_vertex_properties(&mut self, props: Vec<(Uuid, Identifier)>) -> Result<()> { + todo!() + } + + fn delete_edge_properties(&mut self, props: Vec<(Edge, Identifier)>) -> Result<()> { + todo!() + } + + fn create_vertex(&mut self, vertex: &Vertex) -> Result { + todo!() + } + + fn create_edge(&mut self, edge: &Edge) -> Result { + todo!() + } + + fn index_property(&mut self, name: Identifier) -> Result<()> { + todo!() + } + + fn set_vertex_properties(&mut self, vertices: Vec, name: Identifier, value: &Json) -> Result<()> { + todo!() + } + + fn set_edge_properties(&mut self, edges: Vec, name: Identifier, value: &Json) -> Result<()> { + todo!() + } +} + impl Transaction for SledTransaction { fn create_vertex(&self, vertex: &Vertex) -> Result { let vertex_manager = VertexManager::new(&self.holder); @@ -355,7 +446,7 @@ impl Transaction for SledTransaction { Ok(()) } - fn get_vertex_count(&self) -> Result { + fn vertex_count(&self) -> Result { let vertex_manager = VertexManager::new(&self.holder); let iterator = vertex_manager.iterate_for_range(Uuid::default()); Ok(iterator.count() as u64) @@ -528,9 +619,9 @@ impl Transaction for SledTransaction { } } -fn remove_nones_from_iterator(iter: I) -> impl Iterator> +fn remove_nones_from_iterator(iter: I) -> impl Iterator> where - I: Iterator>>, + I: Iterator>>, { iter.filter_map(|item| match item { Err(err) => Some(Err(err)), diff --git a/src/errors.rs b/src/errors.rs index 45e026a..4e42071 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -2,5 +2,5 @@ use indradb::Error as IndraError; use sled::Error as SledError; pub(crate) fn map_err(result: Result) -> Result { - result.map_err(|err| IndraError::Datastore { inner: Box::new(err) }) + result.map_err(|err| IndraError::Datastore { 0: Box::new(err) }) } diff --git a/src/managers.rs b/src/managers.rs index 0fe2567..1d0d63f 100644 --- a/src/managers.rs +++ b/src/managers.rs @@ -2,23 +2,24 @@ use std::io::Cursor; use std::ops::Deref; use std::u8; -use super::errors::map_err; -use crate::datastore::SledHolder; - -use chrono::offset::Utc; use chrono::DateTime; -use indradb::{util, Result, Type, Vertex}; +use chrono::offset::Utc; +use indradb::{Identifier, Result, util, Vertex}; use serde_json::Value as JsonValue; +use sled::{Iter as DbIterator, IVec, Tree}; use sled::Result as SledResult; -use sled::{IVec, Iter as DbIterator, Tree}; use uuid::Uuid; +use crate::datastore::SledHolder; + +use super::errors::map_err; + pub type OwnedPropertyItem = ((Uuid, String), JsonValue); -pub type VertexItem = (Uuid, Type); -pub type EdgeRangeItem = (Uuid, Type, DateTime, Uuid); -pub type EdgePropertyItem = ((Uuid, Type, Uuid, String), JsonValue); +pub type VertexItem = (Uuid, Identifier); +pub type EdgeRangeItem = (Uuid, Identifier, DateTime, Uuid); +pub type EdgePropertyItem = ((Uuid, Identifier, Uuid, String), JsonValue); -fn take_while_prefixed(iterator: DbIterator, prefix: Vec) -> impl Iterator> { +fn take_while_prefixed(iterator: DbIterator, prefix: Vec) -> impl Iterator> { iterator.take_while(move |item| -> bool { match item { Ok((k, _)) => k.starts_with(&prefix), @@ -48,17 +49,17 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { Ok(map_err(self.tree.get(&self.key(id)))?.is_some()) } - pub fn get(&self, id: Uuid) -> Result> { + pub fn get(&self, id: Uuid) -> Result> { match map_err(self.tree.get(&self.key(id)))? { Some(value_bytes) => { let mut cursor = Cursor::new(value_bytes.deref()); - Ok(Some(util::read_type(&mut cursor))) + Ok(Some(util::read_identifier(&mut cursor))) } None => Ok(None), } } - fn iterate(&self, iterator: DbIterator) -> impl Iterator> + '_ { + fn iterate(&self, iterator: DbIterator) -> impl Iterator> + '_ { iterator.map(move |item| -> Result { let (k, v) = map_err(item)?; @@ -69,12 +70,12 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { }; let mut cursor = Cursor::new(v); - let t = util::read_type(&mut cursor); + let t = util::read_identifier(&mut cursor); Ok((id, t)) }) } - pub fn iterate_for_range(&self, id: Uuid) -> impl Iterator> + '_ { + pub fn iterate_for_range(&self, id: Uuid) -> impl Iterator> + '_ { let low_key = util::build(&[util::Component::Uuid(id)]); let low_key_bytes: &[u8] = low_key.as_ref(); let iter = self.tree.range(low_key_bytes..); @@ -83,7 +84,7 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { pub fn create(&self, vertex: &Vertex) -> Result<()> { let key = self.key(vertex.id); - map_err(self.tree.insert(&key, util::build(&[util::Component::Type(&vertex.t)])))?; + map_err(self.tree.insert(&key, util::build(&[util::Component::Identifier(vertex.t.clone())])))?; Ok(()) } @@ -139,6 +140,17 @@ pub struct EdgeManager<'db: 'tree, 'tree> { pub tree: &'tree Tree, } +fn time_from_bytes(value_bytes: IVec) -> Result>> { + let mut cursor = Cursor::new(value_bytes.deref()); + let time: i64 = util::read_fixed_length_string(&mut cursor).parse()?; + let time = DateTime::from_timestamp_micros(time); + Ok(time) +} + +fn time_to_string(time: DateTime) -> String { + time.timestamp_micros().to_string() +} + impl<'db, 'tree> EdgeManager<'db, 'tree> { pub fn new(ds: &'db SledHolder) -> Self { EdgeManager { @@ -147,25 +159,24 @@ impl<'db, 'tree> EdgeManager<'db, 'tree> { } } - fn key(&self, outbound_id: Uuid, t: &Type, inbound_id: Uuid) -> Vec { + fn key(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid) -> Vec { util::build(&[ util::Component::Uuid(outbound_id), - util::Component::Type(t), + util::Component::Identifier(t.clone()), util::Component::Uuid(inbound_id), ]) } - pub fn get(&self, outbound_id: Uuid, t: &Type, inbound_id: Uuid) -> Result>> { + pub fn get(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid) -> Result>> { match map_err(self.tree.get(self.key(outbound_id, t, inbound_id)))? { Some(value_bytes) => { - let mut cursor = Cursor::new(value_bytes.deref()); - Ok(Some(util::read_datetime(&mut cursor))) + time_from_bytes(value_bytes) } None => Ok(None), } } - pub fn set(&self, outbound_id: Uuid, t: &Type, inbound_id: Uuid, new_update_datetime: DateTime) -> Result<()> { + pub fn set(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, new_update_datetime: DateTime) -> Result<()> { let edge_range_manager = EdgeRangeManager::new(self.holder); let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); @@ -177,14 +188,14 @@ impl<'db, 'tree> EdgeManager<'db, 'tree> { let key = self.key(outbound_id, t, inbound_id); map_err( self.tree - .insert(key, util::build(&[util::Component::DateTime(new_update_datetime)])), + .insert(key, util::build(&[util::Component::FixedLengthString(&time_to_string(new_update_datetime))])), )?; edge_range_manager.set(outbound_id, t, new_update_datetime, inbound_id)?; reversed_edge_range_manager.set(inbound_id, t, new_update_datetime, outbound_id)?; Ok(()) } - pub fn delete(&self, outbound_id: Uuid, t: &Type, inbound_id: Uuid, update_datetime: DateTime) -> Result<()> { + pub fn delete(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, update_datetime: DateTime) -> Result<()> { map_err(self.tree.remove(&self.key(outbound_id, t, inbound_id)))?; let edge_range_manager = EdgeRangeManager::new(self.holder); @@ -222,16 +233,16 @@ impl<'tree> EdgeRangeManager<'tree> { } } - fn key(&self, first_id: Uuid, t: &Type, update_datetime: DateTime, second_id: Uuid) -> Vec { + fn key(&self, first_id: Uuid, t: &Identifier, update_datetime: DateTime, second_id: Uuid) -> Vec { util::build(&[ util::Component::Uuid(first_id), - util::Component::Type(t), + util::Component::Identifier(t), util::Component::DateTime(update_datetime), util::Component::Uuid(second_id), ]) } - fn iterate<'it>(&self, iterator: DbIterator, prefix: Vec) -> impl Iterator> + 'it { + fn iterate<'it>(&self, iterator: DbIterator, prefix: Vec) -> impl Iterator> + 'it { let filtered = take_while_prefixed(iterator, prefix); filtered.map(move |item| -> Result { let (k, _) = map_err(item)?; @@ -249,7 +260,7 @@ impl<'tree> EdgeRangeManager<'tree> { id: Uuid, t: Option<&Type>, high: Option>, - ) -> Result> + 'iter>> { + ) -> Result> + 'iter>> { match t { Some(t) => { let high = high.unwrap_or_else(|| *util::MAX_DATETIME); @@ -292,7 +303,7 @@ impl<'tree> EdgeRangeManager<'tree> { pub fn iterate_for_owner<'iter, 'trans: 'iter>( &'trans self, id: Uuid, - ) -> impl Iterator> + 'iter { + ) -> impl Iterator> + 'iter { let prefix: Vec = util::build(&[util::Component::Uuid(id)]); let iterator = self.tree.scan_prefix(&prefix); self.iterate(iterator, prefix) @@ -326,7 +337,7 @@ impl<'tree> VertexPropertyManager<'tree> { ]) } - pub fn iterate_for_owner(&self, vertex_id: Uuid) -> Result> + '_> { + pub fn iterate_for_owner(&self, vertex_id: Uuid) -> Result> + '_> { let prefix = util::build(&[util::Component::Uuid(vertex_id)]); let iterator = self.tree.scan_prefix(&prefix); @@ -386,7 +397,7 @@ impl<'tree> EdgePropertyManager<'tree> { outbound_id: Uuid, t: &'a Type, inbound_id: Uuid, - ) -> Result> + 'a>> { + ) -> Result> + 'a>> { let prefix = util::build(&[ util::Component::Uuid(outbound_id), util::Component::Type(t), From b81036d5cfa2e6b641bf366069104a1fd1273366 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Fri, 5 Jul 2024 16:06:35 +0200 Subject: [PATCH 02/28] First shot at trying to implement new Transaction API, not building, needs error handling. --- src/datastore.rs | 502 +++++++++-------------------------------------- src/managers.rs | 58 ++++-- 2 files changed, 133 insertions(+), 427 deletions(-) diff --git a/src/datastore.rs b/src/datastore.rs index 10a53e8..a25b64f 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -1,11 +1,9 @@ -use std::{u64, usize}; use std::path::Path; use std::sync::Arc; +use std::u64; use chrono::offset::Utc; -use indradb::{Datastore, DynIter, Edge, EdgeDirection, EdgeProperties, EdgeProperty, Identifier, Json, NamedProperty, Result, Transaction, Vertex, VertexProperties, VertexProperty}; -use indradb::util::next_uuid; -use serde_json::Value as JsonValue; +use indradb::{Datastore, DynIter, Edge, Identifier, Json, Result, Transaction, Vertex}; use sled::{Config, Db, Tree}; use uuid::Uuid; @@ -97,197 +95,29 @@ impl<'ds> SledDatastore { } impl Datastore for SledDatastore { - type Transaction = SledTransaction; + type Transaction<'a> = SledTransaction; - - fn transaction(&self) -> Result { - Ok(SledTransaction::new(self.holder.clone())) + fn transaction(&self) -> Self::Transaction<'_> { + SledTransaction::new(self.holder.clone()) } } + /// A transaction that is backed by Sled. pub struct SledTransaction { holder: Arc, } -pub struct SledTransactionFour { - holder: Arc, -} - impl SledTransaction { fn new(holder: Arc) -> Self { SledTransaction { holder } } - - #[allow(clippy::needless_collect)] - fn vertex_query_to_iterator<'iter, 'trans: 'iter>( - &'trans self, - q: VertexQuery, - ) -> Result> + 'iter>> { - match q { - VertexQuery::Range(q) => { - let vertex_manager = VertexManager::new(&self.holder); - - let next_uuid = match q.start_id { - Some(start_id) => { - match next_uuid(start_id) { - Ok(next_uuid) => next_uuid, - // If we get an error back, it's because - // `start_id` is the maximum possible value. We - // know that no vertices exist whose ID is greater - // than the maximum possible value, so just return - // an empty list. - Err(_) => return Ok(Box::new(vec![].into_iter())), - } - } - None => Uuid::default(), - }; - - let mut iter: Box>> = - Box::new(vertex_manager.iterate_for_range(next_uuid)); - - if let Some(ref t) = q.t { - iter = Box::new(iter.filter(move |item| match item { - Ok((_, v)) => v == t, - Err(_) => true, - })); - } - - let results: Vec> = iter.take(q.limit as usize).collect(); - Ok(Box::new(results.into_iter())) - } - VertexQuery::Specific(q) => { - let vertex_manager = VertexManager::new(&self.holder); - - let iter = q.ids.into_iter().map(move |id| match vertex_manager.get(id)? { - Some(value) => Ok(Some((id, value))), - None => Ok(None), - }); - - Ok(Box::new(remove_nones_from_iterator(iter))) - } - VertexQuery::Pipe(q) => { - let vertex_manager = VertexManager::new(&self.holder); - let edge_iterator = self.edge_query_to_iterator(*q.inner)?; - let direction = q.direction; - - let iter = edge_iterator.map(move |item| { - let (outbound_id, _, _, inbound_id) = item?; - - let id = match direction { - EdgeDirection::Outbound => outbound_id, - EdgeDirection::Inbound => inbound_id, - }; - - match vertex_manager.get(id)? { - Some(value) => Ok(Some((id, value))), - None => Ok(None), - } - }); - - let mut iter: Box>> = Box::new(remove_nones_from_iterator(iter)); - - if let Some(ref t) = q.t { - iter = Box::new(iter.filter(move |item| match item { - Ok((_, v)) => v == t, - Err(_) => true, - })); - } - - let results: Vec> = iter.take(q.limit as usize).collect(); - Ok(Box::new(results.into_iter())) - } - } - } - - fn edge_query_to_iterator<'iter, 'trans: 'iter>( - &'trans self, - q: EdgeQuery, - ) -> Result> + 'iter>> { - match q { - EdgeQuery::Specific(q) => { - let edge_manager = EdgeManager::new(&self.holder); - - let edges = q.keys.into_iter().map(move |key| { - match edge_manager.get(key.outbound_id, &key.t, key.inbound_id)? { - Some(update_datetime) => { - Ok(Some((key.outbound_id, key.t.clone(), update_datetime, key.inbound_id))) - } - None => Ok(None), - } - }); - - let iterator = remove_nones_from_iterator(edges); - Ok(Box::new(iterator)) - } - EdgeQuery::Pipe(q) => { - let vertex_iterator = self.vertex_query_to_iterator(*q.inner)?; - - let edge_range_manager = match q.direction { - EdgeDirection::Outbound => EdgeRangeManager::new(&self.holder), - EdgeDirection::Inbound => EdgeRangeManager::new_reversed(&self.holder), - }; - - // Ideally we'd use iterators all the way down, but things - // start breaking apart due to conditional expressions not - // returning the same type signature, issues with `Result`s - // and some of the iterators, etc. So at this point, we'll - // just resort to building a vector. - let mut edges: Vec> = Vec::new(); - - for item in vertex_iterator { - let (id, _) = item?; - let edge_iterator = edge_range_manager.iterate_for_range(id, q.t.as_ref(), q.high)?; - - for item in edge_iterator { - match item { - Ok(( - edge_range_first_id, - edge_range_t, - edge_range_update_datetime, - edge_range_second_id, - )) => { - if let Some(low) = q.low { - if edge_range_update_datetime < low { - break; - } - } - - edges.push(match q.direction { - EdgeDirection::Outbound => Ok(( - edge_range_first_id, - edge_range_t, - edge_range_update_datetime, - edge_range_second_id, - )), - EdgeDirection::Inbound => Ok(( - edge_range_second_id, - edge_range_t, - edge_range_update_datetime, - edge_range_first_id, - )), - }) - } - Err(_) => edges.push(item), - } - - if edges.len() == q.limit as usize { - break; - } - } - } - - Ok(Box::new(edges.into_iter())) - } - } - } } -impl<'a> Transaction<'a> for SledTransactionFour { +impl<'a> Transaction<'a> for SledTransaction { fn vertex_count(&self) -> u64 { let vertex_manager = VertexManager::new(&self.holder); - let iterator = vertex_manager.iterate_for_range(Uuid::default()); - iterator.count() as u64 + vertex_manager.count().into() } fn all_vertices(&'a self) -> Result> { @@ -305,184 +135,115 @@ impl<'a> Transaction<'a> for SledTransactionFour { fn range_vertices(&'a self, offset: Uuid) -> Result> { let vertex_manager = VertexManager::new(&self.holder); let iter = vertex_manager.iterate_for_range(offset); - Ok(iter.into()) + iter.into() } fn specific_vertices(&'a self, ids: Vec) -> Result> { let vertex_manager = VertexManager::new(&self.holder); - let iter = ids.into_iter().filter_map(move |id| vertex_manager.get(id)?) - .map(|a|); - Ok(iter) + let iter = ids.into_iter().filter_map(move |id| vertex_manager.get(id)?.map(|t| (id, t))) + .map(|(id, t)| Vertex { id, t }); + iter.into() } fn vertex_ids_with_property(&'a self, name: Identifier) -> Result>> { - let vertex_manager = VertexManager::new(&self.holder); - let iter = ids.into_iter().map(move |id| match vertex_manager.get(id)? { - Some(value) => Ok(Some((id, value))), - None => Ok(None), - }); + let prop_manager = VertexPropertyManager::new(&self.holder.vertex_properties); + let iter = self.all_vertices()? + .filter(|v| + { + let property_result = v.as_ref().map(|v| prop_manager.get(v.id, name.as_str())? + .is_some()).unwrap_or_default(); + property_result + }); - Ok(Box::new(remove_nones_from_iterator(iter))) + iter.into() } fn vertex_ids_with_property_value(&'a self, name: Identifier, value: &Json) -> Result>> { - todo!() + let prop_manager = VertexPropertyManager::new(&self.holder.vertex_properties); + let iter = self.all_vertices()? + .filter(|v| + { + let property_result = v.as_ref().map(|v| + prop_manager.get(v.id, name.as_str())? + .map(|i| i == value.into()) + .unwrap_or_default())?; + property_result + }); + + iter.into() } fn edge_count(&self) -> u64 { - todo!() + let edge_manager = EdgeManager::new(&self.holder); + edge_manager.count().into() } fn all_edges(&'a self) -> Result> { - todo!() + let range_manager = EdgeRangeManager::new(&self.holder); + range_manager.iterate_for_range(Uuid::default(), None, None).into() } fn range_edges(&'a self, offset: Edge) -> Result> { - todo!() + let range_manager = EdgeRangeManager::new(&self.holder); + range_manager.iterate_for_range(offset.inbound_id, Some(&offset.t), None).into() } fn range_reversed_edges(&'a self, offset: Edge) -> Result> { - todo!() + let range_manager = EdgeRangeManager::new_reversed(&self.holder); + range_manager.iterate_for_range(offset.inbound_id, Some(&offset.t), None).into() } fn specific_edges(&'a self, edges: Vec) -> Result> { - todo!() + let edge_manager = EdgeManager::new(&self.holder); + edges.into_iter().map(|e| edge_manager.get(e.outbound_id, &e.t, e.inbound_id)).into() } fn edges_with_property(&'a self, name: Identifier) -> Result>> { - todo!() + let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); + self.all_edges()?.filter(|r| { + let has_property = r.as_ref().map(|e| edge_property_manager.get(e.outbound_id, &e.t, e.inbound_id, name.as_str())?.is_some())?; + has_property + }).into() } fn edges_with_property_value(&'a self, name: Identifier, value: &Json) -> Result>> { - todo!() + let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); + self.all_edges()?.filter(|r| { + let has_property = r.as_ref().map(|e| edge_property_manager.get(e.outbound_id, &e.t, e.inbound_id, name.as_str())? == value.into())?; + has_property + }).into() } fn vertex_property(&self, vertex: &Vertex, name: Identifier) -> Result> { - todo!() + let vertex_property_manager = VertexPropertyManager::new(&self.holder.vertex_properties); + vertex_property_manager.get(vertex.id, &name).into() } fn all_vertex_properties_for_vertex(&'a self, vertex: &Vertex) -> Result> { - todo!() + let vertex_property_manager = VertexPropertyManager::new(&self.holder.vertex_properties); + vertex_property_manager.iterate_for_owner(vertex.id) } fn edge_property(&self, edge: &Edge, name: Identifier) -> Result> { - todo!() + let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); + edge_property_manager.get(edge.outbound_id, &edge.t, edge.inbound_id, name.as_str()).into() } fn all_edge_properties_for_edge(&'a self, edge: &Edge) -> Result> { - todo!() + let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); + edge_property_manager.iterate_for_owner(edge.outbound_id, &edge.t, edge.inbound_id).into() } fn delete_vertices(&mut self, vertices: Vec) -> Result<()> { - todo!() - } - - fn delete_edges(&mut self, edges: Vec) -> Result<()> { - todo!() - } - - fn delete_vertex_properties(&mut self, props: Vec<(Uuid, Identifier)>) -> Result<()> { - todo!() - } - - fn delete_edge_properties(&mut self, props: Vec<(Edge, Identifier)>) -> Result<()> { - todo!() - } - - fn create_vertex(&mut self, vertex: &Vertex) -> Result { - todo!() - } - - fn create_edge(&mut self, edge: &Edge) -> Result { - todo!() - } - - fn index_property(&mut self, name: Identifier) -> Result<()> { - todo!() - } - - fn set_vertex_properties(&mut self, vertices: Vec, name: Identifier, value: &Json) -> Result<()> { - todo!() - } - - fn set_edge_properties(&mut self, edges: Vec, name: Identifier, value: &Json) -> Result<()> { - todo!() - } -} - -impl Transaction for SledTransaction { - fn create_vertex(&self, vertex: &Vertex) -> Result { - let vertex_manager = VertexManager::new(&self.holder); - - if vertex_manager.exists(vertex.id)? { - Ok(false) - } else { - vertex_manager.create(vertex)?; - Ok(true) - } - } - - fn get_vertices>(&self, q: Q) -> Result> { - let iterator = self.vertex_query_to_iterator(q.into())?; - - let mapped = iterator.map(move |item| { - let (id, t) = item?; - let vertex = Vertex::with_id(id, t); - Ok(vertex) - }); - - mapped.collect() - } - - fn delete_vertices>(&self, q: Q) -> Result<()> { - let iterator = self.vertex_query_to_iterator(q.into())?; let vertex_manager = VertexManager::new(&self.holder); - - for item in iterator { - let (id, _) = item?; - vertex_manager.delete(id)?; - } - - Ok(()) + vertices.iter().for_each(|v| vertex_manager.delete(v.id)?).into() } - fn vertex_count(&self) -> Result { - let vertex_manager = VertexManager::new(&self.holder); - let iterator = vertex_manager.iterate_for_range(Uuid::default()); - Ok(iterator.count() as u64) - } - - fn create_edge(&self, key: &EdgeKey) -> Result { - let vertex_manager = VertexManager::new(&self.holder); - - if !vertex_manager.exists(key.outbound_id)? || !vertex_manager.exists(key.inbound_id)? { - Ok(false) - } else { - let edge_manager = EdgeManager::new(&self.holder); - edge_manager.set(key.outbound_id, &key.t, key.inbound_id, Utc::now())?; - Ok(true) - } - } - - fn get_edges>(&self, q: Q) -> Result> { - let iterator = self.edge_query_to_iterator(q.into())?; - - let mapped = iterator.map(move |item: Result| { - let (outbound_id, t, update_datetime, inbound_id) = item?; - let key = EdgeKey::new(outbound_id, t, inbound_id); - let edge = Edge::new(key, update_datetime); - Ok(edge) - }); - - mapped.collect() - } - - fn delete_edges>(&self, q: Q) -> Result<()> { + fn delete_edges(&mut self, edges: Vec) -> Result<()> { let edge_manager = EdgeManager::new(&self.holder); let vertex_manager = VertexManager::new(&self.holder); - let iterator = self.edge_query_to_iterator(q.into())?; - for item in iterator { + for item in edges.iter() { let (outbound_id, t, update_datetime, inbound_id) = item?; if vertex_manager.get(outbound_id)?.is_some() { @@ -492,133 +253,62 @@ impl Transaction for SledTransaction { Ok(()) } - fn get_edge_count(&self, id: Uuid, t: Option<&Type>, direction: EdgeDirection) -> Result { - let edge_range_manager = match direction { - EdgeDirection::Outbound => EdgeRangeManager::new(&self.holder), - EdgeDirection::Inbound => EdgeRangeManager::new_reversed(&self.holder), - }; - - let iter = edge_range_manager.iterate_for_range(id, t, None)?; - let count = iter.count(); - - Ok(count as u64) - } - - fn get_vertex_properties(&self, q: VertexPropertyQuery) -> Result> { - let manager = VertexPropertyManager::new(&self.holder.vertex_properties); - let mut properties = Vec::new(); - - for item in self.vertex_query_to_iterator(q.inner)? { - let (id, _) = item?; - let value = manager.get(id, &q.name)?; - - if let Some(value) = value { - properties.push(VertexProperty::new(id, value)); - } - } - - Ok(properties) - } - - fn get_all_vertex_properties>(&self, q: Q) -> Result> { - let manager = VertexPropertyManager::new(&self.holder.vertex_properties); - let iterator = self.vertex_query_to_iterator(q.into())?; - - let iter = iterator.map(move |item| { - let (id, t) = item?; - let vertex = Vertex::with_id(id, t); - - let it = manager.iterate_for_owner(id)?; - let props: Result> = it.collect(); - let props_iter = props?.into_iter(); - let props = props_iter - .map(|((_, name), value)| NamedProperty::new(name, value)) - .collect(); - - Ok(VertexProperties::new(vertex, props)) - }); - - iter.collect() - } - - fn set_vertex_properties(&self, q: VertexPropertyQuery, value: &JsonValue) -> Result<()> { - let manager = VertexPropertyManager::new(&self.holder.vertex_properties); - - for item in self.vertex_query_to_iterator(q.inner)? { - let (id, _) = item?; - manager.set(id, &q.name, value)?; + fn delete_vertex_properties(&mut self, props: Vec<(Uuid, Identifier)>) -> Result<()> { + let vertex_properties = VertexPropertyManager::new(&self.holder.vertex_properties); + for (id, prop) in props { + vertex_properties.delete(id, prop.as_str())? } Ok(()) } - fn delete_vertex_properties(&self, q: VertexPropertyQuery) -> Result<()> { - let manager = VertexPropertyManager::new(&self.holder.vertex_properties); - - for item in self.vertex_query_to_iterator(q.inner)? { - let (id, _) = item?; - manager.delete(id, &q.name)?; + fn delete_edge_properties(&mut self, props: Vec<(Edge, Identifier)>) -> Result<()> { + let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); + for (edge, prop) in props { + edge_property_manager.delete(edge.outbound_id, &edge.t, edge.inbound_id, prop.as_str())?; } Ok(()) } - fn get_edge_properties(&self, q: EdgePropertyQuery) -> Result> { - let manager = EdgePropertyManager::new(&self.holder.edge_properties); - let mut properties = Vec::new(); + fn create_vertex(&mut self, vertex: &Vertex) -> Result { + let vertex_manager = VertexManager::new(&self.holder); + vertex_manager.create(vertex) + } - for item in self.edge_query_to_iterator(q.inner)? { - let (outbound_id, t, _, inbound_id) = item?; - let value = manager.get(outbound_id, &t, inbound_id, &q.name)?; + fn create_edge(&mut self, edge: &Edge) -> Result { + let vertex_manager = VertexManager::new(&self.holder); - if let Some(value) = value { - let key = EdgeKey::new(outbound_id, t, inbound_id); - properties.push(EdgeProperty::new(key, value)); - } + if !vertex_manager.exists(edge.outbound_id)? || !vertex_manager.exists(edge.inbound_id)? { + Ok(false) + } else { + let edge_manager = EdgeManager::new(&self.holder); + edge_manager.set(edge.outbound_id, &edge.t, edge.inbound_id, Utc::now())?; + Ok(true) } - - Ok(properties) } - fn get_all_edge_properties>(&self, q: Q) -> Result> { - let manager = EdgePropertyManager::new(&self.holder.edge_properties); - let iterator = self.edge_query_to_iterator(q.into())?; - - let iter = iterator.map(move |item| { - let (out_id, t, time, in_id) = item?; - let edge = Edge::new(EdgeKey::new(out_id, t.clone(), in_id), time); - let it = manager.iterate_for_owner(out_id, &t, in_id)?; - let props: Result> = it.collect(); - let props_iter = props?.into_iter(); - let props = props_iter - .map(|((_, _, _, name), value)| NamedProperty::new(name, value)) - .collect(); - - Ok(EdgeProperties::new(edge, props)) - }); - - iter.collect() + fn index_property(&mut self, _name: Identifier) -> Result<()> { + // oh boy we really want this i guess + Ok(()) } - fn set_edge_properties(&self, q: EdgePropertyQuery, value: &JsonValue) -> Result<()> { - let manager = EdgePropertyManager::new(&self.holder.edge_properties); - - for item in self.edge_query_to_iterator(q.inner)? { - let (outbound_id, t, _, inbound_id) = item?; - manager.set(outbound_id, &t, inbound_id, &q.name, value)?; + fn set_vertex_properties(&mut self, vertices: Vec, name: Identifier, value: &Json) -> Result<()> { + let vertex_property_manager = VertexPropertyManager::new(&self.holder.vertex_properties); + for v in vertices { + vertex_property_manager.set(v, name.as_str(), &value)?; } Ok(()) } - fn delete_edge_properties(&self, q: EdgePropertyQuery) -> Result<()> { - let manager = EdgePropertyManager::new(&self.holder.edge_properties); - - for item in self.edge_query_to_iterator(q.inner)? { - let (outbound_id, t, _, inbound_id) = item?; - manager.delete(outbound_id, &t, inbound_id, &q.name)?; + fn set_edge_properties(&mut self, edges: Vec, name: Identifier, value: &Json) -> Result<()> { + let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); + for edge in edges { + edge_property_manager.set(edge.outbound_id, &edge.t, edge.inbound_id, name.as_str(), &value)?; } Ok(()) } } + fn remove_nones_from_iterator(iter: I) -> impl Iterator> where I: Iterator>>, diff --git a/src/managers.rs b/src/managers.rs index 1d0d63f..f4b7e88 100644 --- a/src/managers.rs +++ b/src/managers.rs @@ -41,6 +41,10 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { } } + pub fn count(&self) -> u64 { + self.tree.iter().count() as u64 + } + fn key(&self, id: Uuid) -> Vec { util::build(&[util::Component::Uuid(id)]) } @@ -82,10 +86,13 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { self.iterate(iter) } - pub fn create(&self, vertex: &Vertex) -> Result<()> { + pub fn create(&self, vertex: &Vertex) -> Result { let key = self.key(vertex.id); + if self.tree.contains_key(&key) { + return Ok(false); + } map_err(self.tree.insert(&key, util::build(&[util::Component::Identifier(vertex.t.clone())])))?; - Ok(()) + Ok(true) } pub fn delete(&self, id: Uuid) -> Result<()> { @@ -142,11 +149,15 @@ pub struct EdgeManager<'db: 'tree, 'tree> { fn time_from_bytes(value_bytes: IVec) -> Result>> { let mut cursor = Cursor::new(value_bytes.deref()); + time_from_cursor(&mut cursor) +} +fn time_from_cursor(mut cursor: &mut Cursor) -> Result>> { let time: i64 = util::read_fixed_length_string(&mut cursor).parse()?; let time = DateTime::from_timestamp_micros(time); Ok(time) } + fn time_to_string(time: DateTime) -> String { time.timestamp_micros().to_string() } @@ -176,6 +187,10 @@ impl<'db, 'tree> EdgeManager<'db, 'tree> { } } + pub fn count(&self) -> u64 { + self.tree.iter().count() as u64 + } + pub fn set(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, new_update_datetime: DateTime) -> Result<()> { let edge_range_manager = EdgeRangeManager::new(self.holder); let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); @@ -236,8 +251,8 @@ impl<'tree> EdgeRangeManager<'tree> { fn key(&self, first_id: Uuid, t: &Identifier, update_datetime: DateTime, second_id: Uuid) -> Vec { util::build(&[ util::Component::Uuid(first_id), - util::Component::Identifier(t), - util::Component::DateTime(update_datetime), + util::Component::Identifier(t.clone()), + util::Component::FixedLengthString(&time_to_string(update_datetime)), util::Component::Uuid(second_id), ]) } @@ -248,8 +263,9 @@ impl<'tree> EdgeRangeManager<'tree> { let (k, _) = map_err(item)?; let mut cursor = Cursor::new(k); let first_id = util::read_uuid(&mut cursor); - let t = util::read_type(&mut cursor); - let update_datetime = util::read_datetime(&mut cursor); + let t = util::read_identifier(&mut cursor); + + let update_datetime = time_from_cursor(&mut cursor)?.unwrap_or_default(); let second_id = util::read_uuid(&mut cursor); Ok((first_id, t, update_datetime, second_id)) }) @@ -258,17 +274,17 @@ impl<'tree> EdgeRangeManager<'tree> { pub fn iterate_for_range<'iter, 'trans: 'iter>( &'trans self, id: Uuid, - t: Option<&Type>, + t: Option<&Identifier>, high: Option>, ) -> Result> + 'iter>> { match t { Some(t) => { - let high = high.unwrap_or_else(|| *util::MAX_DATETIME); - let prefix = util::build(&[util::Component::Uuid(id), util::Component::Type(t)]); + let high: DateTime = high.unwrap_or_else(|| DateTime::::MAX_UTC); + let prefix = util::build(&[util::Component::Uuid(id), util::Component::Identifier(t.clone())]); let low_key = util::build(&[ util::Component::Uuid(id), - util::Component::Type(t), - util::Component::DateTime(high), + util::Component::Identifier(t.clone()), + util::Component::FixedLengthString(&time_to_string(high)), ]); let low_key_bytes: &[u8] = low_key.as_ref(); let iterator = self.tree.range(low_key_bytes..); @@ -309,13 +325,13 @@ impl<'tree> EdgeRangeManager<'tree> { self.iterate(iterator, prefix) } - pub fn set(&self, first_id: Uuid, t: &Type, update_datetime: DateTime, second_id: Uuid) -> Result<()> { + pub fn set(&self, first_id: Uuid, t: &Identifier, update_datetime: DateTime, second_id: Uuid) -> Result<()> { let key = self.key(first_id, t, update_datetime, second_id); map_err(self.tree.insert(&key, &[]))?; Ok(()) } - pub fn delete(&self, first_id: Uuid, t: &Type, update_datetime: DateTime, second_id: Uuid) -> Result<()> { + pub fn delete(&self, first_id: Uuid, t: &Identifier, update_datetime: DateTime, second_id: Uuid) -> Result<()> { map_err(self.tree.remove(&self.key(first_id, t, update_datetime, second_id)))?; Ok(()) } @@ -383,10 +399,10 @@ impl<'tree> EdgePropertyManager<'tree> { EdgePropertyManager { tree } } - fn key(&self, outbound_id: Uuid, t: &Type, inbound_id: Uuid, name: &str) -> Vec { + fn key(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str) -> Vec { util::build(&[ util::Component::Uuid(outbound_id), - util::Component::Type(t), + util::Component::Identifier(t.clone()), util::Component::Uuid(inbound_id), util::Component::FixedLengthString(name), ]) @@ -395,12 +411,12 @@ impl<'tree> EdgePropertyManager<'tree> { pub fn iterate_for_owner<'a>( &'a self, outbound_id: Uuid, - t: &'a Type, + t: &'a Identifier, inbound_id: Uuid, ) -> Result> + 'a>> { let prefix = util::build(&[ util::Component::Uuid(outbound_id), - util::Component::Type(t), + util::Component::Identifier(t.clone()), util::Component::Uuid(inbound_id), ]); @@ -413,7 +429,7 @@ impl<'tree> EdgePropertyManager<'tree> { let edge_property_outbound_id = util::read_uuid(&mut cursor); debug_assert_eq!(edge_property_outbound_id, outbound_id); - let edge_property_t = util::read_type(&mut cursor); + let edge_property_t = util::read_identifier(&mut cursor); debug_assert_eq!(&edge_property_t, t); let edge_property_inbound_id = util::read_uuid(&mut cursor); @@ -436,7 +452,7 @@ impl<'tree> EdgePropertyManager<'tree> { Ok(Box::new(mapped)) } - pub fn get(&self, outbound_id: Uuid, t: &Type, inbound_id: Uuid, name: &str) -> Result> { + pub fn get(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str) -> Result> { let key = self.key(outbound_id, t, inbound_id, name); match map_err(self.tree.get(&key))? { @@ -445,14 +461,14 @@ impl<'tree> EdgePropertyManager<'tree> { } } - pub fn set(&self, outbound_id: Uuid, t: &Type, inbound_id: Uuid, name: &str, value: &JsonValue) -> Result<()> { + pub fn set(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str, value: &JsonValue) -> Result<()> { let key = self.key(outbound_id, t, inbound_id, name); let value_json = serde_json::to_vec(value)?; map_err(self.tree.insert(key.as_slice(), value_json.as_slice()))?; Ok(()) } - pub fn delete(&self, outbound_id: Uuid, t: &Type, inbound_id: Uuid, name: &str) -> Result<()> { + pub fn delete(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str) -> Result<()> { map_err(self.tree.remove(&self.key(outbound_id, t, inbound_id, name)))?; Ok(()) } From 10d1c22308be40c0c65eebbd91ea9588d2121edd Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Fri, 5 Jul 2024 17:12:32 +0200 Subject: [PATCH 03/28] Fixed some issues with type conversion and error handling. Lifetime questions remain open. --- src/datastore.rs | 131 ++++++++++++++++++++++++++++++++--------------- src/managers.rs | 6 +-- 2 files changed, 92 insertions(+), 45 deletions(-) diff --git a/src/datastore.rs b/src/datastore.rs index a25b64f..b5f3f5c 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use std::u64; use chrono::offset::Utc; -use indradb::{Datastore, DynIter, Edge, Identifier, Json, Result, Transaction, Vertex}; +use indradb::{Datastore, DynIter, Edge, Error, Identifier, Json, Result, Transaction, Vertex}; use sled::{Config, Db, Tree}; use uuid::Uuid; @@ -126,23 +126,25 @@ impl<'a> Transaction<'a> for SledTransaction { let mapped = iterator.map(move |item| { let (id, t) = item?; let vertex = Vertex::with_id(id, t); - Ok(vertex) + Ok::(vertex) }); - Ok(mapped.into()) + Ok(Box::new(mapped)) } fn range_vertices(&'a self, offset: Uuid) -> Result> { let vertex_manager = VertexManager::new(&self.holder); - let iter = vertex_manager.iterate_for_range(offset); - iter.into() + let iter = vertex_manager.iterate_for_range(offset).map(|e| e.map(|v| Vertex::with_id(v.0, v.1))); + Ok(Box::new(iter)) } fn specific_vertices(&'a self, ids: Vec) -> Result> { let vertex_manager = VertexManager::new(&self.holder); - let iter = ids.into_iter().filter_map(move |id| vertex_manager.get(id)?.map(|t| (id, t))) - .map(|(id, t)| Vertex { id, t }); - iter.into() + let iter = ids.into_iter().filter_map(move |id| { + let v = vertex_manager.get(id).transpose(); + v.map(|v| v.map(|v| Vertex::with_id(id, v))) + }); + Ok(Box::new(iter)) } fn vertex_ids_with_property(&'a self, name: Identifier) -> Result>> { @@ -150,12 +152,14 @@ impl<'a> Transaction<'a> for SledTransaction { let iter = self.all_vertices()? .filter(|v| { - let property_result = v.as_ref().map(|v| prop_manager.get(v.id, name.as_str())? - .is_some()).unwrap_or_default(); - property_result - }); + let property_result = v.as_ref().and_then(|v| prop_manager.get(v.id, name.as_str()).as_ref()); + if let Ok(v) = property_result { + return v.is_some(); + } + return false; + }).map(|v| v.map(|v| v.id)); - iter.into() + Ok(Some(Box::new(iter))) } fn vertex_ids_with_property_value(&'a self, name: Identifier, value: &Json) -> Result>> { @@ -163,14 +167,15 @@ impl<'a> Transaction<'a> for SledTransaction { let iter = self.all_vertices()? .filter(|v| { - let property_result = v.as_ref().map(|v| - prop_manager.get(v.id, name.as_str())? - .map(|i| i == value.into()) - .unwrap_or_default())?; - property_result - }); + let property_result = v.as_ref().and_then(|v| + prop_manager.get(v.id, name.as_str()).as_ref()); + if let Ok(Some(v)) = property_result { + return *v == *value.0; + } + false + }).map(|v| v.map(|v| v.id)); - iter.into() + Ok(Some(Box::new(iter))) } fn edge_count(&self) -> u64 { @@ -180,63 +185,104 @@ impl<'a> Transaction<'a> for SledTransaction { fn all_edges(&'a self) -> Result> { let range_manager = EdgeRangeManager::new(&self.holder); - range_manager.iterate_for_range(Uuid::default(), None, None).into() + let iter = range_manager.iterate_for_range(Uuid::default(), None, None).map(|i| i.map(|e| e.map(|(outbound_id, t, _time, inbound_id)| Edge { + outbound_id, + t, + inbound_id, + })))?; + Ok(Box::new(iter)) } fn range_edges(&'a self, offset: Edge) -> Result> { let range_manager = EdgeRangeManager::new(&self.holder); - range_manager.iterate_for_range(offset.inbound_id, Some(&offset.t), None).into() + let iter = range_manager.iterate_for_range(offset.inbound_id, Some(&offset.t), None)?; + let iter = iter.map(|r| r.map(|(outbound_id, t, _time, inbound_id)| Edge { + outbound_id, + t, + inbound_id, + })); + Ok(Box::new(iter)) } fn range_reversed_edges(&'a self, offset: Edge) -> Result> { let range_manager = EdgeRangeManager::new_reversed(&self.holder); - range_manager.iterate_for_range(offset.inbound_id, Some(&offset.t), None).into() + let iter = range_manager.iterate_for_range(offset.inbound_id, Some(&offset.t), None)?; + let iter = iter.map(|r| r.map(|(outbound_id, t, _time, inbound_id)| Edge { + outbound_id, + t, + inbound_id, + })); + Ok(Box::new(iter)) } fn specific_edges(&'a self, edges: Vec) -> Result> { let edge_manager = EdgeManager::new(&self.holder); - edges.into_iter().map(|e| edge_manager.get(e.outbound_id, &e.t, e.inbound_id)).into() + let iter = edges.into_iter().filter(|e| { + let r = edge_manager.get(e.outbound_id, &e.t, e.inbound_id).transpose(); + if let Some(Ok(_)) = r { + return true; + } + false + }).map(|e| Ok(e)); + Ok(Box::new(iter)) } fn edges_with_property(&'a self, name: Identifier) -> Result>> { let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); - self.all_edges()?.filter(|r| { - let has_property = r.as_ref().map(|e| edge_property_manager.get(e.outbound_id, &e.t, e.inbound_id, name.as_str())?.is_some())?; - has_property - }).into() + let iter = self.all_edges()?.filter(|r| { + let has_property = r.as_ref().and_then(|e| edge_property_manager.get(e.outbound_id, &e.t, e.inbound_id, name.as_str()).as_ref()); + if let Ok(prop) = has_property { + return prop.is_some(); + } + false + }); + Ok(Some(Box::new(iter))) } fn edges_with_property_value(&'a self, name: Identifier, value: &Json) -> Result>> { let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); - self.all_edges()?.filter(|r| { - let has_property = r.as_ref().map(|e| edge_property_manager.get(e.outbound_id, &e.t, e.inbound_id, name.as_str())? == value.into())?; - has_property - }).into() + let iter = self.all_edges()?.filter(|r| { + let has_property = r.as_ref().and_then(|e| edge_property_manager.get(e.outbound_id, &e.t, e.inbound_id, name.as_str()).as_ref()); + if let Ok(Some(v)) = has_property { + return *v == **value; + } + false + }); + Ok(Some(Box::new(iter))) } fn vertex_property(&self, vertex: &Vertex, name: Identifier) -> Result> { let vertex_property_manager = VertexPropertyManager::new(&self.holder.vertex_properties); - vertex_property_manager.get(vertex.id, &name).into() + let r = vertex_property_manager.get(vertex.id, &name)?; + Ok(r.map(|v| v.into())) } fn all_vertex_properties_for_vertex(&'a self, vertex: &Vertex) -> Result> { let vertex_property_manager = VertexPropertyManager::new(&self.holder.vertex_properties); - vertex_property_manager.iterate_for_owner(vertex.id) + let iter = vertex_property_manager.iterate_for_owner(vertex.id)?; + let iter = iter.map(|r| r.and_then(|((_, name), val)| Ok((Identifier::new(name)?, Json::new(val))))); + Ok(Box::new(iter)) } fn edge_property(&self, edge: &Edge, name: Identifier) -> Result> { let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); - edge_property_manager.get(edge.outbound_id, &edge.t, edge.inbound_id, name.as_str()).into() + let result = edge_property_manager.get(edge.outbound_id, &edge.t, edge.inbound_id, name.as_str())?; + Ok(result.map(|v| Json::new(v))) } fn all_edge_properties_for_edge(&'a self, edge: &Edge) -> Result> { let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); - edge_property_manager.iterate_for_owner(edge.outbound_id, &edge.t, edge.inbound_id).into() + let iter = edge_property_manager.iterate_for_owner(edge.outbound_id, &edge.t, edge.inbound_id)?; + let iter = iter.map(|e| e.map(|((_, id, _, _), val)| (id, Json::new(val)))); + Ok(Box::new(iter)) } fn delete_vertices(&mut self, vertices: Vec) -> Result<()> { let vertex_manager = VertexManager::new(&self.holder); - vertices.iter().for_each(|v| vertex_manager.delete(v.id)?).into() + for v in vertices { + vertex_manager.delete(v.id)? + } + Ok(()) } fn delete_edges(&mut self, edges: Vec) -> Result<()> { @@ -244,11 +290,12 @@ impl<'a> Transaction<'a> for SledTransaction { let vertex_manager = VertexManager::new(&self.holder); for item in edges.iter() { - let (outbound_id, t, update_datetime, inbound_id) = item?; - - if vertex_manager.get(outbound_id)?.is_some() { - edge_manager.delete(outbound_id, &t, inbound_id, update_datetime)?; - }; + let edge = edge_manager.get(item.outbound_id, &item.t, item.inbound_id)?; + if let Some(t) = edge { + if vertex_manager.get(item.outbound_id)?.is_some() { + edge_manager.delete(item.outbound_id, &item.t, item.inbound_id, t)?; + }; + } } Ok(()) } diff --git a/src/managers.rs b/src/managers.rs index f4b7e88..9280e74 100644 --- a/src/managers.rs +++ b/src/managers.rs @@ -88,7 +88,7 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { pub fn create(&self, vertex: &Vertex) -> Result { let key = self.key(vertex.id); - if self.tree.contains_key(&key) { + if map_err(self.tree.contains_key(&key))? { return Ok(false); } map_err(self.tree.insert(&key, util::build(&[util::Component::Identifier(vertex.t.clone())])))?; @@ -151,8 +151,8 @@ fn time_from_bytes(value_bytes: IVec) -> Result>> { let mut cursor = Cursor::new(value_bytes.deref()); time_from_cursor(&mut cursor) } -fn time_from_cursor(mut cursor: &mut Cursor) -> Result>> { - let time: i64 = util::read_fixed_length_string(&mut cursor).parse()?; +fn time_from_cursor>(mut cursor: &mut Cursor) -> Result>> { + let time: i64 = util::read_fixed_length_string(&mut cursor).parse().unwrap(); let time = DateTime::from_timestamp_micros(time); Ok(time) } From e70dc418abe4f367cff3ba03eb17077af26bb1ff Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Fri, 5 Jul 2024 23:15:19 +0200 Subject: [PATCH 04/28] Clean managers and prepare for indexed managers for values. --- src/datastore.rs | 148 ++++++++++++++++++++++------------------------- src/lib.rs | 6 +- src/managers.rs | 92 +++++++---------------------- 3 files changed, 93 insertions(+), 153 deletions(-) diff --git a/src/datastore.rs b/src/datastore.rs index b5f3f5c..95a8820 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -1,8 +1,6 @@ use std::path::Path; -use std::sync::Arc; use std::u64; -use chrono::offset::Utc; use indradb::{Datastore, DynIter, Edge, Error, Identifier, Json, Result, Transaction, Vertex}; use sled::{Config, Db, Tree}; use uuid::Uuid; @@ -32,19 +30,21 @@ impl SledConfig { /// Creates a new sled datastore. pub fn open>(self, path: P) -> Result { Ok(SledDatastore { - holder: Arc::new(SledHolder::new(path, self)?), + holder: SledHolder::new(path, self)?, }) } } /// The meat of a Sled datastore pub struct SledHolder { - pub(crate) db: Arc, // Derefs to Tree, holds the vertices + pub(crate) db: Db, // Derefs to Tree, holds the vertices pub(crate) edges: Tree, pub(crate) edge_ranges: Tree, pub(crate) reversed_edge_ranges: Tree, pub(crate) vertex_properties: Tree, pub(crate) edge_properties: Tree, + pub(crate) edge_property_values: Tree, + pub(crate) vertex_property_values: Tree, } impl<'ds> SledHolder { @@ -72,14 +72,16 @@ impl<'ds> SledHolder { reversed_edge_ranges: map_err(db.open_tree("reversed_edge_ranges"))?, vertex_properties: map_err(db.open_tree("vertex_properties"))?, edge_properties: map_err(db.open_tree("edge_properties"))?, - db: Arc::new(db), + vertex_property_values: map_err(db.open_tree("vertex_property_values"))?, + edge_property_values: map_err(db.open_tree("edge_property_values"))?, + db, }) } } /// A datastore that is backed by Sled. pub struct SledDatastore { - pub(crate) holder: Arc, + pub(crate) holder: SledHolder, } impl<'ds> SledDatastore { @@ -89,40 +91,50 @@ impl<'ds> SledDatastore { /// * `path`: The file path to the Sled database. pub fn new>(path: P) -> Result { Ok(SledDatastore { - holder: Arc::new(SledHolder::new(path, SledConfig::default())?), + holder: SledHolder::new(path, SledConfig::default())?, }) } } impl Datastore for SledDatastore { - type Transaction<'a> = SledTransaction; + type Transaction<'a> = SledTransaction<'a> + where + Self: 'a; + fn transaction(&self) -> Self::Transaction<'_> { - SledTransaction::new(self.holder.clone()) + SledTransaction { + holder: &self.holder, + vertex_manager: VertexManager::new(&self.holder), + edge_manager: EdgeManager::new(&self.holder), + edge_range_manager: EdgeRangeManager::new(&self.holder), + edge_range_manager_rev: EdgeRangeManager::new_reversed(&self.holder), + edge_property_manager: EdgePropertyManager::new(&self.holder.edge_properties), + vertex_property_manager: VertexPropertyManager::new(&self.holder.vertex_properties), + } } } /// A transaction that is backed by Sled. -pub struct SledTransaction { - holder: Arc, -} - -impl SledTransaction { - fn new(holder: Arc) -> Self { - SledTransaction { holder } - } +pub struct SledTransaction<'a> { + holder: &'a SledHolder, + vertex_manager: VertexManager<'a, 'a>, + edge_manager: EdgeManager<'a, 'a>, + edge_property_manager: EdgePropertyManager<'a>, + vertex_property_manager: VertexPropertyManager<'a>, + edge_range_manager: EdgeRangeManager<'a>, + edge_range_manager_rev: EdgeRangeManager<'a>, } -impl<'a> Transaction<'a> for SledTransaction { +impl<'a> Transaction<'a> for SledTransaction<'a> { fn vertex_count(&self) -> u64 { let vertex_manager = VertexManager::new(&self.holder); vertex_manager.count().into() } fn all_vertices(&'a self) -> Result> { - let vertex_manager = VertexManager::new(&self.holder); - let iterator = vertex_manager.iterate_for_range(Uuid::default()); + let iterator = self.vertex_manager.iterate_for_range(Uuid::default()); let mapped = iterator.map(move |item| { let (id, t) = item?; let vertex = Vertex::with_id(id, t); @@ -133,26 +145,23 @@ impl<'a> Transaction<'a> for SledTransaction { } fn range_vertices(&'a self, offset: Uuid) -> Result> { - let vertex_manager = VertexManager::new(&self.holder); - let iter = vertex_manager.iterate_for_range(offset).map(|e| e.map(|v| Vertex::with_id(v.0, v.1))); + let iter = self.vertex_manager.iterate_for_range(offset).map(|e| e.map(|v| Vertex::with_id(v.0, v.1))); Ok(Box::new(iter)) } fn specific_vertices(&'a self, ids: Vec) -> Result> { - let vertex_manager = VertexManager::new(&self.holder); let iter = ids.into_iter().filter_map(move |id| { - let v = vertex_manager.get(id).transpose(); + let v = self.vertex_manager.get(id).transpose(); v.map(|v| v.map(|v| Vertex::with_id(id, v))) }); Ok(Box::new(iter)) } fn vertex_ids_with_property(&'a self, name: Identifier) -> Result>> { - let prop_manager = VertexPropertyManager::new(&self.holder.vertex_properties); let iter = self.all_vertices()? .filter(|v| { - let property_result = v.as_ref().and_then(|v| prop_manager.get(v.id, name.as_str()).as_ref()); + let property_result = v.as_ref().and_then(|v| self.vertex_property_manager.get(v.id, name.as_str()).as_ref()); if let Ok(v) = property_result { return v.is_some(); } @@ -163,12 +172,11 @@ impl<'a> Transaction<'a> for SledTransaction { } fn vertex_ids_with_property_value(&'a self, name: Identifier, value: &Json) -> Result>> { - let prop_manager = VertexPropertyManager::new(&self.holder.vertex_properties); let iter = self.all_vertices()? .filter(|v| { let property_result = v.as_ref().and_then(|v| - prop_manager.get(v.id, name.as_str()).as_ref()); + self.vertex_property_manager.get(v.id, name.as_str()).as_ref()); if let Ok(Some(v)) = property_result { return *v == *value.0; } @@ -184,8 +192,7 @@ impl<'a> Transaction<'a> for SledTransaction { } fn all_edges(&'a self) -> Result> { - let range_manager = EdgeRangeManager::new(&self.holder); - let iter = range_manager.iterate_for_range(Uuid::default(), None, None).map(|i| i.map(|e| e.map(|(outbound_id, t, _time, inbound_id)| Edge { + let iter = self.edge_range_manager.iterate_for_range(Uuid::default(), None).map(|i| i.map(|e| e.map(|(outbound_id, t, inbound_id)| Edge { outbound_id, t, inbound_id, @@ -194,9 +201,8 @@ impl<'a> Transaction<'a> for SledTransaction { } fn range_edges(&'a self, offset: Edge) -> Result> { - let range_manager = EdgeRangeManager::new(&self.holder); - let iter = range_manager.iterate_for_range(offset.inbound_id, Some(&offset.t), None)?; - let iter = iter.map(|r| r.map(|(outbound_id, t, _time, inbound_id)| Edge { + let iter = self.edge_range_manager.iterate_for_range(offset.inbound_id, Some(&offset.t))?; + let iter = iter.map(|r| r.map(|(outbound_id, t, inbound_id)| Edge { outbound_id, t, inbound_id, @@ -205,9 +211,8 @@ impl<'a> Transaction<'a> for SledTransaction { } fn range_reversed_edges(&'a self, offset: Edge) -> Result> { - let range_manager = EdgeRangeManager::new_reversed(&self.holder); - let iter = range_manager.iterate_for_range(offset.inbound_id, Some(&offset.t), None)?; - let iter = iter.map(|r| r.map(|(outbound_id, t, _time, inbound_id)| Edge { + let iter = self.edge_range_manager_rev.iterate_for_range(offset.inbound_id, Some(&offset.t))?; + let iter = iter.map(|r| r.map(|(outbound_id, t, inbound_id)| Edge { outbound_id, t, inbound_id, @@ -216,21 +221,20 @@ impl<'a> Transaction<'a> for SledTransaction { } fn specific_edges(&'a self, edges: Vec) -> Result> { - let edge_manager = EdgeManager::new(&self.holder); - let iter = edges.into_iter().filter(|e| { - let r = edge_manager.get(e.outbound_id, &e.t, e.inbound_id).transpose(); - if let Some(Ok(_)) = r { - return true; + let iter: Vec<_> = edges.into_iter().filter(|e| { + let r = self.edge_range_manager.contains(e); + if let Ok(r) = r { + r + } else { + false } - false - }).map(|e| Ok(e)); - Ok(Box::new(iter)) + }).map(|e| Ok(e)).collect(); + Ok(Box::new(iter.into_iter())) } fn edges_with_property(&'a self, name: Identifier) -> Result>> { - let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); let iter = self.all_edges()?.filter(|r| { - let has_property = r.as_ref().and_then(|e| edge_property_manager.get(e.outbound_id, &e.t, e.inbound_id, name.as_str()).as_ref()); + let has_property = r.as_ref().and_then(|e| self.edge_property_manager.get(e.outbound_id, &e.t, e.inbound_id, name.as_str()).as_ref()); if let Ok(prop) = has_property { return prop.is_some(); } @@ -240,9 +244,8 @@ impl<'a> Transaction<'a> for SledTransaction { } fn edges_with_property_value(&'a self, name: Identifier, value: &Json) -> Result>> { - let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); let iter = self.all_edges()?.filter(|r| { - let has_property = r.as_ref().and_then(|e| edge_property_manager.get(e.outbound_id, &e.t, e.inbound_id, name.as_str()).as_ref()); + let has_property = r.as_ref().and_then(|e| self.edge_property_manager.get(e.outbound_id, &e.t, e.inbound_id, name.as_str()).as_ref()); if let Ok(Some(v)) = has_property { return *v == **value; } @@ -252,51 +255,41 @@ impl<'a> Transaction<'a> for SledTransaction { } fn vertex_property(&self, vertex: &Vertex, name: Identifier) -> Result> { - let vertex_property_manager = VertexPropertyManager::new(&self.holder.vertex_properties); - let r = vertex_property_manager.get(vertex.id, &name)?; + let r = self.vertex_property_manager.get(vertex.id, &name)?; Ok(r.map(|v| v.into())) } fn all_vertex_properties_for_vertex(&'a self, vertex: &Vertex) -> Result> { - let vertex_property_manager = VertexPropertyManager::new(&self.holder.vertex_properties); - let iter = vertex_property_manager.iterate_for_owner(vertex.id)?; + let iter = self.vertex_property_manager.iterate_for_owner(vertex.id)?; let iter = iter.map(|r| r.and_then(|((_, name), val)| Ok((Identifier::new(name)?, Json::new(val))))); Ok(Box::new(iter)) } fn edge_property(&self, edge: &Edge, name: Identifier) -> Result> { - let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); - let result = edge_property_manager.get(edge.outbound_id, &edge.t, edge.inbound_id, name.as_str())?; + let result = self.edge_property_manager.get(edge.outbound_id, &edge.t, edge.inbound_id, name.as_str())?; Ok(result.map(|v| Json::new(v))) } fn all_edge_properties_for_edge(&'a self, edge: &Edge) -> Result> { - let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); - let iter = edge_property_manager.iterate_for_owner(edge.outbound_id, &edge.t, edge.inbound_id)?; - let iter = iter.map(|e| e.map(|((_, id, _, _), val)| (id, Json::new(val)))); + let iter: Vec<_> = self.edge_property_manager.iterate_for_owner(edge.outbound_id, &edge.t, edge.inbound_id)?.collect(); + let iter = iter.into_iter().map(|e| e.map(|((_, id, _, _), val)| (id, Json::new(val)))); Ok(Box::new(iter)) } fn delete_vertices(&mut self, vertices: Vec) -> Result<()> { - let vertex_manager = VertexManager::new(&self.holder); for v in vertices { - vertex_manager.delete(v.id)? + self.vertex_manager.delete(v.id)? } Ok(()) } fn delete_edges(&mut self, edges: Vec) -> Result<()> { - let edge_manager = EdgeManager::new(&self.holder); - let vertex_manager = VertexManager::new(&self.holder); - for item in edges.iter() { - let edge = edge_manager.get(item.outbound_id, &item.t, item.inbound_id)?; - if let Some(t) = edge { - if vertex_manager.get(item.outbound_id)?.is_some() { - edge_manager.delete(item.outbound_id, &item.t, item.inbound_id, t)?; - }; - } + if self.vertex_manager.get(item.outbound_id)?.is_some() { + self.edge_manager.delete(item.outbound_id, &item.t, item.inbound_id)?; + }; } + Ok(()) } @@ -309,26 +302,25 @@ impl<'a> Transaction<'a> for SledTransaction { } fn delete_edge_properties(&mut self, props: Vec<(Edge, Identifier)>) -> Result<()> { - let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); for (edge, prop) in props { - edge_property_manager.delete(edge.outbound_id, &edge.t, edge.inbound_id, prop.as_str())?; + self.edge_property_manager.delete(edge.outbound_id, &edge.t, edge.inbound_id, prop.as_str())?; } Ok(()) } fn create_vertex(&mut self, vertex: &Vertex) -> Result { - let vertex_manager = VertexManager::new(&self.holder); - vertex_manager.create(vertex) + self.vertex_manager.create(vertex) } fn create_edge(&mut self, edge: &Edge) -> Result { - let vertex_manager = VertexManager::new(&self.holder); + let outbound_exists = self.vertex_manager.exists(edge.outbound_id)?; + let inbound_exists = self.vertex_manager.exists(edge.inbound_id)?; - if !vertex_manager.exists(edge.outbound_id)? || !vertex_manager.exists(edge.inbound_id)? { + if !outbound_exists || !inbound_exists { Ok(false) } else { let edge_manager = EdgeManager::new(&self.holder); - edge_manager.set(edge.outbound_id, &edge.t, edge.inbound_id, Utc::now())?; + edge_manager.set(edge.outbound_id, &edge.t, edge.inbound_id)?; Ok(true) } } @@ -339,17 +331,15 @@ impl<'a> Transaction<'a> for SledTransaction { } fn set_vertex_properties(&mut self, vertices: Vec, name: Identifier, value: &Json) -> Result<()> { - let vertex_property_manager = VertexPropertyManager::new(&self.holder.vertex_properties); for v in vertices { - vertex_property_manager.set(v, name.as_str(), &value)?; + self.vertex_property_manager.set(v, name.as_str(), &value)?; } Ok(()) } fn set_edge_properties(&mut self, edges: Vec, name: Identifier, value: &Json) -> Result<()> { - let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); for edge in edges { - edge_property_manager.set(edge.outbound_id, &edge.t, edge.inbound_id, name.as_str(), &value)?; + self.edge_property_manager.set(edge.outbound_id, &edge.t, edge.inbound_id, name.as_str(), &value)?; } Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 11f4543..fa6ef33 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,25 +3,23 @@ #![cfg_attr(feature = "bench-suite", feature(test))] extern crate chrono; - #[cfg(any(feature = "bench-suite", feature = "test-suite"))] #[macro_use] extern crate indradb; #[cfg(not(any(feature = "bench-suite", feature = "test-suite")))] extern crate indradb; - extern crate serde_json; extern crate sled; #[cfg(any(feature = "bench-suite", feature = "test-suite"))] extern crate tempfile; extern crate uuid; +pub use self::datastore::{SledConfig, SledDatastore, SledTransaction}; + mod datastore; mod errors; mod managers; -pub use self::datastore::{SledConfig, SledDatastore, SledTransaction}; - mod normal_config { #[cfg(feature = "bench-suite")] full_bench_impl!({ diff --git a/src/managers.rs b/src/managers.rs index 9280e74..9d5844f 100644 --- a/src/managers.rs +++ b/src/managers.rs @@ -2,9 +2,7 @@ use std::io::Cursor; use std::ops::Deref; use std::u8; -use chrono::DateTime; -use chrono::offset::Utc; -use indradb::{Identifier, Result, util, Vertex}; +use indradb::{Edge, Identifier, Result, util, Vertex}; use serde_json::Value as JsonValue; use sled::{Iter as DbIterator, IVec, Tree}; use sled::Result as SledResult; @@ -16,7 +14,7 @@ use super::errors::map_err; pub type OwnedPropertyItem = ((Uuid, String), JsonValue); pub type VertexItem = (Uuid, Identifier); -pub type EdgeRangeItem = (Uuid, Identifier, DateTime, Uuid); +pub type EdgeRangeItem = (Uuid, Identifier, Uuid); pub type EdgePropertyItem = ((Uuid, Identifier, Uuid, String), JsonValue); fn take_while_prefixed(iterator: DbIterator, prefix: Vec) -> impl Iterator> { @@ -109,13 +107,12 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { { let edge_range_manager = EdgeRangeManager::new(self.holder); for item in edge_range_manager.iterate_for_owner(id) { - let (edge_range_outbound_id, edge_range_t, edge_range_update_datetime, edge_range_inbound_id) = item?; + let (edge_range_outbound_id, edge_range_t, edge_range_inbound_id) = item?; debug_assert_eq!(edge_range_outbound_id, id); edge_manager.delete( edge_range_outbound_id, &edge_range_t, edge_range_inbound_id, - edge_range_update_datetime, )?; } } @@ -126,7 +123,6 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { let ( reversed_edge_range_inbound_id, reversed_edge_range_t, - reversed_edge_range_update_datetime, reversed_edge_range_outbound_id, ) = item?; debug_assert_eq!(reversed_edge_range_inbound_id, id); @@ -134,7 +130,6 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { reversed_edge_range_outbound_id, &reversed_edge_range_t, reversed_edge_range_inbound_id, - reversed_edge_range_update_datetime, )?; } } @@ -147,20 +142,6 @@ pub struct EdgeManager<'db: 'tree, 'tree> { pub tree: &'tree Tree, } -fn time_from_bytes(value_bytes: IVec) -> Result>> { - let mut cursor = Cursor::new(value_bytes.deref()); - time_from_cursor(&mut cursor) -} -fn time_from_cursor>(mut cursor: &mut Cursor) -> Result>> { - let time: i64 = util::read_fixed_length_string(&mut cursor).parse().unwrap(); - let time = DateTime::from_timestamp_micros(time); - Ok(time) -} - - -fn time_to_string(time: DateTime) -> String { - time.timestamp_micros().to_string() -} impl<'db, 'tree> EdgeManager<'db, 'tree> { pub fn new(ds: &'db SledHolder) -> Self { @@ -178,46 +159,33 @@ impl<'db, 'tree> EdgeManager<'db, 'tree> { ]) } - pub fn get(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid) -> Result>> { - match map_err(self.tree.get(self.key(outbound_id, t, inbound_id)))? { - Some(value_bytes) => { - time_from_bytes(value_bytes) - } - None => Ok(None), - } - } - pub fn count(&self) -> u64 { self.tree.iter().count() as u64 } - pub fn set(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, new_update_datetime: DateTime) -> Result<()> { + pub fn set(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid) -> Result<()> { let edge_range_manager = EdgeRangeManager::new(self.holder); let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); - if let Some(update_datetime) = self.get(outbound_id, t, inbound_id)? { - edge_range_manager.delete(outbound_id, t, update_datetime, inbound_id)?; - reversed_edge_range_manager.delete(inbound_id, t, update_datetime, outbound_id)?; - } let key = self.key(outbound_id, t, inbound_id); map_err( self.tree - .insert(key, util::build(&[util::Component::FixedLengthString(&time_to_string(new_update_datetime))])), + .insert(key, IVec::default()), )?; - edge_range_manager.set(outbound_id, t, new_update_datetime, inbound_id)?; - reversed_edge_range_manager.set(inbound_id, t, new_update_datetime, outbound_id)?; + edge_range_manager.set(outbound_id, t, inbound_id)?; + reversed_edge_range_manager.set(inbound_id, t, outbound_id)?; Ok(()) } - pub fn delete(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, update_datetime: DateTime) -> Result<()> { + pub fn delete(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid) -> Result<()> { map_err(self.tree.remove(&self.key(outbound_id, t, inbound_id)))?; let edge_range_manager = EdgeRangeManager::new(self.holder); - edge_range_manager.delete(outbound_id, t, update_datetime, inbound_id)?; + edge_range_manager.delete(outbound_id, t, inbound_id)?; let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); - reversed_edge_range_manager.delete(inbound_id, t, update_datetime, outbound_id)?; + reversed_edge_range_manager.delete(inbound_id, t, outbound_id)?; let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); for item in edge_property_manager.iterate_for_owner(outbound_id, t, inbound_id)? { @@ -248,15 +216,19 @@ impl<'tree> EdgeRangeManager<'tree> { } } - fn key(&self, first_id: Uuid, t: &Identifier, update_datetime: DateTime, second_id: Uuid) -> Vec { + fn key(&self, first_id: Uuid, t: &Identifier, second_id: Uuid) -> Vec { util::build(&[ util::Component::Uuid(first_id), util::Component::Identifier(t.clone()), - util::Component::FixedLengthString(&time_to_string(update_datetime)), util::Component::Uuid(second_id), ]) } + pub(crate) fn contains(&self, edge: &Edge) -> Result { + let key = self.key(edge.outbound_id, &edge.t, edge.inbound_id); + map_err(self.tree.contains_key(key)) + } + fn iterate<'it>(&self, iterator: DbIterator, prefix: Vec) -> impl Iterator> + 'it { let filtered = take_while_prefixed(iterator, prefix); filtered.map(move |item| -> Result { @@ -265,9 +237,8 @@ impl<'tree> EdgeRangeManager<'tree> { let first_id = util::read_uuid(&mut cursor); let t = util::read_identifier(&mut cursor); - let update_datetime = time_from_cursor(&mut cursor)?.unwrap_or_default(); let second_id = util::read_uuid(&mut cursor); - Ok((first_id, t, update_datetime, second_id)) + Ok((first_id, t, second_id)) }) } @@ -275,16 +246,13 @@ impl<'tree> EdgeRangeManager<'tree> { &'trans self, id: Uuid, t: Option<&Identifier>, - high: Option>, ) -> Result> + 'iter>> { match t { Some(t) => { - let high: DateTime = high.unwrap_or_else(|| DateTime::::MAX_UTC); let prefix = util::build(&[util::Component::Uuid(id), util::Component::Identifier(t.clone())]); let low_key = util::build(&[ util::Component::Uuid(id), util::Component::Identifier(t.clone()), - util::Component::FixedLengthString(&time_to_string(high)), ]); let low_key_bytes: &[u8] = low_key.as_ref(); let iterator = self.tree.range(low_key_bytes..); @@ -295,23 +263,7 @@ impl<'tree> EdgeRangeManager<'tree> { let prefix_bytes: &[u8] = prefix.as_ref(); let iterator = self.tree.range(prefix_bytes..); let mapped = self.iterate(iterator, prefix); - - if let Some(high) = high { - // We can filter out `update_datetime`s greater than - // `high` via key prefix filtering, so instead we handle - // it here - after the key has been deserialized. - let filtered = mapped.filter(move |item| { - if let Ok((_, _, update_datetime, _)) = *item { - update_datetime <= high - } else { - true - } - }); - - Ok(Box::new(filtered)) - } else { - Ok(Box::new(mapped)) - } + Ok(Box::new(mapped)) } } } @@ -325,14 +277,14 @@ impl<'tree> EdgeRangeManager<'tree> { self.iterate(iterator, prefix) } - pub fn set(&self, first_id: Uuid, t: &Identifier, update_datetime: DateTime, second_id: Uuid) -> Result<()> { - let key = self.key(first_id, t, update_datetime, second_id); + pub fn set(&self, first_id: Uuid, t: &Identifier, second_id: Uuid) -> Result<()> { + let key = self.key(first_id, t, second_id); map_err(self.tree.insert(&key, &[]))?; Ok(()) } - pub fn delete(&self, first_id: Uuid, t: &Identifier, update_datetime: DateTime, second_id: Uuid) -> Result<()> { - map_err(self.tree.remove(&self.key(first_id, t, update_datetime, second_id)))?; + pub fn delete(&self, first_id: Uuid, t: &Identifier, second_id: Uuid) -> Result<()> { + map_err(self.tree.remove(&self.key(first_id, t, second_id)))?; Ok(()) } } From 2ab75d0b211bf3a9e63b17582dd69afabae02f94 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Fri, 5 Jul 2024 23:21:37 +0200 Subject: [PATCH 05/28] Cut down managers into separate files --- src/datastore.rs | 8 +- src/managers.rs | 427 ------------------------ src/managers/edge_manager.rs | 74 ++++ src/managers/edge_property_manager.rs | 95 ++++++ src/managers/edge_range_manager.rs | 100 ++++++ src/managers/mod.rs | 21 ++ src/managers/vertex_manager.rs | 127 +++++++ src/managers/vertex_property_manager.rs | 64 ++++ 8 files changed, 488 insertions(+), 428 deletions(-) delete mode 100644 src/managers.rs create mode 100644 src/managers/edge_manager.rs create mode 100644 src/managers/edge_property_manager.rs create mode 100644 src/managers/edge_range_manager.rs create mode 100644 src/managers/mod.rs create mode 100644 src/managers/vertex_manager.rs create mode 100644 src/managers/vertex_property_manager.rs diff --git a/src/datastore.rs b/src/datastore.rs index 95a8820..359e1b2 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -5,8 +5,14 @@ use indradb::{Datastore, DynIter, Edge, Error, Identifier, Json, Result, Transac use sled::{Config, Db, Tree}; use uuid::Uuid; +use managers::edge_manager::EdgeManager; +use managers::edge_range_manager::EdgeRangeManager; +use managers::vertex_property_manager::VertexPropertyManager; + +use crate::managers::edge_property_manager::EdgePropertyManager; +use crate::managers::vertex_manager::VertexManager; + use super::errors::map_err; -use super::managers::*; #[derive(Copy, Clone, Default, Debug)] pub struct SledConfig { diff --git a/src/managers.rs b/src/managers.rs deleted file mode 100644 index 9d5844f..0000000 --- a/src/managers.rs +++ /dev/null @@ -1,427 +0,0 @@ -use std::io::Cursor; -use std::ops::Deref; -use std::u8; - -use indradb::{Edge, Identifier, Result, util, Vertex}; -use serde_json::Value as JsonValue; -use sled::{Iter as DbIterator, IVec, Tree}; -use sled::Result as SledResult; -use uuid::Uuid; - -use crate::datastore::SledHolder; - -use super::errors::map_err; - -pub type OwnedPropertyItem = ((Uuid, String), JsonValue); -pub type VertexItem = (Uuid, Identifier); -pub type EdgeRangeItem = (Uuid, Identifier, Uuid); -pub type EdgePropertyItem = ((Uuid, Identifier, Uuid, String), JsonValue); - -fn take_while_prefixed(iterator: DbIterator, prefix: Vec) -> impl Iterator> { - iterator.take_while(move |item| -> bool { - match item { - Ok((k, _)) => k.starts_with(&prefix), - Err(_) => false, - } - }) -} - -pub struct VertexManager<'db: 'tree, 'tree> { - pub holder: &'db SledHolder, - pub tree: &'tree Tree, -} - -impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { - pub fn new(ds: &'db SledHolder) -> Self { - VertexManager { - holder: ds, - tree: ds.db.deref(), - } - } - - pub fn count(&self) -> u64 { - self.tree.iter().count() as u64 - } - - fn key(&self, id: Uuid) -> Vec { - util::build(&[util::Component::Uuid(id)]) - } - - pub fn exists(&self, id: Uuid) -> Result { - Ok(map_err(self.tree.get(&self.key(id)))?.is_some()) - } - - pub fn get(&self, id: Uuid) -> Result> { - match map_err(self.tree.get(&self.key(id)))? { - Some(value_bytes) => { - let mut cursor = Cursor::new(value_bytes.deref()); - Ok(Some(util::read_identifier(&mut cursor))) - } - None => Ok(None), - } - } - - fn iterate(&self, iterator: DbIterator) -> impl Iterator> + '_ { - iterator.map(move |item| -> Result { - let (k, v) = map_err(item)?; - - let id = { - debug_assert_eq!(k.len(), 16); - let mut cursor = Cursor::new(k); - util::read_uuid(&mut cursor) - }; - - let mut cursor = Cursor::new(v); - let t = util::read_identifier(&mut cursor); - Ok((id, t)) - }) - } - - pub fn iterate_for_range(&self, id: Uuid) -> impl Iterator> + '_ { - let low_key = util::build(&[util::Component::Uuid(id)]); - let low_key_bytes: &[u8] = low_key.as_ref(); - let iter = self.tree.range(low_key_bytes..); - self.iterate(iter) - } - - pub fn create(&self, vertex: &Vertex) -> Result { - let key = self.key(vertex.id); - if map_err(self.tree.contains_key(&key))? { - return Ok(false); - } - map_err(self.tree.insert(&key, util::build(&[util::Component::Identifier(vertex.t.clone())])))?; - Ok(true) - } - - pub fn delete(&self, id: Uuid) -> Result<()> { - map_err(self.tree.remove(&self.key(id)))?; - - let vertex_property_manager = VertexPropertyManager::new(&self.holder.vertex_properties); - for item in vertex_property_manager.iterate_for_owner(id)? { - let ((vertex_property_owner_id, vertex_property_name), _) = item?; - vertex_property_manager.delete(vertex_property_owner_id, &vertex_property_name[..])?; - } - - let edge_manager = EdgeManager::new(self.holder); - - { - let edge_range_manager = EdgeRangeManager::new(self.holder); - for item in edge_range_manager.iterate_for_owner(id) { - let (edge_range_outbound_id, edge_range_t, edge_range_inbound_id) = item?; - debug_assert_eq!(edge_range_outbound_id, id); - edge_manager.delete( - edge_range_outbound_id, - &edge_range_t, - edge_range_inbound_id, - )?; - } - } - - { - let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); - for item in reversed_edge_range_manager.iterate_for_owner(id) { - let ( - reversed_edge_range_inbound_id, - reversed_edge_range_t, - reversed_edge_range_outbound_id, - ) = item?; - debug_assert_eq!(reversed_edge_range_inbound_id, id); - edge_manager.delete( - reversed_edge_range_outbound_id, - &reversed_edge_range_t, - reversed_edge_range_inbound_id, - )?; - } - } - Ok(()) - } -} - -pub struct EdgeManager<'db: 'tree, 'tree> { - pub holder: &'db SledHolder, - pub tree: &'tree Tree, -} - - -impl<'db, 'tree> EdgeManager<'db, 'tree> { - pub fn new(ds: &'db SledHolder) -> Self { - EdgeManager { - holder: ds, - tree: &ds.edges, - } - } - - fn key(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid) -> Vec { - util::build(&[ - util::Component::Uuid(outbound_id), - util::Component::Identifier(t.clone()), - util::Component::Uuid(inbound_id), - ]) - } - - pub fn count(&self) -> u64 { - self.tree.iter().count() as u64 - } - - pub fn set(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid) -> Result<()> { - let edge_range_manager = EdgeRangeManager::new(self.holder); - let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); - - - let key = self.key(outbound_id, t, inbound_id); - map_err( - self.tree - .insert(key, IVec::default()), - )?; - edge_range_manager.set(outbound_id, t, inbound_id)?; - reversed_edge_range_manager.set(inbound_id, t, outbound_id)?; - Ok(()) - } - - pub fn delete(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid) -> Result<()> { - map_err(self.tree.remove(&self.key(outbound_id, t, inbound_id)))?; - - let edge_range_manager = EdgeRangeManager::new(self.holder); - edge_range_manager.delete(outbound_id, t, inbound_id)?; - - let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); - reversed_edge_range_manager.delete(inbound_id, t, outbound_id)?; - - let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); - for item in edge_property_manager.iterate_for_owner(outbound_id, t, inbound_id)? { - let ((edge_property_outbound_id, edge_property_t, edge_property_inbound_id, edge_property_name), _) = item?; - edge_property_manager.delete( - edge_property_outbound_id, - &edge_property_t, - edge_property_inbound_id, - &edge_property_name[..], - )?; - } - Ok(()) - } -} - -pub struct EdgeRangeManager<'tree> { - pub tree: &'tree Tree, -} - -impl<'tree> EdgeRangeManager<'tree> { - pub fn new<'db: 'tree>(ds: &'db SledHolder) -> Self { - EdgeRangeManager { tree: &ds.edge_ranges } - } - - pub fn new_reversed<'db: 'tree>(ds: &'db SledHolder) -> Self { - EdgeRangeManager { - tree: &ds.reversed_edge_ranges, - } - } - - fn key(&self, first_id: Uuid, t: &Identifier, second_id: Uuid) -> Vec { - util::build(&[ - util::Component::Uuid(first_id), - util::Component::Identifier(t.clone()), - util::Component::Uuid(second_id), - ]) - } - - pub(crate) fn contains(&self, edge: &Edge) -> Result { - let key = self.key(edge.outbound_id, &edge.t, edge.inbound_id); - map_err(self.tree.contains_key(key)) - } - - fn iterate<'it>(&self, iterator: DbIterator, prefix: Vec) -> impl Iterator> + 'it { - let filtered = take_while_prefixed(iterator, prefix); - filtered.map(move |item| -> Result { - let (k, _) = map_err(item)?; - let mut cursor = Cursor::new(k); - let first_id = util::read_uuid(&mut cursor); - let t = util::read_identifier(&mut cursor); - - let second_id = util::read_uuid(&mut cursor); - Ok((first_id, t, second_id)) - }) - } - - pub fn iterate_for_range<'iter, 'trans: 'iter>( - &'trans self, - id: Uuid, - t: Option<&Identifier>, - ) -> Result> + 'iter>> { - match t { - Some(t) => { - let prefix = util::build(&[util::Component::Uuid(id), util::Component::Identifier(t.clone())]); - let low_key = util::build(&[ - util::Component::Uuid(id), - util::Component::Identifier(t.clone()), - ]); - let low_key_bytes: &[u8] = low_key.as_ref(); - let iterator = self.tree.range(low_key_bytes..); - Ok(Box::new(self.iterate(iterator, prefix))) - } - None => { - let prefix = util::build(&[util::Component::Uuid(id)]); - let prefix_bytes: &[u8] = prefix.as_ref(); - let iterator = self.tree.range(prefix_bytes..); - let mapped = self.iterate(iterator, prefix); - Ok(Box::new(mapped)) - } - } - } - - pub fn iterate_for_owner<'iter, 'trans: 'iter>( - &'trans self, - id: Uuid, - ) -> impl Iterator> + 'iter { - let prefix: Vec = util::build(&[util::Component::Uuid(id)]); - let iterator = self.tree.scan_prefix(&prefix); - self.iterate(iterator, prefix) - } - - pub fn set(&self, first_id: Uuid, t: &Identifier, second_id: Uuid) -> Result<()> { - let key = self.key(first_id, t, second_id); - map_err(self.tree.insert(&key, &[]))?; - Ok(()) - } - - pub fn delete(&self, first_id: Uuid, t: &Identifier, second_id: Uuid) -> Result<()> { - map_err(self.tree.remove(&self.key(first_id, t, second_id)))?; - Ok(()) - } -} - -pub struct VertexPropertyManager<'tree> { - pub tree: &'tree Tree, -} - -impl<'tree> VertexPropertyManager<'tree> { - pub fn new(tree: &'tree Tree) -> Self { - VertexPropertyManager { tree } - } - - fn key(&self, vertex_id: Uuid, name: &str) -> Vec { - util::build(&[ - util::Component::Uuid(vertex_id), - util::Component::FixedLengthString(name), - ]) - } - - pub fn iterate_for_owner(&self, vertex_id: Uuid) -> Result> + '_> { - let prefix = util::build(&[util::Component::Uuid(vertex_id)]); - let iterator = self.tree.scan_prefix(&prefix); - - Ok(iterator.map(move |item| -> Result { - let (k, v) = map_err(item)?; - let mut cursor = Cursor::new(k); - let owner_id = util::read_uuid(&mut cursor); - debug_assert_eq!(vertex_id, owner_id); - let name = util::read_fixed_length_string(&mut cursor); - let value = serde_json::from_slice(&v)?; - Ok(((owner_id, name), value)) - })) - } - - pub fn get(&self, vertex_id: Uuid, name: &str) -> Result> { - let key = self.key(vertex_id, name); - - match map_err(self.tree.get(&key))? { - Some(value_bytes) => Ok(Some(serde_json::from_slice(&value_bytes)?)), - None => Ok(None), - } - } - - pub fn set(&self, vertex_id: Uuid, name: &str, value: &JsonValue) -> Result<()> { - let key = self.key(vertex_id, name); - let value_json = serde_json::to_vec(value)?; - map_err(self.tree.insert(key.as_slice(), value_json.as_slice()))?; - Ok(()) - } - - pub fn delete(&self, vertex_id: Uuid, name: &str) -> Result<()> { - map_err(self.tree.remove(&self.key(vertex_id, name)))?; - Ok(()) - } -} - -pub struct EdgePropertyManager<'tree> { - pub tree: &'tree Tree, -} - -impl<'tree> EdgePropertyManager<'tree> { - pub fn new(tree: &'tree Tree) -> Self { - EdgePropertyManager { tree } - } - - fn key(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str) -> Vec { - util::build(&[ - util::Component::Uuid(outbound_id), - util::Component::Identifier(t.clone()), - util::Component::Uuid(inbound_id), - util::Component::FixedLengthString(name), - ]) - } - - pub fn iterate_for_owner<'a>( - &'a self, - outbound_id: Uuid, - t: &'a Identifier, - inbound_id: Uuid, - ) -> Result> + 'a>> { - let prefix = util::build(&[ - util::Component::Uuid(outbound_id), - util::Component::Identifier(t.clone()), - util::Component::Uuid(inbound_id), - ]); - - let iterator = self.tree.scan_prefix(&prefix); - - let mapped = iterator.map(move |item| -> Result { - let (k, v) = map_err(item)?; - let mut cursor = Cursor::new(k); - - let edge_property_outbound_id = util::read_uuid(&mut cursor); - debug_assert_eq!(edge_property_outbound_id, outbound_id); - - let edge_property_t = util::read_identifier(&mut cursor); - debug_assert_eq!(&edge_property_t, t); - - let edge_property_inbound_id = util::read_uuid(&mut cursor); - debug_assert_eq!(edge_property_inbound_id, inbound_id); - - let edge_property_name = util::read_fixed_length_string(&mut cursor); - - let value = serde_json::from_slice(&v)?; - Ok(( - ( - edge_property_outbound_id, - edge_property_t, - edge_property_inbound_id, - edge_property_name, - ), - value, - )) - }); - - Ok(Box::new(mapped)) - } - - pub fn get(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str) -> Result> { - let key = self.key(outbound_id, t, inbound_id, name); - - match map_err(self.tree.get(&key))? { - Some(ref value_bytes) => Ok(Some(serde_json::from_slice(value_bytes)?)), - None => Ok(None), - } - } - - pub fn set(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str, value: &JsonValue) -> Result<()> { - let key = self.key(outbound_id, t, inbound_id, name); - let value_json = serde_json::to_vec(value)?; - map_err(self.tree.insert(key.as_slice(), value_json.as_slice()))?; - Ok(()) - } - - pub fn delete(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str) -> Result<()> { - map_err(self.tree.remove(&self.key(outbound_id, t, inbound_id, name)))?; - Ok(()) - } -} diff --git a/src/managers/edge_manager.rs b/src/managers/edge_manager.rs new file mode 100644 index 0000000..ee1c42e --- /dev/null +++ b/src/managers/edge_manager.rs @@ -0,0 +1,74 @@ +use std::u8; + +use indradb::{Identifier, util}; +use sled::{IVec, Tree}; +use uuid::Uuid; + +use datastore::SledHolder; +use errors::map_err; + +use crate::managers::edge_property_manager::EdgePropertyManager; +use crate::managers::edge_range_manager::EdgeRangeManager; + +pub struct EdgeManager<'db: 'tree, 'tree> { + pub holder: &'db SledHolder, + pub tree: &'tree Tree, +} + +impl<'db, 'tree> EdgeManager<'db, 'tree> { + pub fn new(ds: &'db SledHolder) -> Self { + EdgeManager { + holder: ds, + tree: &ds.edges, + } + } + + fn key(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid) -> Vec { + util::build(&[ + util::Component::Uuid(outbound_id), + util::Component::Identifier(t.clone()), + util::Component::Uuid(inbound_id), + ]) + } + + pub fn count(&self) -> u64 { + self.tree.iter().count() as u64 + } + + pub fn set(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid) -> indradb::Result<()> { + let edge_range_manager = EdgeRangeManager::new(self.holder); + let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); + + + let key = self.key(outbound_id, t, inbound_id); + map_err( + self.tree + .insert(key, IVec::default()), + )?; + edge_range_manager.set(outbound_id, t, inbound_id)?; + reversed_edge_range_manager.set(inbound_id, t, outbound_id)?; + Ok(()) + } + + pub fn delete(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid) -> indradb::Result<()> { + map_err(self.tree.remove(&self.key(outbound_id, t, inbound_id)))?; + + let edge_range_manager = EdgeRangeManager::new(self.holder); + edge_range_manager.delete(outbound_id, t, inbound_id)?; + + let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); + reversed_edge_range_manager.delete(inbound_id, t, outbound_id)?; + + let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); + for item in edge_property_manager.iterate_for_owner(outbound_id, t, inbound_id)? { + let ((edge_property_outbound_id, edge_property_t, edge_property_inbound_id, edge_property_name), _) = item?; + edge_property_manager.delete( + edge_property_outbound_id, + &edge_property_t, + edge_property_inbound_id, + &edge_property_name[..], + )?; + } + Ok(()) + } +} \ No newline at end of file diff --git a/src/managers/edge_property_manager.rs b/src/managers/edge_property_manager.rs new file mode 100644 index 0000000..1fa0712 --- /dev/null +++ b/src/managers/edge_property_manager.rs @@ -0,0 +1,95 @@ +use std::io::Cursor; +use std::u8; + +use indradb::{Identifier, util}; +use serde_json::Value as JsonValue; +use sled::Tree; +use uuid::Uuid; + +use errors::map_err; + +pub type EdgePropertyItem = ((Uuid, Identifier, Uuid, String), JsonValue); + +pub struct EdgePropertyManager<'tree> { + pub tree: &'tree Tree, +} + +impl<'tree> EdgePropertyManager<'tree> { + pub fn new(tree: &'tree Tree) -> Self { + EdgePropertyManager { tree } + } + + fn key(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str) -> Vec { + util::build(&[ + util::Component::Uuid(outbound_id), + util::Component::Identifier(t.clone()), + util::Component::Uuid(inbound_id), + util::Component::FixedLengthString(name), + ]) + } + + pub fn iterate_for_owner<'a>( + &'a self, + outbound_id: Uuid, + t: &'a Identifier, + inbound_id: Uuid, + ) -> indradb::Result> + 'a>> { + let prefix = util::build(&[ + util::Component::Uuid(outbound_id), + util::Component::Identifier(t.clone()), + util::Component::Uuid(inbound_id), + ]); + + let iterator = self.tree.scan_prefix(&prefix); + + let mapped = iterator.map(move |item| -> indradb::Result { + let (k, v) = map_err(item)?; + let mut cursor = Cursor::new(k); + + let edge_property_outbound_id = util::read_uuid(&mut cursor); + debug_assert_eq!(edge_property_outbound_id, outbound_id); + + let edge_property_t = util::read_identifier(&mut cursor); + debug_assert_eq!(&edge_property_t, t); + + let edge_property_inbound_id = util::read_uuid(&mut cursor); + debug_assert_eq!(edge_property_inbound_id, inbound_id); + + let edge_property_name = util::read_fixed_length_string(&mut cursor); + + let value = serde_json::from_slice(&v)?; + Ok(( + ( + edge_property_outbound_id, + edge_property_t, + edge_property_inbound_id, + edge_property_name, + ), + value, + )) + }); + + Ok(Box::new(mapped)) + } + + pub fn get(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str) -> indradb::Result> { + let key = self.key(outbound_id, t, inbound_id, name); + + match map_err(self.tree.get(&key))? { + Some(ref value_bytes) => Ok(Some(serde_json::from_slice(value_bytes)?)), + None => Ok(None), + } + } + + pub fn set(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str, value: &JsonValue) -> indradb::Result<()> { + let key = self.key(outbound_id, t, inbound_id, name); + let value_json = serde_json::to_vec(value)?; + map_err(self.tree.insert(key.as_slice(), value_json.as_slice()))?; + Ok(()) + } + + pub fn delete(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str) -> indradb::Result<()> { + map_err(self.tree.remove(&self.key(outbound_id, t, inbound_id, name)))?; + Ok(()) + } +} \ No newline at end of file diff --git a/src/managers/edge_range_manager.rs b/src/managers/edge_range_manager.rs new file mode 100644 index 0000000..a0abb83 --- /dev/null +++ b/src/managers/edge_range_manager.rs @@ -0,0 +1,100 @@ +use std::io::Cursor; +use std::u8; + +use indradb::{Edge, Identifier, util}; +use sled::{Iter as DbIterator, Tree}; +use uuid::Uuid; + +use datastore::SledHolder; +use errors::map_err; +use managers; + +pub type EdgeRangeItem = (Uuid, Identifier, Uuid); + +pub struct EdgeRangeManager<'tree> { + pub tree: &'tree Tree, +} + +impl<'tree> EdgeRangeManager<'tree> { + pub fn new<'db: 'tree>(ds: &'db SledHolder) -> Self { + EdgeRangeManager { tree: &ds.edge_ranges } + } + + pub fn new_reversed<'db: 'tree>(ds: &'db SledHolder) -> Self { + EdgeRangeManager { + tree: &ds.reversed_edge_ranges, + } + } + + fn key(&self, first_id: Uuid, t: &Identifier, second_id: Uuid) -> Vec { + util::build(&[ + util::Component::Uuid(first_id), + util::Component::Identifier(t.clone()), + util::Component::Uuid(second_id), + ]) + } + + pub(crate) fn contains(&self, edge: &Edge) -> indradb::Result { + let key = self.key(edge.outbound_id, &edge.t, edge.inbound_id); + map_err(self.tree.contains_key(key)) + } + + fn iterate<'it>(&self, iterator: DbIterator, prefix: Vec) -> impl Iterator> + 'it { + let filtered = managers::take_while_prefixed(iterator, prefix); + filtered.map(move |item| -> indradb::Result { + let (k, _) = map_err(item)?; + let mut cursor = Cursor::new(k); + let first_id = util::read_uuid(&mut cursor); + let t = util::read_identifier(&mut cursor); + + let second_id = util::read_uuid(&mut cursor); + Ok((first_id, t, second_id)) + }) + } + + pub fn iterate_for_range<'iter, 'trans: 'iter>( + &'trans self, + id: Uuid, + t: Option<&Identifier>, + ) -> indradb::Result> + 'iter>> { + match t { + Some(t) => { + let prefix = util::build(&[util::Component::Uuid(id), util::Component::Identifier(t.clone())]); + let low_key = util::build(&[ + util::Component::Uuid(id), + util::Component::Identifier(t.clone()), + ]); + let low_key_bytes: &[u8] = low_key.as_ref(); + let iterator = self.tree.range(low_key_bytes..); + Ok(Box::new(self.iterate(iterator, prefix))) + } + None => { + let prefix = util::build(&[util::Component::Uuid(id)]); + let prefix_bytes: &[u8] = prefix.as_ref(); + let iterator = self.tree.range(prefix_bytes..); + let mapped = self.iterate(iterator, prefix); + Ok(Box::new(mapped)) + } + } + } + + pub fn iterate_for_owner<'iter, 'trans: 'iter>( + &'trans self, + id: Uuid, + ) -> impl Iterator> + 'iter { + let prefix: Vec = util::build(&[util::Component::Uuid(id)]); + let iterator = self.tree.scan_prefix(&prefix); + self.iterate(iterator, prefix) + } + + pub fn set(&self, first_id: Uuid, t: &Identifier, second_id: Uuid) -> indradb::Result<()> { + let key = self.key(first_id, t, second_id); + map_err(self.tree.insert(&key, &[]))?; + Ok(()) + } + + pub fn delete(&self, first_id: Uuid, t: &Identifier, second_id: Uuid) -> indradb::Result<()> { + map_err(self.tree.remove(&self.key(first_id, t, second_id)))?; + Ok(()) + } +} \ No newline at end of file diff --git a/src/managers/mod.rs b/src/managers/mod.rs new file mode 100644 index 0000000..eac1412 --- /dev/null +++ b/src/managers/mod.rs @@ -0,0 +1,21 @@ +use std::u8; + +use sled::{Iter as DbIterator, IVec}; +use sled::Result as SledResult; + +pub(crate) mod vertex_manager; +pub(crate) mod edge_manager; +pub(crate) mod edge_range_manager; +pub(crate) mod vertex_property_manager; +pub(crate) mod edge_property_manager; + +fn take_while_prefixed(iterator: DbIterator, prefix: Vec) -> impl Iterator> { + iterator.take_while(move |item| -> bool { + match item { + Ok((k, _)) => k.starts_with(&prefix), + Err(_) => false, + } + }) +} + + diff --git a/src/managers/vertex_manager.rs b/src/managers/vertex_manager.rs new file mode 100644 index 0000000..252d520 --- /dev/null +++ b/src/managers/vertex_manager.rs @@ -0,0 +1,127 @@ +use std::io::Cursor; +use std::ops::Deref; +use std::u8; + +use indradb::{Identifier, util, Vertex}; +use sled::{Iter as DbIterator, Tree}; +use uuid::Uuid; + +use datastore::SledHolder; +use errors::map_err; +use managers::edge_range_manager::EdgeRangeManager; + +use crate::managers::edge_manager::EdgeManager; +use crate::managers::vertex_property_manager::VertexPropertyManager; + +pub type VertexItem = (Uuid, Identifier); + +pub struct VertexManager<'db: 'tree, 'tree> { + pub holder: &'db SledHolder, + pub tree: &'tree Tree, +} + +impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { + pub fn new(ds: &'db SledHolder) -> Self { + VertexManager { + holder: ds, + tree: ds.db.deref(), + } + } + + pub fn count(&self) -> u64 { + self.tree.iter().count() as u64 + } + + fn key(&self, id: Uuid) -> Vec { + util::build(&[util::Component::Uuid(id)]) + } + + pub fn exists(&self, id: Uuid) -> indradb::Result { + Ok(map_err(self.tree.get(&self.key(id)))?.is_some()) + } + + pub fn get(&self, id: Uuid) -> indradb::Result> { + match map_err(self.tree.get(&self.key(id)))? { + Some(value_bytes) => { + let mut cursor = Cursor::new(value_bytes.deref()); + Ok(Some(util::read_identifier(&mut cursor))) + } + None => Ok(None), + } + } + + fn iterate(&self, iterator: DbIterator) -> impl Iterator> + '_ { + iterator.map(move |item| -> indradb::Result { + let (k, v) = map_err(item)?; + + let id = { + debug_assert_eq!(k.len(), 16); + let mut cursor = Cursor::new(k); + util::read_uuid(&mut cursor) + }; + + let mut cursor = Cursor::new(v); + let t = util::read_identifier(&mut cursor); + Ok((id, t)) + }) + } + + pub fn iterate_for_range(&self, id: Uuid) -> impl Iterator> + '_ { + let low_key = util::build(&[util::Component::Uuid(id)]); + let low_key_bytes: &[u8] = low_key.as_ref(); + let iter = self.tree.range(low_key_bytes..); + self.iterate(iter) + } + + pub fn create(&self, vertex: &Vertex) -> indradb::Result { + let key = self.key(vertex.id); + if map_err(self.tree.contains_key(&key))? { + return Ok(false); + } + map_err(self.tree.insert(&key, util::build(&[util::Component::Identifier(vertex.t.clone())])))?; + Ok(true) + } + + pub fn delete(&self, id: Uuid) -> indradb::Result<()> { + map_err(self.tree.remove(&self.key(id)))?; + + let vertex_property_manager = VertexPropertyManager::new(&self.holder.vertex_properties); + for item in vertex_property_manager.iterate_for_owner(id)? { + let ((vertex_property_owner_id, vertex_property_name), _) = item?; + vertex_property_manager.delete(vertex_property_owner_id, &vertex_property_name[..])?; + } + + let edge_manager = EdgeManager::new(self.holder); + + { + let edge_range_manager = EdgeRangeManager::new(self.holder); + for item in edge_range_manager.iterate_for_owner(id) { + let (edge_range_outbound_id, edge_range_t, edge_range_inbound_id) = item?; + debug_assert_eq!(edge_range_outbound_id, id); + edge_manager.delete( + edge_range_outbound_id, + &edge_range_t, + edge_range_inbound_id, + )?; + } + } + + { + let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); + for item in reversed_edge_range_manager.iterate_for_owner(id) { + let ( + reversed_edge_range_inbound_id, + reversed_edge_range_t, + reversed_edge_range_outbound_id, + ) = item?; + debug_assert_eq!(reversed_edge_range_inbound_id, id); + edge_manager.delete( + reversed_edge_range_outbound_id, + &reversed_edge_range_t, + reversed_edge_range_inbound_id, + )?; + } + } + Ok(()) + } +} \ No newline at end of file diff --git a/src/managers/vertex_property_manager.rs b/src/managers/vertex_property_manager.rs new file mode 100644 index 0000000..b534693 --- /dev/null +++ b/src/managers/vertex_property_manager.rs @@ -0,0 +1,64 @@ +use std::io::Cursor; +use std::u8; + +use indradb::util; +use serde_json::Value as JsonValue; +use sled::Tree; +use uuid::Uuid; + +use errors::map_err; + +pub type OwnedPropertyItem = ((Uuid, String), JsonValue); + +pub struct VertexPropertyManager<'tree> { + pub tree: &'tree Tree, +} + +impl<'tree> VertexPropertyManager<'tree> { + pub fn new(tree: &'tree Tree) -> Self { + VertexPropertyManager { tree } + } + + fn key(&self, vertex_id: Uuid, name: &str) -> Vec { + util::build(&[ + util::Component::Uuid(vertex_id), + util::Component::FixedLengthString(name), + ]) + } + + pub fn iterate_for_owner(&self, vertex_id: Uuid) -> indradb::Result> + '_> { + let prefix = util::build(&[util::Component::Uuid(vertex_id)]); + let iterator = self.tree.scan_prefix(&prefix); + + Ok(iterator.map(move |item| -> indradb::Result { + let (k, v) = map_err(item)?; + let mut cursor = Cursor::new(k); + let owner_id = util::read_uuid(&mut cursor); + debug_assert_eq!(vertex_id, owner_id); + let name = util::read_fixed_length_string(&mut cursor); + let value = serde_json::from_slice(&v)?; + Ok(((owner_id, name), value)) + })) + } + + pub fn get(&self, vertex_id: Uuid, name: &str) -> indradb::Result> { + let key = self.key(vertex_id, name); + + match map_err(self.tree.get(&key))? { + Some(value_bytes) => Ok(Some(serde_json::from_slice(&value_bytes)?)), + None => Ok(None), + } + } + + pub fn set(&self, vertex_id: Uuid, name: &str, value: &JsonValue) -> indradb::Result<()> { + let key = self.key(vertex_id, name); + let value_json = serde_json::to_vec(value)?; + map_err(self.tree.insert(key.as_slice(), value_json.as_slice()))?; + Ok(()) + } + + pub fn delete(&self, vertex_id: Uuid, name: &str) -> indradb::Result<()> { + map_err(self.tree.remove(&self.key(vertex_id, name)))?; + Ok(()) + } +} \ No newline at end of file From 66deb584c29cc61f8c8c00161d55d9e2f741d37d Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Sat, 6 Jul 2024 15:52:30 +0200 Subject: [PATCH 06/28] Write some todos into the code as not to lose it :D --- src/datastore.rs | 2 ++ src/managers/edge_property_manager.rs | 2 ++ src/managers/mod.rs | 1 + src/managers/vertex_property_manager.rs | 2 ++ 4 files changed, 7 insertions(+) diff --git a/src/datastore.rs b/src/datastore.rs index 359e1b2..af86af9 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -49,7 +49,9 @@ pub struct SledHolder { pub(crate) reversed_edge_ranges: Tree, pub(crate) vertex_properties: Tree, pub(crate) edge_properties: Tree, + // for prop-name -> value -> ID prefix-indexed lookup pub(crate) edge_property_values: Tree, + // for prop-name -> value -> UUID prefix-indexed lookup pub(crate) vertex_property_values: Tree, } diff --git a/src/managers/edge_property_manager.rs b/src/managers/edge_property_manager.rs index 1fa0712..c554595 100644 --- a/src/managers/edge_property_manager.rs +++ b/src/managers/edge_property_manager.rs @@ -8,6 +8,8 @@ use uuid::Uuid; use errors::map_err; +// TODO: add a second tree here that prefix-indexes all values like key = [PropName,PropValue,(EdgeIdentifier)] + pub type EdgePropertyItem = ((Uuid, Identifier, Uuid, String), JsonValue); pub struct EdgePropertyManager<'tree> { diff --git a/src/managers/mod.rs b/src/managers/mod.rs index eac1412..773abab 100644 --- a/src/managers/mod.rs +++ b/src/managers/mod.rs @@ -19,3 +19,4 @@ fn take_while_prefixed(iterator: DbIterator, prefix: Vec) -> impl Iterator { From b43e257fd1a3dc3e782f21b69cb1f59ad6cb4bbb Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Sun, 7 Jul 2024 02:28:04 +0200 Subject: [PATCH 07/28] Add first shot at item indexing addition for vertex_property_manager.rs --- src/managers/vertex_property_manager.rs | 38 ++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/managers/vertex_property_manager.rs b/src/managers/vertex_property_manager.rs index ada524a..02280e2 100644 --- a/src/managers/vertex_property_manager.rs +++ b/src/managers/vertex_property_manager.rs @@ -1,9 +1,9 @@ use std::io::Cursor; use std::u8; -use indradb::util; +use indradb::{Json, util}; use serde_json::Value as JsonValue; -use sled::Tree; +use sled::{IVec, Tree}; use uuid::Uuid; use errors::map_err; @@ -14,11 +14,12 @@ pub type OwnedPropertyItem = ((Uuid, String), JsonValue); pub struct VertexPropertyManager<'tree> { pub tree: &'tree Tree, + pub value_index_tree: &'tree Tree, } impl<'tree> VertexPropertyManager<'tree> { - pub fn new(tree: &'tree Tree) -> Self { - VertexPropertyManager { tree } + pub fn new(tree: &'tree Tree, value_index_tree: &'tree Tree) -> Self { + VertexPropertyManager { tree, value_index_tree } } fn key(&self, vertex_id: Uuid, name: &str) -> Vec { @@ -28,6 +29,22 @@ impl<'tree> VertexPropertyManager<'tree> { ]) } + fn key_value_index(&self, vertex_id: &Uuid, value: &JsonValue, property_name: &str) -> Vec { + util::build(&[ + util::Component::FixedLengthString(property_name), + util::Component::Json(&Json::new(value.clone())), + util::Component::Uuid(*vertex_id) + ]) + } + + fn read_key_value_index(buf: IVec) -> (String, JsonValue, Uuid) { + let mut cursor = Cursor::new(buf.as_ref()); + let name = util::read_fixed_length_string(&mut cursor); + let value = serde_json::from_reader(cursor.clone()).unwrap(); + let uuid = util::read_uuid(&mut cursor); + (name, value, uuid) + } + pub fn iterate_for_owner(&self, vertex_id: Uuid) -> indradb::Result> + '_> { let prefix = util::build(&[util::Component::Uuid(vertex_id)]); let iterator = self.tree.scan_prefix(&prefix); @@ -56,11 +73,24 @@ impl<'tree> VertexPropertyManager<'tree> { let key = self.key(vertex_id, name); let value_json = serde_json::to_vec(value)?; map_err(self.tree.insert(key.as_slice(), value_json.as_slice()))?; + let value_index_key = self.key_value_index(&vertex_id, value, name); + map_err(self.value_index_tree.insert(value_index_key, &[]))?; Ok(()) } pub fn delete(&self, vertex_id: Uuid, name: &str) -> indradb::Result<()> { map_err(self.tree.remove(&self.key(vertex_id, name)))?; + let prefix = util::build(&[util::Component::FixedLengthString(name)]); + let items = self.value_index_tree.scan_prefix(prefix); + for item in items { + if let Ok((key, _)) = item { + let (n, v, vid) = Self::read_key_value_index(key); + if vertex_id == vid { + let key = self.key_value_index(&vid, &v, n.as_str()); + map_err(self.value_index_tree.remove(key))?; + } + } + } Ok(()) } } \ No newline at end of file From 9c4cfe38cd09e010f9b3ff07513dea60ea8fb691 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Mon, 8 Jul 2024 14:39:27 +0200 Subject: [PATCH 08/28] Fixup key management and strings for property indexing --- src/datastore.rs | 159 ++++++++++++------------ src/managers/vertex_manager.rs | 28 ++--- src/managers/vertex_property_manager.rs | 103 +++++++++++---- 3 files changed, 169 insertions(+), 121 deletions(-) diff --git a/src/datastore.rs b/src/datastore.rs index af86af9..d34bbf7 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -109,7 +109,6 @@ impl Datastore for SledDatastore { where Self: 'a; - fn transaction(&self) -> Self::Transaction<'_> { SledTransaction { holder: &self.holder, @@ -118,12 +117,14 @@ impl Datastore for SledDatastore { edge_range_manager: EdgeRangeManager::new(&self.holder), edge_range_manager_rev: EdgeRangeManager::new_reversed(&self.holder), edge_property_manager: EdgePropertyManager::new(&self.holder.edge_properties), - vertex_property_manager: VertexPropertyManager::new(&self.holder.vertex_properties), + vertex_property_manager: VertexPropertyManager::new( + &self.holder.vertex_properties, + &self.holder.vertex_property_values, + ), } } } - /// A transaction that is backed by Sled. pub struct SledTransaction<'a> { holder: &'a SledHolder, @@ -153,7 +154,10 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } fn range_vertices(&'a self, offset: Uuid) -> Result> { - let iter = self.vertex_manager.iterate_for_range(offset).map(|e| e.map(|v| Vertex::with_id(v.0, v.1))); + let iter = self + .vertex_manager + .iterate_for_range(offset) + .map(|e| e.map(|v| Vertex::with_id(v.0, v.1))); Ok(Box::new(iter)) } @@ -166,31 +170,16 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } fn vertex_ids_with_property(&'a self, name: Identifier) -> Result>> { - let iter = self.all_vertices()? - .filter(|v| - { - let property_result = v.as_ref().and_then(|v| self.vertex_property_manager.get(v.id, name.as_str()).as_ref()); - if let Ok(v) = property_result { - return v.is_some(); - } - return false; - }).map(|v| v.map(|v| v.id)); - + let iter = self.vertex_property_manager.iterate_for_property_name(name)?; + let iter = iter.map(|r| r.map(|(id, _)| id.0)); Ok(Some(Box::new(iter))) } fn vertex_ids_with_property_value(&'a self, name: Identifier, value: &Json) -> Result>> { - let iter = self.all_vertices()? - .filter(|v| - { - let property_result = v.as_ref().and_then(|v| - self.vertex_property_manager.get(v.id, name.as_str()).as_ref()); - if let Ok(Some(v)) = property_result { - return *v == *value.0; - } - false - }).map(|v| v.map(|v| v.id)); - + let iter = self + .vertex_property_manager + .iterate_for_property_name_and_value(name, value)?; + let iter = iter.map(|r| r.map(|(id, _)| id.0)); Ok(Some(Box::new(iter))) } @@ -200,87 +189,99 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } fn all_edges(&'a self) -> Result> { - let iter = self.edge_range_manager.iterate_for_range(Uuid::default(), None).map(|i| i.map(|e| e.map(|(outbound_id, t, inbound_id)| Edge { - outbound_id, - t, - inbound_id, - })))?; + let iter = self + .edge_range_manager + .iterate_for_range(Uuid::default(), None) + .map(|i| { + i.map(|e| { + e.map(|(outbound_id, t, inbound_id)| Edge { + outbound_id, + t, + inbound_id, + }) + }) + })?; Ok(Box::new(iter)) } fn range_edges(&'a self, offset: Edge) -> Result> { - let iter = self.edge_range_manager.iterate_for_range(offset.inbound_id, Some(&offset.t))?; - let iter = iter.map(|r| r.map(|(outbound_id, t, inbound_id)| Edge { - outbound_id, - t, - inbound_id, - })); + let iter = self + .edge_range_manager + .iterate_for_range(offset.inbound_id, Some(&offset.t))?; + let iter = iter.map(|r| { + r.map(|(outbound_id, t, inbound_id)| Edge { + outbound_id, + t, + inbound_id, + }) + }); Ok(Box::new(iter)) } fn range_reversed_edges(&'a self, offset: Edge) -> Result> { - let iter = self.edge_range_manager_rev.iterate_for_range(offset.inbound_id, Some(&offset.t))?; - let iter = iter.map(|r| r.map(|(outbound_id, t, inbound_id)| Edge { - outbound_id, - t, - inbound_id, - })); + let iter = self + .edge_range_manager_rev + .iterate_for_range(offset.inbound_id, Some(&offset.t))?; + let iter = iter.map(|r| { + r.map(|(outbound_id, t, inbound_id)| Edge { + outbound_id, + t, + inbound_id, + }) + }); Ok(Box::new(iter)) } fn specific_edges(&'a self, edges: Vec) -> Result> { - let iter: Vec<_> = edges.into_iter().filter(|e| { - let r = self.edge_range_manager.contains(e); - if let Ok(r) = r { - r - } else { - false - } - }).map(|e| Ok(e)).collect(); + let iter: Vec<_> = edges + .into_iter() + .filter(|e| { + let r = self.edge_range_manager.contains(e); + if let Ok(r) = r { + r + } else { + false + } + }) + .map(|e| Ok(e)) + .collect(); Ok(Box::new(iter.into_iter())) } fn edges_with_property(&'a self, name: Identifier) -> Result>> { - let iter = self.all_edges()?.filter(|r| { - let has_property = r.as_ref().and_then(|e| self.edge_property_manager.get(e.outbound_id, &e.t, e.inbound_id, name.as_str()).as_ref()); - if let Ok(prop) = has_property { - return prop.is_some(); - } - false - }); - Ok(Some(Box::new(iter))) + unimplemented!() } fn edges_with_property_value(&'a self, name: Identifier, value: &Json) -> Result>> { - let iter = self.all_edges()?.filter(|r| { - let has_property = r.as_ref().and_then(|e| self.edge_property_manager.get(e.outbound_id, &e.t, e.inbound_id, name.as_str()).as_ref()); - if let Ok(Some(v)) = has_property { - return *v == **value; - } - false - }); - Ok(Some(Box::new(iter))) + unimplemented!() } fn vertex_property(&self, vertex: &Vertex, name: Identifier) -> Result> { - let r = self.vertex_property_manager.get(vertex.id, &name)?; + let r = self.vertex_property_manager.get(vertex.id, name.clone())?; Ok(r.map(|v| v.into())) } fn all_vertex_properties_for_vertex(&'a self, vertex: &Vertex) -> Result> { let iter = self.vertex_property_manager.iterate_for_owner(vertex.id)?; - let iter = iter.map(|r| r.and_then(|((_, name), val)| Ok((Identifier::new(name)?, Json::new(val))))); + let iter = iter.map(|r| r.and_then(|((_, name), val)| Ok((name, Json::new(val))))); Ok(Box::new(iter)) } fn edge_property(&self, edge: &Edge, name: Identifier) -> Result> { - let result = self.edge_property_manager.get(edge.outbound_id, &edge.t, edge.inbound_id, name.as_str())?; + let result = self + .edge_property_manager + .get(edge.outbound_id, &edge.t, edge.inbound_id, name.as_str())?; Ok(result.map(|v| Json::new(v))) } fn all_edge_properties_for_edge(&'a self, edge: &Edge) -> Result> { - let iter: Vec<_> = self.edge_property_manager.iterate_for_owner(edge.outbound_id, &edge.t, edge.inbound_id)?.collect(); - let iter = iter.into_iter().map(|e| e.map(|((_, id, _, _), val)| (id, Json::new(val)))); + let iter: Vec<_> = self + .edge_property_manager + .iterate_for_owner(edge.outbound_id, &edge.t, edge.inbound_id)? + .collect(); + let iter = iter + .into_iter() + .map(|e| e.map(|((_, id, _, _), val)| (id, Json::new(val)))); Ok(Box::new(iter)) } @@ -302,16 +303,16 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } fn delete_vertex_properties(&mut self, props: Vec<(Uuid, Identifier)>) -> Result<()> { - let vertex_properties = VertexPropertyManager::new(&self.holder.vertex_properties); for (id, prop) in props { - vertex_properties.delete(id, prop.as_str())? + self.vertex_property_manager.delete(id, prop)? } Ok(()) } fn delete_edge_properties(&mut self, props: Vec<(Edge, Identifier)>) -> Result<()> { for (edge, prop) in props { - self.edge_property_manager.delete(edge.outbound_id, &edge.t, edge.inbound_id, prop.as_str())?; + self.edge_property_manager + .delete(edge.outbound_id, &edge.t, edge.inbound_id, prop.as_str())?; } Ok(()) } @@ -340,23 +341,23 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { fn set_vertex_properties(&mut self, vertices: Vec, name: Identifier, value: &Json) -> Result<()> { for v in vertices { - self.vertex_property_manager.set(v, name.as_str(), &value)?; + self.vertex_property_manager.set(v, name, &value)?; } Ok(()) } fn set_edge_properties(&mut self, edges: Vec, name: Identifier, value: &Json) -> Result<()> { for edge in edges { - self.edge_property_manager.set(edge.outbound_id, &edge.t, edge.inbound_id, name.as_str(), &value)?; + self.edge_property_manager + .set(edge.outbound_id, &edge.t, edge.inbound_id, name.as_str(), &value)?; } Ok(()) } } - -fn remove_nones_from_iterator(iter: I) -> impl Iterator> +fn remove_nones_from_iterator(iter: I) -> impl Iterator> where - I: Iterator>>, + I: Iterator>>, { iter.filter_map(|item| match item { Err(err) => Some(Err(err)), diff --git a/src/managers/vertex_manager.rs b/src/managers/vertex_manager.rs index 252d520..81e8786 100644 --- a/src/managers/vertex_manager.rs +++ b/src/managers/vertex_manager.rs @@ -50,7 +50,7 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { } } - fn iterate(&self, iterator: DbIterator) -> impl Iterator> + '_ { + fn iterate(&self, iterator: DbIterator) -> impl Iterator> + '_ { iterator.map(move |item| -> indradb::Result { let (k, v) = map_err(item)?; @@ -66,7 +66,7 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { }) } - pub fn iterate_for_range(&self, id: Uuid) -> impl Iterator> + '_ { + pub fn iterate_for_range(&self, id: Uuid) -> impl Iterator> + '_ { let low_key = util::build(&[util::Component::Uuid(id)]); let low_key_bytes: &[u8] = low_key.as_ref(); let iter = self.tree.range(low_key_bytes..); @@ -78,17 +78,21 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { if map_err(self.tree.contains_key(&key))? { return Ok(false); } - map_err(self.tree.insert(&key, util::build(&[util::Component::Identifier(vertex.t.clone())])))?; + map_err( + self.tree + .insert(&key, util::build(&[util::Component::Identifier(vertex.t.clone())])), + )?; Ok(true) } pub fn delete(&self, id: Uuid) -> indradb::Result<()> { map_err(self.tree.remove(&self.key(id)))?; - let vertex_property_manager = VertexPropertyManager::new(&self.holder.vertex_properties); + let vertex_property_manager = + VertexPropertyManager::new(&self.holder.vertex_properties, &self.holder.vertex_property_values); for item in vertex_property_manager.iterate_for_owner(id)? { let ((vertex_property_owner_id, vertex_property_name), _) = item?; - vertex_property_manager.delete(vertex_property_owner_id, &vertex_property_name[..])?; + vertex_property_manager.delete(vertex_property_owner_id, vertex_property_name)?; } let edge_manager = EdgeManager::new(self.holder); @@ -98,22 +102,14 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { for item in edge_range_manager.iterate_for_owner(id) { let (edge_range_outbound_id, edge_range_t, edge_range_inbound_id) = item?; debug_assert_eq!(edge_range_outbound_id, id); - edge_manager.delete( - edge_range_outbound_id, - &edge_range_t, - edge_range_inbound_id, - )?; + edge_manager.delete(edge_range_outbound_id, &edge_range_t, edge_range_inbound_id)?; } } { let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); for item in reversed_edge_range_manager.iterate_for_owner(id) { - let ( - reversed_edge_range_inbound_id, - reversed_edge_range_t, - reversed_edge_range_outbound_id, - ) = item?; + let (reversed_edge_range_inbound_id, reversed_edge_range_t, reversed_edge_range_outbound_id) = item?; debug_assert_eq!(reversed_edge_range_inbound_id, id); edge_manager.delete( reversed_edge_range_outbound_id, @@ -124,4 +120,4 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { } Ok(()) } -} \ No newline at end of file +} diff --git a/src/managers/vertex_property_manager.rs b/src/managers/vertex_property_manager.rs index 02280e2..85653a1 100644 --- a/src/managers/vertex_property_manager.rs +++ b/src/managers/vertex_property_manager.rs @@ -1,16 +1,14 @@ use std::io::Cursor; use std::u8; -use indradb::{Json, util}; +use indradb::{Identifier, Json, util}; use serde_json::Value as JsonValue; use sled::{IVec, Tree}; use uuid::Uuid; use errors::map_err; -// TODO: add a second tree here that prefix-indexes all values like key = [PropName,PropValue,Vertex-UUID] - -pub type OwnedPropertyItem = ((Uuid, String), JsonValue); +pub type OwnedPropertyItem = ((Uuid, Identifier), JsonValue); pub struct VertexPropertyManager<'tree> { pub tree: &'tree Tree, @@ -22,30 +20,64 @@ impl<'tree> VertexPropertyManager<'tree> { VertexPropertyManager { tree, value_index_tree } } - fn key(&self, vertex_id: Uuid, name: &str) -> Vec { - util::build(&[ - util::Component::Uuid(vertex_id), - util::Component::FixedLengthString(name), - ]) + fn key(&self, vertex_id: Uuid, name: Identifier) -> Vec { + util::build(&[util::Component::Uuid(vertex_id), util::Component::Identifier(name)]) } - fn key_value_index(&self, vertex_id: &Uuid, value: &JsonValue, property_name: &str) -> Vec { + fn key_value_index(vertex_id: &Uuid, value: &JsonValue, property_name: Identifier) -> Vec { util::build(&[ - util::Component::FixedLengthString(property_name), + util::Component::Identifier(property_name), util::Component::Json(&Json::new(value.clone())), - util::Component::Uuid(*vertex_id) + util::Component::Uuid(*vertex_id), ]) } - fn read_key_value_index(buf: IVec) -> (String, JsonValue, Uuid) { + fn read_key_value_index(buf: IVec) -> (Identifier, u64, Uuid) { let mut cursor = Cursor::new(buf.as_ref()); - let name = util::read_fixed_length_string(&mut cursor); - let value = serde_json::from_reader(cursor.clone()).unwrap(); + let name = util::read_identifier(&mut cursor); + let value = util::read_u64(&mut cursor); let uuid = util::read_uuid(&mut cursor); (name, value, uuid) } - pub fn iterate_for_owner(&self, vertex_id: Uuid) -> indradb::Result> + '_> { + pub fn iterate_for_property_name( + &self, + name: Identifier, + ) -> indradb::Result> + '_> { + let prefix = util::build(&[util::Component::Identifier(name)]); + let iterator = self.value_index_tree.scan_prefix(&prefix); + + Ok(iterator.map(move |item| -> indradb::Result { + let (k, v) = map_err(item)?; + let (n, _, vid) = Self::read_key_value_index(k); + let value = serde_json::from_slice(&v)?; + Ok(((vid, n), value)) + })) + } + + pub fn iterate_for_property_name_and_value( + &self, + name: Identifier, + value: &JsonValue, + ) -> indradb::Result> + '_> { + let prefix = util::build(&[ + util::Component::Identifier(name), + util::Component::Json(&Json::new(value.clone())), + ]); + let iterator = self.value_index_tree.scan_prefix(&prefix); + + Ok(iterator.map(move |item| -> indradb::Result { + let (k, v) = map_err(item)?; + let (n, _, vid) = Self::read_key_value_index(k); + let value = serde_json::from_slice(&v)?; + Ok(((vid, n), value)) + })) + } + + pub fn iterate_for_owner( + &self, + vertex_id: Uuid, + ) -> indradb::Result> + '_> { let prefix = util::build(&[util::Component::Uuid(vertex_id)]); let iterator = self.tree.scan_prefix(&prefix); @@ -54,13 +86,13 @@ impl<'tree> VertexPropertyManager<'tree> { let mut cursor = Cursor::new(k); let owner_id = util::read_uuid(&mut cursor); debug_assert_eq!(vertex_id, owner_id); - let name = util::read_fixed_length_string(&mut cursor); + let name = util::read_identifier(&mut cursor); let value = serde_json::from_slice(&v)?; Ok(((owner_id, name), value)) })) } - pub fn get(&self, vertex_id: Uuid, name: &str) -> indradb::Result> { + pub fn get(&self, vertex_id: Uuid, name: Identifier) -> indradb::Result> { let key = self.key(vertex_id, name); match map_err(self.tree.get(&key))? { @@ -69,28 +101,47 @@ impl<'tree> VertexPropertyManager<'tree> { } } - pub fn set(&self, vertex_id: Uuid, name: &str, value: &JsonValue) -> indradb::Result<()> { + pub fn set(&self, vertex_id: Uuid, name: Identifier, value: &JsonValue) -> indradb::Result<()> { let key = self.key(vertex_id, name); let value_json = serde_json::to_vec(value)?; map_err(self.tree.insert(key.as_slice(), value_json.as_slice()))?; - let value_index_key = self.key_value_index(&vertex_id, value, name); - map_err(self.value_index_tree.insert(value_index_key, &[]))?; + let value_index_key = Self::key_value_index(&vertex_id, value, name); + map_err(self.value_index_tree.insert(value_index_key, value_json.as_slice()))?; Ok(()) } - pub fn delete(&self, vertex_id: Uuid, name: &str) -> indradb::Result<()> { + pub fn delete(&self, vertex_id: Uuid, name: Identifier) -> indradb::Result<()> { map_err(self.tree.remove(&self.key(vertex_id, name)))?; - let prefix = util::build(&[util::Component::FixedLengthString(name)]); + let prefix = util::build(&[util::Component::Identifier(name)]); let items = self.value_index_tree.scan_prefix(prefix); for item in items { if let Ok((key, _)) = item { - let (n, v, vid) = Self::read_key_value_index(key); + let (n, v, vid) = Self::read_key_value_index(key.clone()); if vertex_id == vid { - let key = self.key_value_index(&vid, &v, n.as_str()); map_err(self.value_index_tree.remove(key))?; } } } Ok(()) } -} \ No newline at end of file +} +#[cfg(test)] +mod test { + use serde_json::json; + use uuid::{Context, Timestamp}; + + use super::*; + + #[test] + fn test_index_key_and_reco() { + let context = Context::new(24); + let uuid = Uuid::new_v1(Timestamp::now(context), &[1, 2, 3, 4, 5, 6]); + let name = Identifier::new("_changesetID").unwrap(); + let value = json! {"Changesets/25dfc1e7-fdd1-4027-9e98-48a8429a9c70"}; + let key = VertexPropertyManager::key_value_index(&uuid, &value, name); + + let (n, v, id) = VertexPropertyManager::read_key_value_index(key.into()); + assert_eq!(n, name); + assert_eq!(uuid, id); + } +} From 112784317240a5e53dbfb4d2d5300e8e73ef33eb Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Mon, 8 Jul 2024 15:08:16 +0200 Subject: [PATCH 09/28] Fixup range iteration of edges --- src/datastore.rs | 27 ++++++++---------- src/managers/edge_range_manager.rs | 44 +++++++++++++----------------- 2 files changed, 30 insertions(+), 41 deletions(-) diff --git a/src/datastore.rs b/src/datastore.rs index d34bbf7..e0d9798 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -189,25 +189,21 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } fn all_edges(&'a self) -> Result> { - let iter = self - .edge_range_manager - .iterate_for_range(Uuid::default(), None) - .map(|i| { - i.map(|e| { - e.map(|(outbound_id, t, inbound_id)| Edge { - outbound_id, - t, - inbound_id, - }) - }) - })?; + let iter = self.edge_range_manager.iterate_for_all(); + let iter = iter.map(|e| { + e.map(|(outbound_id, t, inbound_id)| Edge { + outbound_id, + t, + inbound_id, + }) + }); Ok(Box::new(iter)) } fn range_edges(&'a self, offset: Edge) -> Result> { let iter = self .edge_range_manager - .iterate_for_range(offset.inbound_id, Some(&offset.t))?; + .iterate_for_range(offset.outbound_id, offset.t, offset.inbound_id); let iter = iter.map(|r| { r.map(|(outbound_id, t, inbound_id)| Edge { outbound_id, @@ -221,7 +217,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { fn range_reversed_edges(&'a self, offset: Edge) -> Result> { let iter = self .edge_range_manager_rev - .iterate_for_range(offset.inbound_id, Some(&offset.t))?; + .iterate_for_range(offset.outbound_id, offset.t, offset.inbound_id); let iter = iter.map(|r| { r.map(|(outbound_id, t, inbound_id)| Edge { outbound_id, @@ -328,8 +324,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { if !outbound_exists || !inbound_exists { Ok(false) } else { - let edge_manager = EdgeManager::new(&self.holder); - edge_manager.set(edge.outbound_id, &edge.t, edge.inbound_id)?; + self.edge_manager.set(edge.outbound_id, &edge.t, edge.inbound_id)?; Ok(true) } } diff --git a/src/managers/edge_range_manager.rs b/src/managers/edge_range_manager.rs index a0abb83..05bd7f1 100644 --- a/src/managers/edge_range_manager.rs +++ b/src/managers/edge_range_manager.rs @@ -39,7 +39,11 @@ impl<'tree> EdgeRangeManager<'tree> { map_err(self.tree.contains_key(key)) } - fn iterate<'it>(&self, iterator: DbIterator, prefix: Vec) -> impl Iterator> + 'it { + fn iterate<'it>( + &self, + iterator: DbIterator, + prefix: Vec, + ) -> impl Iterator> + 'it { let filtered = managers::take_while_prefixed(iterator, prefix); filtered.map(move |item| -> indradb::Result { let (k, _) = map_err(item)?; @@ -54,34 +58,24 @@ impl<'tree> EdgeRangeManager<'tree> { pub fn iterate_for_range<'iter, 'trans: 'iter>( &'trans self, - id: Uuid, - t: Option<&Identifier>, - ) -> indradb::Result> + 'iter>> { - match t { - Some(t) => { - let prefix = util::build(&[util::Component::Uuid(id), util::Component::Identifier(t.clone())]); - let low_key = util::build(&[ - util::Component::Uuid(id), - util::Component::Identifier(t.clone()), - ]); - let low_key_bytes: &[u8] = low_key.as_ref(); - let iterator = self.tree.range(low_key_bytes..); - Ok(Box::new(self.iterate(iterator, prefix))) - } - None => { - let prefix = util::build(&[util::Component::Uuid(id)]); - let prefix_bytes: &[u8] = prefix.as_ref(); - let iterator = self.tree.range(prefix_bytes..); - let mapped = self.iterate(iterator, prefix); - Ok(Box::new(mapped)) - } - } + outbound_id: Uuid, + t: Identifier, + inbound_id: Uuid, + ) -> impl Iterator> { + let offset = self.key(outbound_id, &t, inbound_id); + let iterator = self.tree.range(offset..); + self.iterate(iterator, vec![]).map(|item| item) + } + + pub fn iterate_for_all(&self) -> impl Iterator> { + let iterator = self.tree.iter(); + self.iterate(iterator, vec![]).map(|item| item) } pub fn iterate_for_owner<'iter, 'trans: 'iter>( &'trans self, id: Uuid, - ) -> impl Iterator> + 'iter { + ) -> impl Iterator> + 'iter { let prefix: Vec = util::build(&[util::Component::Uuid(id)]); let iterator = self.tree.scan_prefix(&prefix); self.iterate(iterator, prefix) @@ -97,4 +91,4 @@ impl<'tree> EdgeRangeManager<'tree> { map_err(self.tree.remove(&self.key(first_id, t, second_id)))?; Ok(()) } -} \ No newline at end of file +} From 7df15c615da39ca28cab4a1c14b0ab8f14e12ce8 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Mon, 8 Jul 2024 16:19:59 +0200 Subject: [PATCH 10/28] Fixup range iteration of edges with properties --- src/datastore.rs | 34 +++--- src/managers/edge_manager.rs | 21 ++-- src/managers/edge_property_manager.rs | 151 +++++++++++++++++++++--- src/managers/vertex_property_manager.rs | 4 +- 4 files changed, 155 insertions(+), 55 deletions(-) diff --git a/src/datastore.rs b/src/datastore.rs index e0d9798..b40d956 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -116,7 +116,10 @@ impl Datastore for SledDatastore { edge_manager: EdgeManager::new(&self.holder), edge_range_manager: EdgeRangeManager::new(&self.holder), edge_range_manager_rev: EdgeRangeManager::new_reversed(&self.holder), - edge_property_manager: EdgePropertyManager::new(&self.holder.edge_properties), + edge_property_manager: EdgePropertyManager::new( + &self.holder.edge_properties, + &self.holder.edge_property_values, + ), vertex_property_manager: VertexPropertyManager::new( &self.holder.vertex_properties, &self.holder.vertex_property_values, @@ -245,11 +248,15 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } fn edges_with_property(&'a self, name: Identifier) -> Result>> { - unimplemented!() + let iter = self.edge_property_manager.iterate_for_property_name(name)?; + Ok(Some(Box::new(iter))) } fn edges_with_property_value(&'a self, name: Identifier, value: &Json) -> Result>> { - unimplemented!() + let iter = self + .edge_property_manager + .iterate_for_property_name_and_value(name, value)?; + Ok(Some(Box::new(iter))) } fn vertex_property(&self, vertex: &Vertex, name: Identifier) -> Result> { @@ -266,7 +273,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { fn edge_property(&self, edge: &Edge, name: Identifier) -> Result> { let result = self .edge_property_manager - .get(edge.outbound_id, &edge.t, edge.inbound_id, name.as_str())?; + .get(edge.outbound_id, &edge.t, edge.inbound_id, name)?; Ok(result.map(|v| Json::new(v))) } @@ -275,9 +282,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { .edge_property_manager .iterate_for_owner(edge.outbound_id, &edge.t, edge.inbound_id)? .collect(); - let iter = iter - .into_iter() - .map(|e| e.map(|((_, id, _, _), val)| (id, Json::new(val)))); + let iter = iter.into_iter().map(|e| e.map(|((_, id), val)| (id, Json::new(val)))); Ok(Box::new(iter)) } @@ -308,7 +313,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { fn delete_edge_properties(&mut self, props: Vec<(Edge, Identifier)>) -> Result<()> { for (edge, prop) in props { self.edge_property_manager - .delete(edge.outbound_id, &edge.t, edge.inbound_id, prop.as_str())?; + .delete(edge.outbound_id, &edge.t, edge.inbound_id, prop)?; } Ok(()) } @@ -344,19 +349,8 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { fn set_edge_properties(&mut self, edges: Vec, name: Identifier, value: &Json) -> Result<()> { for edge in edges { self.edge_property_manager - .set(edge.outbound_id, &edge.t, edge.inbound_id, name.as_str(), &value)?; + .set(edge.outbound_id, &edge.t, edge.inbound_id, name, &value)?; } Ok(()) } } - -fn remove_nones_from_iterator(iter: I) -> impl Iterator> -where - I: Iterator>>, -{ - iter.filter_map(|item| match item { - Err(err) => Some(Err(err)), - Ok(Some(value)) => Some(Ok(value)), - _ => None, - }) -} diff --git a/src/managers/edge_manager.rs b/src/managers/edge_manager.rs index ee1c42e..139b346 100644 --- a/src/managers/edge_manager.rs +++ b/src/managers/edge_manager.rs @@ -39,12 +39,8 @@ impl<'db, 'tree> EdgeManager<'db, 'tree> { let edge_range_manager = EdgeRangeManager::new(self.holder); let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); - let key = self.key(outbound_id, t, inbound_id); - map_err( - self.tree - .insert(key, IVec::default()), - )?; + map_err(self.tree.insert(key, IVec::default()))?; edge_range_manager.set(outbound_id, t, inbound_id)?; reversed_edge_range_manager.set(inbound_id, t, outbound_id)?; Ok(()) @@ -59,16 +55,13 @@ impl<'db, 'tree> EdgeManager<'db, 'tree> { let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); reversed_edge_range_manager.delete(inbound_id, t, outbound_id)?; - let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties); + let edge_property_manager = + EdgePropertyManager::new(&self.holder.edge_properties, &self.holder.edge_property_values); + for item in edge_property_manager.iterate_for_owner(outbound_id, t, inbound_id)? { - let ((edge_property_outbound_id, edge_property_t, edge_property_inbound_id, edge_property_name), _) = item?; - edge_property_manager.delete( - edge_property_outbound_id, - &edge_property_t, - edge_property_inbound_id, - &edge_property_name[..], - )?; + let ((edge, id), _) = item?; + edge_property_manager.delete(edge.outbound_id, &edge.t, edge.inbound_id, id)?; } Ok(()) } -} \ No newline at end of file +} diff --git a/src/managers/edge_property_manager.rs b/src/managers/edge_property_manager.rs index c554595..baf8ec3 100644 --- a/src/managers/edge_property_manager.rs +++ b/src/managers/edge_property_manager.rs @@ -1,41 +1,85 @@ use std::io::Cursor; use std::u8; -use indradb::{Identifier, util}; -use serde_json::Value as JsonValue; -use sled::Tree; +use indradb::{Edge, Identifier, Json, util}; +use serde_json::{Value as JsonValue, Value}; +use sled::{IVec, Tree}; use uuid::Uuid; use errors::map_err; -// TODO: add a second tree here that prefix-indexes all values like key = [PropName,PropValue,(EdgeIdentifier)] - -pub type EdgePropertyItem = ((Uuid, Identifier, Uuid, String), JsonValue); +pub type EdgePropertyItem = ((Edge, Identifier), JsonValue); pub struct EdgePropertyManager<'tree> { pub tree: &'tree Tree, + pub value_index_tree: &'tree Tree, } impl<'tree> EdgePropertyManager<'tree> { - pub fn new(tree: &'tree Tree) -> Self { - EdgePropertyManager { tree } + pub fn new(tree: &'tree Tree, value_index_tree: &'tree Tree) -> Self { + EdgePropertyManager { tree, value_index_tree } } - fn key(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str) -> Vec { + fn key(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: Identifier) -> Vec { util::build(&[ util::Component::Uuid(outbound_id), util::Component::Identifier(t.clone()), util::Component::Uuid(inbound_id), - util::Component::FixedLengthString(name), + util::Component::Identifier(name), ]) } + pub fn iterate_for_property_name( + &self, + name: Identifier, + ) -> indradb::Result> + '_> { + let prefix = util::build(&[util::Component::Identifier(name.clone())]); + let iterator = self.value_index_tree.scan_prefix(&prefix); + + Ok(iterator.map(move |item| -> indradb::Result { + let (k, _v) = map_err(item)?; + let (_p, _, edge) = Self::read_key_value_index(k); + Ok(edge) + })) + } + + pub fn iterate_for_property_name_and_value( + &'tree self, + name: Identifier, + value: &JsonValue, + ) -> indradb::Result> + 'tree> { + let value = value.clone(); + let prefix = util::build(&[util::Component::Identifier(name.clone())]); + let iterator = self.value_index_tree.scan_prefix(&prefix); + + Ok(iterator + .map(move |item| -> indradb::Result<(Edge, Value)> { + let (k, v) = map_err(item)?; + + let (_p, _, edge) = Self::read_key_value_index(k); + let val = serde_json::from_slice(&v)?; + Ok((edge, val)) + }) + .filter_map(move |item| { + item.map_or_else( + |e| Some(Err(e)), + |(edge, val)| { + if val == value { + Some(Ok(edge)) + } else { + None + } + }, + ) + })) + } + pub fn iterate_for_owner<'a>( &'a self, outbound_id: Uuid, t: &'a Identifier, inbound_id: Uuid, - ) -> indradb::Result> + 'a>> { + ) -> indradb::Result> + 'a>> { let prefix = util::build(&[ util::Component::Uuid(outbound_id), util::Component::Identifier(t.clone()), @@ -57,14 +101,16 @@ impl<'tree> EdgePropertyManager<'tree> { let edge_property_inbound_id = util::read_uuid(&mut cursor); debug_assert_eq!(edge_property_inbound_id, inbound_id); - let edge_property_name = util::read_fixed_length_string(&mut cursor); + let edge_property_name = util::read_identifier(&mut cursor); let value = serde_json::from_slice(&v)?; Ok(( ( - edge_property_outbound_id, - edge_property_t, - edge_property_inbound_id, + Edge { + outbound_id: edge_property_outbound_id, + t: edge_property_t.clone(), + inbound_id: edge_property_inbound_id, + }, edge_property_name, ), value, @@ -74,7 +120,13 @@ impl<'tree> EdgePropertyManager<'tree> { Ok(Box::new(mapped)) } - pub fn get(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str) -> indradb::Result> { + pub fn get( + &self, + outbound_id: Uuid, + t: &Identifier, + inbound_id: Uuid, + name: Identifier, + ) -> indradb::Result> { let key = self.key(outbound_id, t, inbound_id, name); match map_err(self.tree.get(&key))? { @@ -83,15 +135,76 @@ impl<'tree> EdgePropertyManager<'tree> { } } - pub fn set(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str, value: &JsonValue) -> indradb::Result<()> { + fn key_value_index(edge: Edge, value: &JsonValue, property_name: Identifier) -> Vec { + util::build(&[ + util::Component::Identifier(property_name), + util::Component::Json(&Json::new(value.clone())), + util::Component::Uuid(edge.outbound_id), + util::Component::Identifier(edge.t.clone()), + util::Component::Uuid(edge.inbound_id), + ]) + } + + fn read_key_value_index(buf: IVec) -> (Identifier, u64, Edge) { + let mut cursor = Cursor::new(buf.as_ref()); + let name = util::read_identifier(&mut cursor); + let value = util::read_u64(&mut cursor); + let outbound_id = util::read_uuid(&mut cursor); + let t = util::read_identifier(&mut cursor); + let inbound_id = util::read_uuid(&mut cursor); + ( + name, + value, + Edge { + outbound_id, + t, + inbound_id, + }, + ) + } + + pub fn set( + &self, + outbound_id: Uuid, + t: &Identifier, + inbound_id: Uuid, + name: Identifier, + value: &JsonValue, + ) -> indradb::Result<()> { let key = self.key(outbound_id, t, inbound_id, name); let value_json = serde_json::to_vec(value)?; map_err(self.tree.insert(key.as_slice(), value_json.as_slice()))?; + let value_key = Self::key_value_index( + Edge { + outbound_id, + t: t.clone(), + inbound_id, + }, + value, + name, + ); + map_err( + self.value_index_tree + .insert(value_key.as_slice(), value_json.as_slice()), + )?; Ok(()) } - pub fn delete(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: &str) -> indradb::Result<()> { + pub fn delete(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: Identifier) -> indradb::Result<()> { + let owner = Edge { + outbound_id, + t: t.clone(), + inbound_id, + }; map_err(self.tree.remove(&self.key(outbound_id, t, inbound_id, name)))?; + let prefix = util::build(&[util::Component::Identifier(t.clone())]); + for edge in self.value_index_tree.scan_prefix(prefix) { + let (k, _) = map_err(edge)?; + let (_n, _, vid) = Self::read_key_value_index(k.clone()); + if vid == owner { + map_err(self.value_index_tree.remove(&k))?; + } + } Ok(()) } -} \ No newline at end of file +} diff --git a/src/managers/vertex_property_manager.rs b/src/managers/vertex_property_manager.rs index 85653a1..c408b24 100644 --- a/src/managers/vertex_property_manager.rs +++ b/src/managers/vertex_property_manager.rs @@ -116,7 +116,7 @@ impl<'tree> VertexPropertyManager<'tree> { let items = self.value_index_tree.scan_prefix(prefix); for item in items { if let Ok((key, _)) = item { - let (n, v, vid) = Self::read_key_value_index(key.clone()); + let (_n, _v, vid) = Self::read_key_value_index(key.clone()); if vertex_id == vid { map_err(self.value_index_tree.remove(key))?; } @@ -140,7 +140,7 @@ mod test { let value = json! {"Changesets/25dfc1e7-fdd1-4027-9e98-48a8429a9c70"}; let key = VertexPropertyManager::key_value_index(&uuid, &value, name); - let (n, v, id) = VertexPropertyManager::read_key_value_index(key.into()); + let (n, _v, id) = VertexPropertyManager::read_key_value_index(key.into()); assert_eq!(n, name); assert_eq!(uuid, id); } From 1464427f537d1ea880cc70b57848cd753984fb2c Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Mon, 8 Jul 2024 17:30:10 +0200 Subject: [PATCH 11/28] Clippy lints --- Cargo.toml | 1 + src/datastore.rs | 72 ++++++++----------------- src/errors.rs | 2 +- src/lib.rs | 1 + src/managers/edge_manager.rs | 33 ++++++------ src/managers/edge_property_manager.rs | 55 ++++++++++--------- src/managers/edge_range_manager.rs | 66 ++++++++++------------- src/managers/mod.rs | 21 +------- src/managers/vertex_manager.rs | 25 ++++----- src/managers/vertex_property_manager.rs | 26 ++++----- 10 files changed, 123 insertions(+), 179 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 15808c2..61af4ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,3 +25,4 @@ serde_json = "1.0" sled = { version = "0.34" } tempfile = { version = "3.10", optional = true } uuid = { version = "1.9", features = ["v1", "serde"] } +ecow = { version = "0.2.2" } \ No newline at end of file diff --git a/src/datastore.rs b/src/datastore.rs index b40d956..2b4861e 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -1,5 +1,4 @@ use std::path::Path; -use std::u64; use indradb::{Datastore, DynIter, Edge, Error, Identifier, Json, Result, Transaction, Vertex}; use sled::{Config, Db, Tree}; @@ -55,7 +54,7 @@ pub struct SledHolder { pub(crate) vertex_property_values: Tree, } -impl<'ds> SledHolder { +impl SledHolder { /// The meat of a Sled datastore. /// /// # Arguments @@ -92,7 +91,7 @@ pub struct SledDatastore { pub(crate) holder: SledHolder, } -impl<'ds> SledDatastore { +impl SledDatastore { /// Creates a new Sled datastore. /// /// # Arguments @@ -141,8 +140,8 @@ pub struct SledTransaction<'a> { impl<'a> Transaction<'a> for SledTransaction<'a> { fn vertex_count(&self) -> u64 { - let vertex_manager = VertexManager::new(&self.holder); - vertex_manager.count().into() + let vertex_manager = VertexManager::new(self.holder); + vertex_manager.count() } fn all_vertices(&'a self) -> Result> { @@ -187,47 +186,25 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } fn edge_count(&self) -> u64 { - let edge_manager = EdgeManager::new(&self.holder); - edge_manager.count().into() + let edge_manager = EdgeManager::new(self.holder); + edge_manager.count() } fn all_edges(&'a self) -> Result> { let iter = self.edge_range_manager.iterate_for_all(); - let iter = iter.map(|e| { - e.map(|(outbound_id, t, inbound_id)| Edge { - outbound_id, - t, - inbound_id, - }) - }); + Ok(Box::new(iter)) } fn range_edges(&'a self, offset: Edge) -> Result> { - let iter = self - .edge_range_manager - .iterate_for_range(offset.outbound_id, offset.t, offset.inbound_id); - let iter = iter.map(|r| { - r.map(|(outbound_id, t, inbound_id)| Edge { - outbound_id, - t, - inbound_id, - }) - }); + let iter = self.edge_range_manager.iterate_for_range(&offset); + Ok(Box::new(iter)) } fn range_reversed_edges(&'a self, offset: Edge) -> Result> { - let iter = self - .edge_range_manager_rev - .iterate_for_range(offset.outbound_id, offset.t, offset.inbound_id); - let iter = iter.map(|r| { - r.map(|(outbound_id, t, inbound_id)| Edge { - outbound_id, - t, - inbound_id, - }) - }); + let iter = self.edge_range_manager_rev.iterate_for_range(&offset); + Ok(Box::new(iter)) } @@ -242,7 +219,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { false } }) - .map(|e| Ok(e)) + .map(Ok) .collect(); Ok(Box::new(iter.into_iter())) } @@ -260,29 +237,26 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } fn vertex_property(&self, vertex: &Vertex, name: Identifier) -> Result> { - let r = self.vertex_property_manager.get(vertex.id, name.clone())?; + let r = self.vertex_property_manager.get(vertex.id, name)?; Ok(r.map(|v| v.into())) } fn all_vertex_properties_for_vertex(&'a self, vertex: &Vertex) -> Result> { let iter = self.vertex_property_manager.iterate_for_owner(vertex.id)?; - let iter = iter.map(|r| r.and_then(|((_, name), val)| Ok((name, Json::new(val))))); + let iter = iter.map(|r| r.map(|((_, name), val)| (name, Json::new(val)))); Ok(Box::new(iter)) } fn edge_property(&self, edge: &Edge, name: Identifier) -> Result> { let result = self .edge_property_manager - .get(edge.outbound_id, &edge.t, edge.inbound_id, name)?; - Ok(result.map(|v| Json::new(v))) + .get(edge.outbound_id, edge.t, edge.inbound_id, name)?; + Ok(result.map(Json::new)) } fn all_edge_properties_for_edge(&'a self, edge: &Edge) -> Result> { - let iter: Vec<_> = self - .edge_property_manager - .iterate_for_owner(edge.outbound_id, &edge.t, edge.inbound_id)? - .collect(); - let iter = iter.into_iter().map(|e| e.map(|((_, id), val)| (id, Json::new(val)))); + let iter = self.edge_property_manager.iterate_for_owner(edge)?; + let iter = iter.map(|e| e.map(|((_, id), val)| (id, Json::new(val)))); Ok(Box::new(iter)) } @@ -296,7 +270,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { fn delete_edges(&mut self, edges: Vec) -> Result<()> { for item in edges.iter() { if self.vertex_manager.get(item.outbound_id)?.is_some() { - self.edge_manager.delete(item.outbound_id, &item.t, item.inbound_id)?; + self.edge_manager.delete(item)?; }; } @@ -313,7 +287,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { fn delete_edge_properties(&mut self, props: Vec<(Edge, Identifier)>) -> Result<()> { for (edge, prop) in props { self.edge_property_manager - .delete(edge.outbound_id, &edge.t, edge.inbound_id, prop)?; + .delete(edge.outbound_id, edge.t, edge.inbound_id, prop)?; } Ok(()) } @@ -329,7 +303,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { if !outbound_exists || !inbound_exists { Ok(false) } else { - self.edge_manager.set(edge.outbound_id, &edge.t, edge.inbound_id)?; + self.edge_manager.set(edge)?; Ok(true) } } @@ -341,7 +315,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { fn set_vertex_properties(&mut self, vertices: Vec, name: Identifier, value: &Json) -> Result<()> { for v in vertices { - self.vertex_property_manager.set(v, name, &value)?; + self.vertex_property_manager.set(v, name, value)?; } Ok(()) } @@ -349,7 +323,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { fn set_edge_properties(&mut self, edges: Vec, name: Identifier, value: &Json) -> Result<()> { for edge in edges { self.edge_property_manager - .set(edge.outbound_id, &edge.t, edge.inbound_id, name, &value)?; + .set(edge.outbound_id, edge.t, edge.inbound_id, name, value)?; } Ok(()) } diff --git a/src/errors.rs b/src/errors.rs index 4e42071..406b339 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -2,5 +2,5 @@ use indradb::Error as IndraError; use sled::Error as SledError; pub(crate) fn map_err(result: Result) -> Result { - result.map_err(|err| IndraError::Datastore { 0: Box::new(err) }) + result.map_err(|err| IndraError::Datastore(Box::new(err))) } diff --git a/src/lib.rs b/src/lib.rs index fa6ef33..559077b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ #![cfg_attr(feature = "bench-suite", feature(test))] extern crate chrono; +extern crate ecow; #[cfg(any(feature = "bench-suite", feature = "test-suite"))] #[macro_use] extern crate indradb; diff --git a/src/managers/edge_manager.rs b/src/managers/edge_manager.rs index 139b346..3321bf8 100644 --- a/src/managers/edge_manager.rs +++ b/src/managers/edge_manager.rs @@ -1,8 +1,5 @@ -use std::u8; - -use indradb::{Identifier, util}; +use indradb::{Edge, util}; use sled::{IVec, Tree}; -use uuid::Uuid; use datastore::SledHolder; use errors::map_err; @@ -23,11 +20,11 @@ impl<'db, 'tree> EdgeManager<'db, 'tree> { } } - fn key(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid) -> Vec { + fn key(&self, edge: Edge) -> Vec { util::build(&[ - util::Component::Uuid(outbound_id), - util::Component::Identifier(t.clone()), - util::Component::Uuid(inbound_id), + util::Component::Uuid(edge.outbound_id), + util::Component::Identifier(edge.t), + util::Component::Uuid(edge.inbound_id), ]) } @@ -35,32 +32,32 @@ impl<'db, 'tree> EdgeManager<'db, 'tree> { self.tree.iter().count() as u64 } - pub fn set(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid) -> indradb::Result<()> { + pub fn set(&self, edge: &Edge) -> indradb::Result<()> { let edge_range_manager = EdgeRangeManager::new(self.holder); let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); - let key = self.key(outbound_id, t, inbound_id); + let key = self.key(edge.clone()); map_err(self.tree.insert(key, IVec::default()))?; - edge_range_manager.set(outbound_id, t, inbound_id)?; - reversed_edge_range_manager.set(inbound_id, t, outbound_id)?; + edge_range_manager.set(edge)?; + reversed_edge_range_manager.set(edge)?; Ok(()) } - pub fn delete(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid) -> indradb::Result<()> { - map_err(self.tree.remove(&self.key(outbound_id, t, inbound_id)))?; + pub fn delete(&self, edge: &Edge) -> indradb::Result<()> { + map_err(self.tree.remove(self.key(edge.clone())))?; let edge_range_manager = EdgeRangeManager::new(self.holder); - edge_range_manager.delete(outbound_id, t, inbound_id)?; + edge_range_manager.delete(edge)?; let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); - reversed_edge_range_manager.delete(inbound_id, t, outbound_id)?; + reversed_edge_range_manager.delete(edge)?; let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties, &self.holder.edge_property_values); - for item in edge_property_manager.iterate_for_owner(outbound_id, t, inbound_id)? { + for item in edge_property_manager.iterate_for_owner(edge)? { let ((edge, id), _) = item?; - edge_property_manager.delete(edge.outbound_id, &edge.t, edge.inbound_id, id)?; + edge_property_manager.delete(edge.outbound_id, edge.t, edge.inbound_id, id)?; } Ok(()) } diff --git a/src/managers/edge_property_manager.rs b/src/managers/edge_property_manager.rs index baf8ec3..7693c5c 100644 --- a/src/managers/edge_property_manager.rs +++ b/src/managers/edge_property_manager.rs @@ -1,6 +1,6 @@ use std::io::Cursor; -use std::u8; +use ecow::EcoVec; use indradb::{Edge, Identifier, Json, util}; use serde_json::{Value as JsonValue, Value}; use sled::{IVec, Tree}; @@ -20,10 +20,10 @@ impl<'tree> EdgePropertyManager<'tree> { EdgePropertyManager { tree, value_index_tree } } - fn key(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: Identifier) -> Vec { + fn key(&self, outbound_id: Uuid, t: Identifier, inbound_id: Uuid, name: Identifier) -> Vec { util::build(&[ util::Component::Uuid(outbound_id), - util::Component::Identifier(t.clone()), + util::Component::Identifier(t), util::Component::Uuid(inbound_id), util::Component::Identifier(name), ]) @@ -33,8 +33,8 @@ impl<'tree> EdgePropertyManager<'tree> { &self, name: Identifier, ) -> indradb::Result> + '_> { - let prefix = util::build(&[util::Component::Identifier(name.clone())]); - let iterator = self.value_index_tree.scan_prefix(&prefix); + let prefix = util::build(&[util::Component::Identifier(name)]); + let iterator = self.value_index_tree.scan_prefix(prefix); Ok(iterator.map(move |item| -> indradb::Result { let (k, _v) = map_err(item)?; @@ -49,8 +49,8 @@ impl<'tree> EdgePropertyManager<'tree> { value: &JsonValue, ) -> indradb::Result> + 'tree> { let value = value.clone(); - let prefix = util::build(&[util::Component::Identifier(name.clone())]); - let iterator = self.value_index_tree.scan_prefix(&prefix); + let prefix = util::build(&[util::Component::Identifier(name)]); + let iterator = self.value_index_tree.scan_prefix(prefix); Ok(iterator .map(move |item| -> indradb::Result<(Edge, Value)> { @@ -76,30 +76,25 @@ impl<'tree> EdgePropertyManager<'tree> { pub fn iterate_for_owner<'a>( &'a self, - outbound_id: Uuid, - t: &'a Identifier, - inbound_id: Uuid, + edge: &Edge, ) -> indradb::Result> + 'a>> { let prefix = util::build(&[ - util::Component::Uuid(outbound_id), - util::Component::Identifier(t.clone()), - util::Component::Uuid(inbound_id), + util::Component::Uuid(edge.outbound_id), + util::Component::Identifier(edge.t), + util::Component::Uuid(edge.inbound_id), ]); - let iterator = self.tree.scan_prefix(&prefix); + let iterator = self.tree.scan_prefix(prefix); let mapped = iterator.map(move |item| -> indradb::Result { let (k, v) = map_err(item)?; let mut cursor = Cursor::new(k); let edge_property_outbound_id = util::read_uuid(&mut cursor); - debug_assert_eq!(edge_property_outbound_id, outbound_id); let edge_property_t = util::read_identifier(&mut cursor); - debug_assert_eq!(&edge_property_t, t); let edge_property_inbound_id = util::read_uuid(&mut cursor); - debug_assert_eq!(edge_property_inbound_id, inbound_id); let edge_property_name = util::read_identifier(&mut cursor); @@ -108,7 +103,7 @@ impl<'tree> EdgePropertyManager<'tree> { ( Edge { outbound_id: edge_property_outbound_id, - t: edge_property_t.clone(), + t: edge_property_t, inbound_id: edge_property_inbound_id, }, edge_property_name, @@ -123,13 +118,13 @@ impl<'tree> EdgePropertyManager<'tree> { pub fn get( &self, outbound_id: Uuid, - t: &Identifier, + t: Identifier, inbound_id: Uuid, name: Identifier, ) -> indradb::Result> { let key = self.key(outbound_id, t, inbound_id, name); - match map_err(self.tree.get(&key))? { + match map_err(self.tree.get(key))? { Some(ref value_bytes) => Ok(Some(serde_json::from_slice(value_bytes)?)), None => Ok(None), } @@ -140,7 +135,7 @@ impl<'tree> EdgePropertyManager<'tree> { util::Component::Identifier(property_name), util::Component::Json(&Json::new(value.clone())), util::Component::Uuid(edge.outbound_id), - util::Component::Identifier(edge.t.clone()), + util::Component::Identifier(edge.t), util::Component::Uuid(edge.inbound_id), ]) } @@ -166,7 +161,7 @@ impl<'tree> EdgePropertyManager<'tree> { pub fn set( &self, outbound_id: Uuid, - t: &Identifier, + t: Identifier, inbound_id: Uuid, name: Identifier, value: &JsonValue, @@ -177,7 +172,7 @@ impl<'tree> EdgePropertyManager<'tree> { let value_key = Self::key_value_index( Edge { outbound_id, - t: t.clone(), + t, inbound_id, }, value, @@ -190,21 +185,25 @@ impl<'tree> EdgePropertyManager<'tree> { Ok(()) } - pub fn delete(&self, outbound_id: Uuid, t: &Identifier, inbound_id: Uuid, name: Identifier) -> indradb::Result<()> { + pub fn delete(&self, outbound_id: Uuid, t: Identifier, inbound_id: Uuid, name: Identifier) -> indradb::Result<()> { let owner = Edge { outbound_id, - t: t.clone(), + t, inbound_id, }; - map_err(self.tree.remove(&self.key(outbound_id, t, inbound_id, name)))?; - let prefix = util::build(&[util::Component::Identifier(t.clone())]); + map_err(self.tree.remove(self.key(outbound_id, t, inbound_id, name)))?; + let prefix = util::build(&[util::Component::Identifier(t)]); + let mut edges_to_delete = EcoVec::new(); for edge in self.value_index_tree.scan_prefix(prefix) { let (k, _) = map_err(edge)?; let (_n, _, vid) = Self::read_key_value_index(k.clone()); if vid == owner { - map_err(self.value_index_tree.remove(&k))?; + edges_to_delete.push(k); } } + for edge in edges_to_delete { + map_err(self.value_index_tree.remove(&edge))?; + } Ok(()) } } diff --git a/src/managers/edge_range_manager.rs b/src/managers/edge_range_manager.rs index 05bd7f1..aa4b694 100644 --- a/src/managers/edge_range_manager.rs +++ b/src/managers/edge_range_manager.rs @@ -1,15 +1,11 @@ use std::io::Cursor; -use std::u8; -use indradb::{Edge, Identifier, util}; +use indradb::{Edge, util}; use sled::{Iter as DbIterator, Tree}; use uuid::Uuid; use datastore::SledHolder; use errors::map_err; -use managers; - -pub type EdgeRangeItem = (Uuid, Identifier, Uuid); pub struct EdgeRangeManager<'tree> { pub tree: &'tree Tree, @@ -26,69 +22,65 @@ impl<'tree> EdgeRangeManager<'tree> { } } - fn key(&self, first_id: Uuid, t: &Identifier, second_id: Uuid) -> Vec { + fn key(&self, edge: &Edge) -> Vec { util::build(&[ - util::Component::Uuid(first_id), - util::Component::Identifier(t.clone()), - util::Component::Uuid(second_id), + util::Component::Uuid(edge.outbound_id), + util::Component::Identifier(edge.t), + util::Component::Uuid(edge.inbound_id), ]) } pub(crate) fn contains(&self, edge: &Edge) -> indradb::Result { - let key = self.key(edge.outbound_id, &edge.t, edge.inbound_id); + let key = self.key(edge); map_err(self.tree.contains_key(key)) } - fn iterate<'it>( - &self, - iterator: DbIterator, - prefix: Vec, - ) -> impl Iterator> + 'it { - let filtered = managers::take_while_prefixed(iterator, prefix); - filtered.map(move |item| -> indradb::Result { + fn sled_to_edge(iter: DbIterator) -> impl Iterator> { + iter.map(move |item| { let (k, _) = map_err(item)?; let mut cursor = Cursor::new(k); - let first_id = util::read_uuid(&mut cursor); + let outbound_id = util::read_uuid(&mut cursor); let t = util::read_identifier(&mut cursor); - - let second_id = util::read_uuid(&mut cursor); - Ok((first_id, t, second_id)) + let inbound_id = util::read_uuid(&mut cursor); + Ok(Edge { + outbound_id, + t, + inbound_id, + }) }) } pub fn iterate_for_range<'iter, 'trans: 'iter>( &'trans self, - outbound_id: Uuid, - t: Identifier, - inbound_id: Uuid, - ) -> impl Iterator> { - let offset = self.key(outbound_id, &t, inbound_id); + edge: &Edge, + ) -> impl Iterator> { + let offset = self.key(edge); let iterator = self.tree.range(offset..); - self.iterate(iterator, vec![]).map(|item| item) + Self::sled_to_edge(iterator) } - pub fn iterate_for_all(&self) -> impl Iterator> { + pub fn iterate_for_all(&self) -> impl Iterator> { let iterator = self.tree.iter(); - self.iterate(iterator, vec![]).map(|item| item) + Self::sled_to_edge(iterator) } pub fn iterate_for_owner<'iter, 'trans: 'iter>( &'trans self, id: Uuid, - ) -> impl Iterator> + 'iter { + ) -> impl Iterator> + 'iter { let prefix: Vec = util::build(&[util::Component::Uuid(id)]); - let iterator = self.tree.scan_prefix(&prefix); - self.iterate(iterator, prefix) + let iterator = self.tree.scan_prefix(prefix); + Self::sled_to_edge(iterator) } - pub fn set(&self, first_id: Uuid, t: &Identifier, second_id: Uuid) -> indradb::Result<()> { - let key = self.key(first_id, t, second_id); - map_err(self.tree.insert(&key, &[]))?; + pub fn set(&self, edge: &Edge) -> indradb::Result<()> { + let key = self.key(edge); + map_err(self.tree.insert(key, &[]))?; Ok(()) } - pub fn delete(&self, first_id: Uuid, t: &Identifier, second_id: Uuid) -> indradb::Result<()> { - map_err(self.tree.remove(&self.key(first_id, t, second_id)))?; + pub fn delete(&self, edge: &Edge) -> indradb::Result<()> { + map_err(self.tree.remove(self.key(edge)))?; Ok(()) } } diff --git a/src/managers/mod.rs b/src/managers/mod.rs index 773abab..c620a73 100644 --- a/src/managers/mod.rs +++ b/src/managers/mod.rs @@ -1,22 +1,5 @@ -use std::u8; - -use sled::{Iter as DbIterator, IVec}; -use sled::Result as SledResult; - -pub(crate) mod vertex_manager; pub(crate) mod edge_manager; +pub(crate) mod edge_property_manager; pub(crate) mod edge_range_manager; +pub(crate) mod vertex_manager; pub(crate) mod vertex_property_manager; -pub(crate) mod edge_property_manager; - -fn take_while_prefixed(iterator: DbIterator, prefix: Vec) -> impl Iterator> { - iterator.take_while(move |item| -> bool { - match item { - Ok((k, _)) => k.starts_with(&prefix), - Err(_) => false, - } - }) -} - - - diff --git a/src/managers/vertex_manager.rs b/src/managers/vertex_manager.rs index 81e8786..e7c37c6 100644 --- a/src/managers/vertex_manager.rs +++ b/src/managers/vertex_manager.rs @@ -1,6 +1,5 @@ use std::io::Cursor; use std::ops::Deref; -use std::u8; use indradb::{Identifier, util, Vertex}; use sled::{Iter as DbIterator, Tree}; @@ -37,11 +36,11 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { } pub fn exists(&self, id: Uuid) -> indradb::Result { - Ok(map_err(self.tree.get(&self.key(id)))?.is_some()) + Ok(map_err(self.tree.get(self.key(id)))?.is_some()) } pub fn get(&self, id: Uuid) -> indradb::Result> { - match map_err(self.tree.get(&self.key(id)))? { + match map_err(self.tree.get(self.key(id)))? { Some(value_bytes) => { let mut cursor = Cursor::new(value_bytes.deref()); Ok(Some(util::read_identifier(&mut cursor))) @@ -80,13 +79,13 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { } map_err( self.tree - .insert(&key, util::build(&[util::Component::Identifier(vertex.t.clone())])), + .insert(&key, util::build(&[util::Component::Identifier(vertex.t)])), )?; Ok(true) } pub fn delete(&self, id: Uuid) -> indradb::Result<()> { - map_err(self.tree.remove(&self.key(id)))?; + map_err(self.tree.remove(self.key(id)))?; let vertex_property_manager = VertexPropertyManager::new(&self.holder.vertex_properties, &self.holder.vertex_property_values); @@ -100,22 +99,18 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { { let edge_range_manager = EdgeRangeManager::new(self.holder); for item in edge_range_manager.iterate_for_owner(id) { - let (edge_range_outbound_id, edge_range_t, edge_range_inbound_id) = item?; - debug_assert_eq!(edge_range_outbound_id, id); - edge_manager.delete(edge_range_outbound_id, &edge_range_t, edge_range_inbound_id)?; + let edge = item?; + debug_assert_eq!(edge.outbound_id, id); + edge_manager.delete(&edge)?; } } { let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); for item in reversed_edge_range_manager.iterate_for_owner(id) { - let (reversed_edge_range_inbound_id, reversed_edge_range_t, reversed_edge_range_outbound_id) = item?; - debug_assert_eq!(reversed_edge_range_inbound_id, id); - edge_manager.delete( - reversed_edge_range_outbound_id, - &reversed_edge_range_t, - reversed_edge_range_inbound_id, - )?; + let edge = item?; + debug_assert_eq!(edge.inbound_id, id); + edge_manager.delete(&edge)?; } } Ok(()) diff --git a/src/managers/vertex_property_manager.rs b/src/managers/vertex_property_manager.rs index c408b24..aa6428b 100644 --- a/src/managers/vertex_property_manager.rs +++ b/src/managers/vertex_property_manager.rs @@ -1,6 +1,6 @@ use std::io::Cursor; -use std::u8; +use ecow::EcoVec; use indradb::{Identifier, Json, util}; use serde_json::Value as JsonValue; use sled::{IVec, Tree}; @@ -45,7 +45,7 @@ impl<'tree> VertexPropertyManager<'tree> { name: Identifier, ) -> indradb::Result> + '_> { let prefix = util::build(&[util::Component::Identifier(name)]); - let iterator = self.value_index_tree.scan_prefix(&prefix); + let iterator = self.value_index_tree.scan_prefix(prefix); Ok(iterator.map(move |item| -> indradb::Result { let (k, v) = map_err(item)?; @@ -64,7 +64,7 @@ impl<'tree> VertexPropertyManager<'tree> { util::Component::Identifier(name), util::Component::Json(&Json::new(value.clone())), ]); - let iterator = self.value_index_tree.scan_prefix(&prefix); + let iterator = self.value_index_tree.scan_prefix(prefix); Ok(iterator.map(move |item| -> indradb::Result { let (k, v) = map_err(item)?; @@ -79,7 +79,7 @@ impl<'tree> VertexPropertyManager<'tree> { vertex_id: Uuid, ) -> indradb::Result> + '_> { let prefix = util::build(&[util::Component::Uuid(vertex_id)]); - let iterator = self.tree.scan_prefix(&prefix); + let iterator = self.tree.scan_prefix(prefix); Ok(iterator.map(move |item| -> indradb::Result { let (k, v) = map_err(item)?; @@ -95,7 +95,7 @@ impl<'tree> VertexPropertyManager<'tree> { pub fn get(&self, vertex_id: Uuid, name: Identifier) -> indradb::Result> { let key = self.key(vertex_id, name); - match map_err(self.tree.get(&key))? { + match map_err(self.tree.get(key))? { Some(value_bytes) => Ok(Some(serde_json::from_slice(&value_bytes)?)), None => Ok(None), } @@ -111,17 +111,19 @@ impl<'tree> VertexPropertyManager<'tree> { } pub fn delete(&self, vertex_id: Uuid, name: Identifier) -> indradb::Result<()> { - map_err(self.tree.remove(&self.key(vertex_id, name)))?; + map_err(self.tree.remove(self.key(vertex_id, name)))?; let prefix = util::build(&[util::Component::Identifier(name)]); let items = self.value_index_tree.scan_prefix(prefix); - for item in items { - if let Ok((key, _)) = item { - let (_n, _v, vid) = Self::read_key_value_index(key.clone()); - if vertex_id == vid { - map_err(self.value_index_tree.remove(key))?; - } + let mut keys_to_remove = EcoVec::new(); + for (key, _) in items.flatten() { + let (_n, _v, vid) = Self::read_key_value_index(key.clone()); + if vertex_id == vid { + keys_to_remove.push(key); } } + for key in keys_to_remove { + map_err(self.value_index_tree.remove(key))?; + } Ok(()) } } From adb53f42c9b3636eede4f023a4ce34b422f43906 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Mon, 8 Jul 2024 18:03:10 +0200 Subject: [PATCH 12/28] Fix test callsites, not all tests green yet. --- src/lib.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 559077b..2c044cc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,12 +22,14 @@ mod errors; mod managers; mod normal_config { + use indradb::Database; + #[cfg(feature = "bench-suite")] full_bench_impl!({ use super::SledDatastore; use tempfile::tempdir; let path = tempdir().unwrap().into_path(); - SledDatastore::new(path).unwrap() + Database::new(SledDatastore::new(path).unwrap()) }); #[cfg(feature = "test-suite")] @@ -35,17 +37,19 @@ mod normal_config { use super::SledDatastore; use tempfile::tempdir; let path = tempdir().unwrap().into_path(); - SledDatastore::new(path).unwrap() + Database::new(SledDatastore::new(path).unwrap()) }); } mod compression_config { + use indradb::Database; + #[cfg(feature = "bench-suite")] full_bench_impl!({ use super::SledConfig; use tempfile::tempdir; let path = tempdir().unwrap().into_path(); - SledConfig::with_compression(None).open(path).unwrap() + Database::new(SledConfig::with_compression(None).open(path).unwrap()) }); #[cfg(feature = "test-suite")] @@ -53,6 +57,6 @@ mod compression_config { use super::SledConfig; use tempfile::tempdir; let path = tempdir().unwrap().into_path(); - SledConfig::with_compression(None).open(path).unwrap() + Database::new(SledConfig::with_compression(None).open(path).unwrap()) }); } From 3deb26a835a4cc21eb9f289633093087ce7e756f Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Mon, 8 Jul 2024 18:04:34 +0200 Subject: [PATCH 13/28] Fix missing compression feature flag --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 61af4ef..bb76298 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ bench-suite = ["indradb-lib/bench-suite", "tempfile"] chrono = { version = "0.4", features = ["serde"] } indradb-lib = "4.0" serde_json = "1.0" -sled = { version = "0.34" } +sled = { version = "0.34", features = ["compression"] } tempfile = { version = "3.10", optional = true } uuid = { version = "1.9", features = ["v1", "serde"] } ecow = { version = "0.2.2" } \ No newline at end of file From a8e5a13f54bb39cb96f3880cfdf94c6db4145d8d Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Tue, 9 Jul 2024 09:10:09 +0200 Subject: [PATCH 14/28] Refactor transaction out. --- src/datastore.rs | 206 +----------------------------------------- src/lib.rs | 9 +- src/transaction.rs | 219 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 227 insertions(+), 207 deletions(-) create mode 100644 src/transaction.rs diff --git a/src/datastore.rs b/src/datastore.rs index 2b4861e..89e8cb3 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -1,12 +1,12 @@ use std::path::Path; -use indradb::{Datastore, DynIter, Edge, Error, Identifier, Json, Result, Transaction, Vertex}; +use indradb::{Datastore, Result}; use sled::{Config, Db, Tree}; -use uuid::Uuid; use managers::edge_manager::EdgeManager; use managers::edge_range_manager::EdgeRangeManager; use managers::vertex_property_manager::VertexPropertyManager; +use transaction::SledTransaction; use crate::managers::edge_property_manager::EdgePropertyManager; use crate::managers::vertex_manager::VertexManager; @@ -126,205 +126,3 @@ impl Datastore for SledDatastore { } } } - -/// A transaction that is backed by Sled. -pub struct SledTransaction<'a> { - holder: &'a SledHolder, - vertex_manager: VertexManager<'a, 'a>, - edge_manager: EdgeManager<'a, 'a>, - edge_property_manager: EdgePropertyManager<'a>, - vertex_property_manager: VertexPropertyManager<'a>, - edge_range_manager: EdgeRangeManager<'a>, - edge_range_manager_rev: EdgeRangeManager<'a>, -} - -impl<'a> Transaction<'a> for SledTransaction<'a> { - fn vertex_count(&self) -> u64 { - let vertex_manager = VertexManager::new(self.holder); - vertex_manager.count() - } - - fn all_vertices(&'a self) -> Result> { - let iterator = self.vertex_manager.iterate_for_range(Uuid::default()); - let mapped = iterator.map(move |item| { - let (id, t) = item?; - let vertex = Vertex::with_id(id, t); - Ok::(vertex) - }); - - Ok(Box::new(mapped)) - } - - fn range_vertices(&'a self, offset: Uuid) -> Result> { - let iter = self - .vertex_manager - .iterate_for_range(offset) - .map(|e| e.map(|v| Vertex::with_id(v.0, v.1))); - Ok(Box::new(iter)) - } - - fn specific_vertices(&'a self, ids: Vec) -> Result> { - let iter = ids.into_iter().filter_map(move |id| { - let v = self.vertex_manager.get(id).transpose(); - v.map(|v| v.map(|v| Vertex::with_id(id, v))) - }); - Ok(Box::new(iter)) - } - - fn vertex_ids_with_property(&'a self, name: Identifier) -> Result>> { - let iter = self.vertex_property_manager.iterate_for_property_name(name)?; - let iter = iter.map(|r| r.map(|(id, _)| id.0)); - Ok(Some(Box::new(iter))) - } - - fn vertex_ids_with_property_value(&'a self, name: Identifier, value: &Json) -> Result>> { - let iter = self - .vertex_property_manager - .iterate_for_property_name_and_value(name, value)?; - let iter = iter.map(|r| r.map(|(id, _)| id.0)); - Ok(Some(Box::new(iter))) - } - - fn edge_count(&self) -> u64 { - let edge_manager = EdgeManager::new(self.holder); - edge_manager.count() - } - - fn all_edges(&'a self) -> Result> { - let iter = self.edge_range_manager.iterate_for_all(); - - Ok(Box::new(iter)) - } - - fn range_edges(&'a self, offset: Edge) -> Result> { - let iter = self.edge_range_manager.iterate_for_range(&offset); - - Ok(Box::new(iter)) - } - - fn range_reversed_edges(&'a self, offset: Edge) -> Result> { - let iter = self.edge_range_manager_rev.iterate_for_range(&offset); - - Ok(Box::new(iter)) - } - - fn specific_edges(&'a self, edges: Vec) -> Result> { - let iter: Vec<_> = edges - .into_iter() - .filter(|e| { - let r = self.edge_range_manager.contains(e); - if let Ok(r) = r { - r - } else { - false - } - }) - .map(Ok) - .collect(); - Ok(Box::new(iter.into_iter())) - } - - fn edges_with_property(&'a self, name: Identifier) -> Result>> { - let iter = self.edge_property_manager.iterate_for_property_name(name)?; - Ok(Some(Box::new(iter))) - } - - fn edges_with_property_value(&'a self, name: Identifier, value: &Json) -> Result>> { - let iter = self - .edge_property_manager - .iterate_for_property_name_and_value(name, value)?; - Ok(Some(Box::new(iter))) - } - - fn vertex_property(&self, vertex: &Vertex, name: Identifier) -> Result> { - let r = self.vertex_property_manager.get(vertex.id, name)?; - Ok(r.map(|v| v.into())) - } - - fn all_vertex_properties_for_vertex(&'a self, vertex: &Vertex) -> Result> { - let iter = self.vertex_property_manager.iterate_for_owner(vertex.id)?; - let iter = iter.map(|r| r.map(|((_, name), val)| (name, Json::new(val)))); - Ok(Box::new(iter)) - } - - fn edge_property(&self, edge: &Edge, name: Identifier) -> Result> { - let result = self - .edge_property_manager - .get(edge.outbound_id, edge.t, edge.inbound_id, name)?; - Ok(result.map(Json::new)) - } - - fn all_edge_properties_for_edge(&'a self, edge: &Edge) -> Result> { - let iter = self.edge_property_manager.iterate_for_owner(edge)?; - let iter = iter.map(|e| e.map(|((_, id), val)| (id, Json::new(val)))); - Ok(Box::new(iter)) - } - - fn delete_vertices(&mut self, vertices: Vec) -> Result<()> { - for v in vertices { - self.vertex_manager.delete(v.id)? - } - Ok(()) - } - - fn delete_edges(&mut self, edges: Vec) -> Result<()> { - for item in edges.iter() { - if self.vertex_manager.get(item.outbound_id)?.is_some() { - self.edge_manager.delete(item)?; - }; - } - - Ok(()) - } - - fn delete_vertex_properties(&mut self, props: Vec<(Uuid, Identifier)>) -> Result<()> { - for (id, prop) in props { - self.vertex_property_manager.delete(id, prop)? - } - Ok(()) - } - - fn delete_edge_properties(&mut self, props: Vec<(Edge, Identifier)>) -> Result<()> { - for (edge, prop) in props { - self.edge_property_manager - .delete(edge.outbound_id, edge.t, edge.inbound_id, prop)?; - } - Ok(()) - } - - fn create_vertex(&mut self, vertex: &Vertex) -> Result { - self.vertex_manager.create(vertex) - } - - fn create_edge(&mut self, edge: &Edge) -> Result { - let outbound_exists = self.vertex_manager.exists(edge.outbound_id)?; - let inbound_exists = self.vertex_manager.exists(edge.inbound_id)?; - - if !outbound_exists || !inbound_exists { - Ok(false) - } else { - self.edge_manager.set(edge)?; - Ok(true) - } - } - - fn index_property(&mut self, _name: Identifier) -> Result<()> { - // oh boy we really want this i guess - Ok(()) - } - - fn set_vertex_properties(&mut self, vertices: Vec, name: Identifier, value: &Json) -> Result<()> { - for v in vertices { - self.vertex_property_manager.set(v, name, value)?; - } - Ok(()) - } - - fn set_edge_properties(&mut self, edges: Vec, name: Identifier, value: &Json) -> Result<()> { - for edge in edges { - self.edge_property_manager - .set(edge.outbound_id, edge.t, edge.inbound_id, name, value)?; - } - Ok(()) - } -} diff --git a/src/lib.rs b/src/lib.rs index 2c044cc..b1dc516 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,18 +15,19 @@ extern crate sled; extern crate tempfile; extern crate uuid; -pub use self::datastore::{SledConfig, SledDatastore, SledTransaction}; +pub use self::datastore::{SledConfig, SledDatastore}; mod datastore; mod errors; mod managers; +mod transaction; mod normal_config { - use indradb::Database; #[cfg(feature = "bench-suite")] full_bench_impl!({ use super::SledDatastore; + use indradb::Database; use tempfile::tempdir; let path = tempdir().unwrap().into_path(); Database::new(SledDatastore::new(path).unwrap()) @@ -35,6 +36,7 @@ mod normal_config { #[cfg(feature = "test-suite")] full_test_impl!({ use super::SledDatastore; + use indradb::Database; use tempfile::tempdir; let path = tempdir().unwrap().into_path(); Database::new(SledDatastore::new(path).unwrap()) @@ -42,11 +44,11 @@ mod normal_config { } mod compression_config { - use indradb::Database; #[cfg(feature = "bench-suite")] full_bench_impl!({ use super::SledConfig; + use indradb::Database; use tempfile::tempdir; let path = tempdir().unwrap().into_path(); Database::new(SledConfig::with_compression(None).open(path).unwrap()) @@ -55,6 +57,7 @@ mod compression_config { #[cfg(feature = "test-suite")] full_test_impl!({ use super::SledConfig; + use indradb::Database; use tempfile::tempdir; let path = tempdir().unwrap().into_path(); Database::new(SledConfig::with_compression(None).open(path).unwrap()) diff --git a/src/transaction.rs b/src/transaction.rs new file mode 100644 index 0000000..3e68f23 --- /dev/null +++ b/src/transaction.rs @@ -0,0 +1,219 @@ +use indradb::{DynIter, Edge, Error, Identifier, Json, Transaction, Vertex}; +use uuid::Uuid; + +use datastore::SledHolder; +use managers::edge_manager::EdgeManager; +use managers::edge_property_manager::EdgePropertyManager; +use managers::edge_range_manager::EdgeRangeManager; +use managers::vertex_manager::VertexManager; +use managers::vertex_property_manager::VertexPropertyManager; + +/// A transaction that is backed by Sled. +pub struct SledTransaction<'a> { + pub(crate) holder: &'a SledHolder, + pub(crate) vertex_manager: VertexManager<'a, 'a>, + pub(crate) edge_manager: EdgeManager<'a, 'a>, + pub(crate) edge_property_manager: EdgePropertyManager<'a>, + pub(crate) vertex_property_manager: VertexPropertyManager<'a>, + pub(crate) edge_range_manager: EdgeRangeManager<'a>, + pub(crate) edge_range_manager_rev: EdgeRangeManager<'a>, +} + +impl<'a> Transaction<'a> for SledTransaction<'a> { + fn vertex_count(&self) -> u64 { + let vertex_manager = VertexManager::new(self.holder); + vertex_manager.count() + } + + fn all_vertices(&'a self) -> indradb::Result> { + let iterator = self.vertex_manager.iterate_for_range(Uuid::default()); + let mapped = iterator.map(move |item| { + let (id, t) = item?; + let vertex = Vertex::with_id(id, t); + Ok::(vertex) + }); + + Ok(Box::new(mapped)) + } + + fn range_vertices(&'a self, offset: Uuid) -> indradb::Result> { + let iter = self + .vertex_manager + .iterate_for_range(offset) + .map(|e| e.map(|v| Vertex::with_id(v.0, v.1))); + Ok(Box::new(iter)) + } + + fn specific_vertices(&'a self, ids: Vec) -> indradb::Result> { + let iter = ids.into_iter().filter_map(move |id| { + let v = self.vertex_manager.get(id).transpose(); + v.map(|v| v.map(|v| Vertex::with_id(id, v))) + }); + Ok(Box::new(iter)) + } + + fn vertex_ids_with_property(&'a self, name: Identifier) -> indradb::Result>> { + let iter = self.vertex_property_manager.iterate_for_property_name(name)?; + let iter = iter.map(|r| r.map(|(id, _)| id.0)); + Ok(Some(Box::new(iter))) + } + + fn vertex_ids_with_property_value( + &'a self, + name: Identifier, + value: &Json, + ) -> indradb::Result>> { + let iter = self + .vertex_property_manager + .iterate_for_property_name_and_value(name, value)?; + let iter = iter.map(|r| r.map(|(id, _)| id.0)); + Ok(Some(Box::new(iter))) + } + + fn edge_count(&self) -> u64 { + let edge_manager = EdgeManager::new(self.holder); + edge_manager.count() + } + + fn all_edges(&'a self) -> indradb::Result> { + let iter = self.edge_range_manager.iterate_for_all(); + + Ok(Box::new(iter)) + } + + fn range_edges(&'a self, offset: Edge) -> indradb::Result> { + let iter = self.edge_range_manager.iterate_for_range(&offset); + + Ok(Box::new(iter)) + } + + fn range_reversed_edges(&'a self, offset: Edge) -> indradb::Result> { + let iter = self.edge_range_manager_rev.iterate_for_range(&offset); + + Ok(Box::new(iter)) + } + + fn specific_edges(&'a self, edges: Vec) -> indradb::Result> { + let iter: Vec<_> = edges + .into_iter() + .filter(|e| { + let r = self.edge_range_manager.contains(e); + if let Ok(r) = r { + r + } else { + false + } + }) + .map(Ok) + .collect(); + Ok(Box::new(iter.into_iter())) + } + + fn edges_with_property(&'a self, name: Identifier) -> indradb::Result>> { + let iter = self.edge_property_manager.iterate_for_property_name(name)?; + Ok(Some(Box::new(iter))) + } + + fn edges_with_property_value( + &'a self, + name: Identifier, + value: &Json, + ) -> indradb::Result>> { + let iter = self + .edge_property_manager + .iterate_for_property_name_and_value(name, value)?; + Ok(Some(Box::new(iter))) + } + + fn vertex_property(&self, vertex: &Vertex, name: Identifier) -> indradb::Result> { + let r = self.vertex_property_manager.get(vertex.id, name)?; + Ok(r.map(|v| v.into())) + } + + fn all_vertex_properties_for_vertex(&'a self, vertex: &Vertex) -> indradb::Result> { + let iter = self.vertex_property_manager.iterate_for_owner(vertex.id)?; + let iter = iter.map(|r| r.map(|((_, name), val)| (name, Json::new(val)))); + Ok(Box::new(iter)) + } + + fn edge_property(&self, edge: &Edge, name: Identifier) -> indradb::Result> { + let result = self + .edge_property_manager + .get(edge.outbound_id, edge.t, edge.inbound_id, name)?; + Ok(result.map(Json::new)) + } + + fn all_edge_properties_for_edge(&'a self, edge: &Edge) -> indradb::Result> { + let iter = self.edge_property_manager.iterate_for_owner(edge)?; + let iter = iter.map(|e| e.map(|((_, id), val)| (id, Json::new(val)))); + Ok(Box::new(iter)) + } + + fn delete_vertices(&mut self, vertices: Vec) -> indradb::Result<()> { + for v in vertices { + self.vertex_manager.delete(v.id)? + } + Ok(()) + } + + fn delete_edges(&mut self, edges: Vec) -> indradb::Result<()> { + for item in edges.iter() { + if self.vertex_manager.get(item.outbound_id)?.is_some() { + self.edge_manager.delete(item)?; + }; + } + + Ok(()) + } + + fn delete_vertex_properties(&mut self, props: Vec<(Uuid, Identifier)>) -> indradb::Result<()> { + for (id, prop) in props { + self.vertex_property_manager.delete(id, prop)? + } + Ok(()) + } + + fn delete_edge_properties(&mut self, props: Vec<(Edge, Identifier)>) -> indradb::Result<()> { + for (edge, prop) in props { + self.edge_property_manager + .delete(edge.outbound_id, edge.t, edge.inbound_id, prop)?; + } + Ok(()) + } + + fn create_vertex(&mut self, vertex: &Vertex) -> indradb::Result { + self.vertex_manager.create(vertex) + } + + fn create_edge(&mut self, edge: &Edge) -> indradb::Result { + let outbound_exists = self.vertex_manager.exists(edge.outbound_id)?; + let inbound_exists = self.vertex_manager.exists(edge.inbound_id)?; + + if !outbound_exists || !inbound_exists { + Ok(false) + } else { + self.edge_manager.set(edge)?; + Ok(true) + } + } + + fn index_property(&mut self, _name: Identifier) -> indradb::Result<()> { + // oh boy we really want this i guess + Ok(()) + } + + fn set_vertex_properties(&mut self, vertices: Vec, name: Identifier, value: &Json) -> indradb::Result<()> { + for v in vertices { + self.vertex_property_manager.set(v, name, value)?; + } + Ok(()) + } + + fn set_edge_properties(&mut self, edges: Vec, name: Identifier, value: &Json) -> indradb::Result<()> { + for edge in edges { + self.edge_property_manager + .set(edge.outbound_id, edge.t, edge.inbound_id, name, value)?; + } + Ok(()) + } +} From b7349991fbecb6e5734549f924a1cf59eb787a41 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Tue, 9 Jul 2024 09:20:28 +0200 Subject: [PATCH 15/28] make API coherent with rocks datastore --- src/datastore.rs | 2 +- src/lib.rs | 45 ++++++++++++++++----------------------------- 2 files changed, 17 insertions(+), 30 deletions(-) diff --git a/src/datastore.rs b/src/datastore.rs index 89e8cb3..402e212 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -96,7 +96,7 @@ impl SledDatastore { /// /// # Arguments /// * `path`: The file path to the Sled database. - pub fn new>(path: P) -> Result { + pub fn new_db(path: impl AsRef) -> Result { Ok(SledDatastore { holder: SledHolder::new(path, SledConfig::default())?, }) diff --git a/src/lib.rs b/src/lib.rs index b1dc516..dad34ee 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,13 +1,7 @@ //! The Sled datastore implementation. -#![cfg_attr(feature = "bench-suite", feature(test))] - extern crate chrono; extern crate ecow; -#[cfg(any(feature = "bench-suite", feature = "test-suite"))] -#[macro_use] -extern crate indradb; -#[cfg(not(any(feature = "bench-suite", feature = "test-suite")))] extern crate indradb; extern crate serde_json; extern crate sled; @@ -22,43 +16,36 @@ mod errors; mod managers; mod transaction; -mod normal_config { +#[cfg(feature = "bench-suite")] +mod bench { + use indradb::full_bench_impl; + + use super::*; - #[cfg(feature = "bench-suite")] full_bench_impl!({ - use super::SledDatastore; - use indradb::Database; - use tempfile::tempdir; let path = tempdir().unwrap().into_path(); Database::new(SledDatastore::new(path).unwrap()) }); - - #[cfg(feature = "test-suite")] - full_test_impl!({ - use super::SledDatastore; - use indradb::Database; - use tempfile::tempdir; + full_bench_impl!({ let path = tempdir().unwrap().into_path(); - Database::new(SledDatastore::new(path).unwrap()) + Database::new(SledConfig::with_compression(None).open(path).unwrap()) }); } -mod compression_config { +#[cfg(test)] +#[cfg(feature = "test-suite")] +mod test { + use indradb::full_test_impl; + use tempfile::tempdir; - #[cfg(feature = "bench-suite")] - full_bench_impl!({ - use super::SledConfig; - use indradb::Database; - use tempfile::tempdir; + use super::*; + + full_test_impl!({ let path = tempdir().unwrap().into_path(); - Database::new(SledConfig::with_compression(None).open(path).unwrap()) + Database::new(SledDatastore::new(path).unwrap()) }); - #[cfg(feature = "test-suite")] full_test_impl!({ - use super::SledConfig; - use indradb::Database; - use tempfile::tempdir; let path = tempdir().unwrap().into_path(); Database::new(SledConfig::with_compression(None).open(path).unwrap()) }); From 7660d855afaf7ea5150d07bd39247a25e964f2d0 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Tue, 9 Jul 2024 09:46:46 +0200 Subject: [PATCH 16/28] Revert "make API coherent with rocks datastore" This reverts commit b7349991fbecb6e5734549f924a1cf59eb787a41. --- src/datastore.rs | 2 +- src/lib.rs | 45 +++++++++++++++++++++++++++++---------------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/datastore.rs b/src/datastore.rs index 402e212..89e8cb3 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -96,7 +96,7 @@ impl SledDatastore { /// /// # Arguments /// * `path`: The file path to the Sled database. - pub fn new_db(path: impl AsRef) -> Result { + pub fn new>(path: P) -> Result { Ok(SledDatastore { holder: SledHolder::new(path, SledConfig::default())?, }) diff --git a/src/lib.rs b/src/lib.rs index dad34ee..b1dc516 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,13 @@ //! The Sled datastore implementation. +#![cfg_attr(feature = "bench-suite", feature(test))] + extern crate chrono; extern crate ecow; +#[cfg(any(feature = "bench-suite", feature = "test-suite"))] +#[macro_use] +extern crate indradb; +#[cfg(not(any(feature = "bench-suite", feature = "test-suite")))] extern crate indradb; extern crate serde_json; extern crate sled; @@ -16,36 +22,43 @@ mod errors; mod managers; mod transaction; -#[cfg(feature = "bench-suite")] -mod bench { - use indradb::full_bench_impl; - - use super::*; +mod normal_config { + #[cfg(feature = "bench-suite")] full_bench_impl!({ + use super::SledDatastore; + use indradb::Database; + use tempfile::tempdir; let path = tempdir().unwrap().into_path(); Database::new(SledDatastore::new(path).unwrap()) }); - full_bench_impl!({ + + #[cfg(feature = "test-suite")] + full_test_impl!({ + use super::SledDatastore; + use indradb::Database; + use tempfile::tempdir; let path = tempdir().unwrap().into_path(); - Database::new(SledConfig::with_compression(None).open(path).unwrap()) + Database::new(SledDatastore::new(path).unwrap()) }); } -#[cfg(test)] -#[cfg(feature = "test-suite")] -mod test { - use indradb::full_test_impl; - use tempfile::tempdir; +mod compression_config { - use super::*; - - full_test_impl!({ + #[cfg(feature = "bench-suite")] + full_bench_impl!({ + use super::SledConfig; + use indradb::Database; + use tempfile::tempdir; let path = tempdir().unwrap().into_path(); - Database::new(SledDatastore::new(path).unwrap()) + Database::new(SledConfig::with_compression(None).open(path).unwrap()) }); + #[cfg(feature = "test-suite")] full_test_impl!({ + use super::SledConfig; + use indradb::Database; + use tempfile::tempdir; let path = tempdir().unwrap().into_path(); Database::new(SledConfig::with_compression(None).open(path).unwrap()) }); From dcbc468d1efff8f13ffa8d5afc965d1bf95f1566 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Tue, 9 Jul 2024 09:50:13 +0200 Subject: [PATCH 17/28] Change more API to edge-model --- src/managers/edge_manager.rs | 2 +- src/managers/edge_property_manager.rs | 59 ++++++++------------------- src/transaction.rs | 10 ++--- 3 files changed, 20 insertions(+), 51 deletions(-) diff --git a/src/managers/edge_manager.rs b/src/managers/edge_manager.rs index 3321bf8..10996b0 100644 --- a/src/managers/edge_manager.rs +++ b/src/managers/edge_manager.rs @@ -57,7 +57,7 @@ impl<'db, 'tree> EdgeManager<'db, 'tree> { for item in edge_property_manager.iterate_for_owner(edge)? { let ((edge, id), _) = item?; - edge_property_manager.delete(edge.outbound_id, edge.t, edge.inbound_id, id)?; + edge_property_manager.delete(&edge, id)?; } Ok(()) } diff --git a/src/managers/edge_property_manager.rs b/src/managers/edge_property_manager.rs index 7693c5c..c74b128 100644 --- a/src/managers/edge_property_manager.rs +++ b/src/managers/edge_property_manager.rs @@ -4,7 +4,6 @@ use ecow::EcoVec; use indradb::{Edge, Identifier, Json, util}; use serde_json::{Value as JsonValue, Value}; use sled::{IVec, Tree}; -use uuid::Uuid; use errors::map_err; @@ -20,11 +19,11 @@ impl<'tree> EdgePropertyManager<'tree> { EdgePropertyManager { tree, value_index_tree } } - fn key(&self, outbound_id: Uuid, t: Identifier, inbound_id: Uuid, name: Identifier) -> Vec { + fn key(&self, edge: &Edge, name: Identifier) -> Vec { util::build(&[ - util::Component::Uuid(outbound_id), - util::Component::Identifier(t), - util::Component::Uuid(inbound_id), + util::Component::Uuid(edge.outbound_id), + util::Component::Identifier(edge.t), + util::Component::Uuid(edge.inbound_id), util::Component::Identifier(name), ]) } @@ -115,14 +114,8 @@ impl<'tree> EdgePropertyManager<'tree> { Ok(Box::new(mapped)) } - pub fn get( - &self, - outbound_id: Uuid, - t: Identifier, - inbound_id: Uuid, - name: Identifier, - ) -> indradb::Result> { - let key = self.key(outbound_id, t, inbound_id, name); + pub fn get(&self, edge: &Edge, name: Identifier) -> indradb::Result> { + let key = self.key(edge, name); match map_err(self.tree.get(key))? { Some(ref value_bytes) => Ok(Some(serde_json::from_slice(value_bytes)?)), @@ -130,7 +123,7 @@ impl<'tree> EdgePropertyManager<'tree> { } } - fn key_value_index(edge: Edge, value: &JsonValue, property_name: Identifier) -> Vec { + fn key_value_index(edge: &Edge, value: &JsonValue, property_name: Identifier) -> Vec { util::build(&[ util::Component::Identifier(property_name), util::Component::Json(&Json::new(value.clone())), @@ -158,26 +151,11 @@ impl<'tree> EdgePropertyManager<'tree> { ) } - pub fn set( - &self, - outbound_id: Uuid, - t: Identifier, - inbound_id: Uuid, - name: Identifier, - value: &JsonValue, - ) -> indradb::Result<()> { - let key = self.key(outbound_id, t, inbound_id, name); + pub fn set(&self, edge: &Edge, name: Identifier, value: &JsonValue) -> indradb::Result<()> { + let key = self.key(edge, name); let value_json = serde_json::to_vec(value)?; map_err(self.tree.insert(key.as_slice(), value_json.as_slice()))?; - let value_key = Self::key_value_index( - Edge { - outbound_id, - t, - inbound_id, - }, - value, - name, - ); + let value_key = Self::key_value_index(edge, value, name); map_err( self.value_index_tree .insert(value_key.as_slice(), value_json.as_slice()), @@ -185,19 +163,14 @@ impl<'tree> EdgePropertyManager<'tree> { Ok(()) } - pub fn delete(&self, outbound_id: Uuid, t: Identifier, inbound_id: Uuid, name: Identifier) -> indradb::Result<()> { - let owner = Edge { - outbound_id, - t, - inbound_id, - }; - map_err(self.tree.remove(self.key(outbound_id, t, inbound_id, name)))?; - let prefix = util::build(&[util::Component::Identifier(t)]); + pub fn delete(&self, edge: &Edge, name: Identifier) -> indradb::Result<()> { + map_err(self.tree.remove(self.key(edge, name)))?; + let prefix = util::build(&[util::Component::Identifier(edge.t)]); let mut edges_to_delete = EcoVec::new(); - for edge in self.value_index_tree.scan_prefix(prefix) { - let (k, _) = map_err(edge)?; + for current_edge in self.value_index_tree.scan_prefix(prefix) { + let (k, _) = map_err(current_edge)?; let (_n, _, vid) = Self::read_key_value_index(k.clone()); - if vid == owner { + if vid == *edge { edges_to_delete.push(k); } } diff --git a/src/transaction.rs b/src/transaction.rs index 3e68f23..aa44f8d 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -137,9 +137,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } fn edge_property(&self, edge: &Edge, name: Identifier) -> indradb::Result> { - let result = self - .edge_property_manager - .get(edge.outbound_id, edge.t, edge.inbound_id, name)?; + let result = self.edge_property_manager.get(edge, name)?; Ok(result.map(Json::new)) } @@ -175,8 +173,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { fn delete_edge_properties(&mut self, props: Vec<(Edge, Identifier)>) -> indradb::Result<()> { for (edge, prop) in props { - self.edge_property_manager - .delete(edge.outbound_id, edge.t, edge.inbound_id, prop)?; + self.edge_property_manager.delete(&edge, prop)?; } Ok(()) } @@ -211,8 +208,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { fn set_edge_properties(&mut self, edges: Vec, name: Identifier, value: &Json) -> indradb::Result<()> { for edge in edges { - self.edge_property_manager - .set(edge.outbound_id, edge.t, edge.inbound_id, name, value)?; + self.edge_property_manager.set(&edge, name, value)?; } Ok(()) } From f491cafc8975a8611a1cef489559119a5dce254a Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Tue, 9 Jul 2024 09:57:01 +0200 Subject: [PATCH 18/28] Fix edge property deletion --- src/managers/edge_property_manager.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/managers/edge_property_manager.rs b/src/managers/edge_property_manager.rs index c74b128..dee0e21 100644 --- a/src/managers/edge_property_manager.rs +++ b/src/managers/edge_property_manager.rs @@ -156,6 +156,10 @@ impl<'tree> EdgePropertyManager<'tree> { let value_json = serde_json::to_vec(value)?; map_err(self.tree.insert(key.as_slice(), value_json.as_slice()))?; let value_key = Self::key_value_index(edge, value, name); + eprintln!( + "Inserting edge property {:?}, into edge {edge:?}, value_key: {value_key:?}", + name + ); map_err( self.value_index_tree .insert(value_key.as_slice(), value_json.as_slice()), @@ -165,7 +169,7 @@ impl<'tree> EdgePropertyManager<'tree> { pub fn delete(&self, edge: &Edge, name: Identifier) -> indradb::Result<()> { map_err(self.tree.remove(self.key(edge, name)))?; - let prefix = util::build(&[util::Component::Identifier(edge.t)]); + let prefix = util::build(&[util::Component::Identifier(name)]); let mut edges_to_delete = EcoVec::new(); for current_edge in self.value_index_tree.scan_prefix(prefix) { let (k, _) = map_err(current_edge)?; From 07a6df8926d41a04e202ca6f5beb967620517be2 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Tue, 9 Jul 2024 10:01:17 +0200 Subject: [PATCH 19/28] Add syncing, remove dirty logs --- src/managers/edge_property_manager.rs | 5 +---- src/transaction.rs | 7 ++++++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/managers/edge_property_manager.rs b/src/managers/edge_property_manager.rs index dee0e21..e1a2303 100644 --- a/src/managers/edge_property_manager.rs +++ b/src/managers/edge_property_manager.rs @@ -156,10 +156,7 @@ impl<'tree> EdgePropertyManager<'tree> { let value_json = serde_json::to_vec(value)?; map_err(self.tree.insert(key.as_slice(), value_json.as_slice()))?; let value_key = Self::key_value_index(edge, value, name); - eprintln!( - "Inserting edge property {:?}, into edge {edge:?}, value_key: {value_key:?}", - name - ); + map_err( self.value_index_tree .insert(value_key.as_slice(), value_json.as_slice()), diff --git a/src/transaction.rs b/src/transaction.rs index aa44f8d..ef0b6a1 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -2,6 +2,7 @@ use indradb::{DynIter, Edge, Error, Identifier, Json, Transaction, Vertex}; use uuid::Uuid; use datastore::SledHolder; +use errors::map_err; use managers::edge_manager::EdgeManager; use managers::edge_property_manager::EdgePropertyManager; use managers::edge_range_manager::EdgeRangeManager; @@ -24,7 +25,6 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { let vertex_manager = VertexManager::new(self.holder); vertex_manager.count() } - fn all_vertices(&'a self) -> indradb::Result> { let iterator = self.vertex_manager.iterate_for_range(Uuid::default()); let mapped = iterator.map(move |item| { @@ -178,6 +178,11 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { Ok(()) } + fn sync(&self) -> indradb::Result<()> { + let _ = map_err(self.holder.db.flush())?; + Ok(()) + } + fn create_vertex(&mut self, vertex: &Vertex) -> indradb::Result { self.vertex_manager.create(vertex) } From 7a7b64078b9560264a120711f7497523cb56449c Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Tue, 9 Jul 2024 11:27:54 +0200 Subject: [PATCH 20/28] Remove some bugs in reverse edge indexing --- src/lib.rs | 10 ++++++++++ src/managers/edge_manager.rs | 5 +++-- src/managers/vertex_manager.rs | 8 -------- src/transaction.rs | 1 + 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b1dc516..77f46e0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,8 @@ extern crate sled; extern crate tempfile; extern crate uuid; +use indradb::Edge; + pub use self::datastore::{SledConfig, SledDatastore}; mod datastore; @@ -63,3 +65,11 @@ mod compression_config { Database::new(SledConfig::with_compression(None).open(path).unwrap()) }); } + +fn reverse_edge(edge: &Edge) -> Edge { + Edge { + outbound_id: edge.inbound_id, + t: edge.t.clone(), + inbound_id: edge.outbound_id, + } +} diff --git a/src/managers/edge_manager.rs b/src/managers/edge_manager.rs index 10996b0..64ada6b 100644 --- a/src/managers/edge_manager.rs +++ b/src/managers/edge_manager.rs @@ -3,6 +3,7 @@ use sled::{IVec, Tree}; use datastore::SledHolder; use errors::map_err; +use reverse_edge; use crate::managers::edge_property_manager::EdgePropertyManager; use crate::managers::edge_range_manager::EdgeRangeManager; @@ -39,7 +40,7 @@ impl<'db, 'tree> EdgeManager<'db, 'tree> { let key = self.key(edge.clone()); map_err(self.tree.insert(key, IVec::default()))?; edge_range_manager.set(edge)?; - reversed_edge_range_manager.set(edge)?; + reversed_edge_range_manager.set(&reverse_edge(&edge))?; Ok(()) } @@ -50,7 +51,7 @@ impl<'db, 'tree> EdgeManager<'db, 'tree> { edge_range_manager.delete(edge)?; let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); - reversed_edge_range_manager.delete(edge)?; + reversed_edge_range_manager.delete(&reverse_edge(&edge))?; let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties, &self.holder.edge_property_values); diff --git a/src/managers/vertex_manager.rs b/src/managers/vertex_manager.rs index e7c37c6..cefb52e 100644 --- a/src/managers/vertex_manager.rs +++ b/src/managers/vertex_manager.rs @@ -105,14 +105,6 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { } } - { - let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); - for item in reversed_edge_range_manager.iterate_for_owner(id) { - let edge = item?; - debug_assert_eq!(edge.inbound_id, id); - edge_manager.delete(&edge)?; - } - } Ok(()) } } diff --git a/src/transaction.rs b/src/transaction.rs index ef0b6a1..cd3b082 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -8,6 +8,7 @@ use managers::edge_property_manager::EdgePropertyManager; use managers::edge_range_manager::EdgeRangeManager; use managers::vertex_manager::VertexManager; use managers::vertex_property_manager::VertexPropertyManager; +use reverse_edge; /// A transaction that is backed by Sled. pub struct SledTransaction<'a> { From 405ee166053193e29cc742f9219872db90f631af Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Tue, 9 Jul 2024 14:34:02 +0200 Subject: [PATCH 21/28] Cleanup key parsing for easier understanding. --- src/managers/edge_property_manager.rs | 38 +++++++++++---------------- src/transaction.rs | 1 - 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/src/managers/edge_property_manager.rs b/src/managers/edge_property_manager.rs index e1a2303..0d14dfd 100644 --- a/src/managers/edge_property_manager.rs +++ b/src/managers/edge_property_manager.rs @@ -28,6 +28,20 @@ impl<'tree> EdgePropertyManager<'tree> { ]) } + fn read_key(buf: IVec) -> (Edge, Identifier) { + let mut cursor = Cursor::new(buf.as_ref()); + let edge_property_outbound_id = util::read_uuid(&mut cursor); + let edge_property_t = util::read_identifier(&mut cursor); + let edge_property_inbound_id = util::read_uuid(&mut cursor); + let edge_property_name = util::read_identifier(&mut cursor); + let edge = Edge { + outbound_id: edge_property_outbound_id, + t: edge_property_t, + inbound_id: edge_property_inbound_id, + }; + (edge, edge_property_name) + } + pub fn iterate_for_property_name( &self, name: Identifier, @@ -84,31 +98,11 @@ impl<'tree> EdgePropertyManager<'tree> { ]); let iterator = self.tree.scan_prefix(prefix); - let mapped = iterator.map(move |item| -> indradb::Result { let (k, v) = map_err(item)?; - let mut cursor = Cursor::new(k); - - let edge_property_outbound_id = util::read_uuid(&mut cursor); - - let edge_property_t = util::read_identifier(&mut cursor); - - let edge_property_inbound_id = util::read_uuid(&mut cursor); - - let edge_property_name = util::read_identifier(&mut cursor); - + let (edge, p_name) = Self::read_key(k); let value = serde_json::from_slice(&v)?; - Ok(( - ( - Edge { - outbound_id: edge_property_outbound_id, - t: edge_property_t, - inbound_id: edge_property_inbound_id, - }, - edge_property_name, - ), - value, - )) + Ok(((edge, p_name), value)) }); Ok(Box::new(mapped)) diff --git a/src/transaction.rs b/src/transaction.rs index cd3b082..ef0b6a1 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -8,7 +8,6 @@ use managers::edge_property_manager::EdgePropertyManager; use managers::edge_range_manager::EdgeRangeManager; use managers::vertex_manager::VertexManager; use managers::vertex_property_manager::VertexPropertyManager; -use reverse_edge; /// A transaction that is backed by Sled. pub struct SledTransaction<'a> { From aad11df2776fac4948673ae578051ab36766e8bf Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Tue, 9 Jul 2024 15:35:55 +0200 Subject: [PATCH 22/28] Add metadata storage for indexed values. Fixup all unit tests in the suite. --- Cargo.toml | 3 +- src/datastore.rs | 4 ++ src/errors.rs | 20 ++++++ src/lib.rs | 3 +- src/managers/edge_manager.rs | 4 +- src/managers/edge_property_manager.rs | 8 +++ src/managers/metadata.rs | 95 +++++++++++++++++++++++++ src/managers/mod.rs | 1 + src/managers/vertex_property_manager.rs | 7 ++ src/transaction.rs | 19 ++++- 10 files changed, 158 insertions(+), 6 deletions(-) create mode 100644 src/managers/metadata.rs diff --git a/Cargo.toml b/Cargo.toml index bb76298..942610b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,4 +25,5 @@ serde_json = "1.0" sled = { version = "0.34", features = ["compression"] } tempfile = { version = "3.10", optional = true } uuid = { version = "1.9", features = ["v1", "serde"] } -ecow = { version = "0.2.2" } \ No newline at end of file +ecow = { version = "0.2.2" } +thiserror = { version = "1.0" } \ No newline at end of file diff --git a/src/datastore.rs b/src/datastore.rs index 89e8cb3..f8a4db2 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -5,6 +5,7 @@ use sled::{Config, Db, Tree}; use managers::edge_manager::EdgeManager; use managers::edge_range_manager::EdgeRangeManager; +use managers::metadata::MetaDataManager; use managers::vertex_property_manager::VertexPropertyManager; use transaction::SledTransaction; @@ -52,6 +53,7 @@ pub struct SledHolder { pub(crate) edge_property_values: Tree, // for prop-name -> value -> UUID prefix-indexed lookup pub(crate) vertex_property_values: Tree, + pub(crate) metadata: Tree, } impl SledHolder { @@ -81,6 +83,7 @@ impl SledHolder { edge_properties: map_err(db.open_tree("edge_properties"))?, vertex_property_values: map_err(db.open_tree("vertex_property_values"))?, edge_property_values: map_err(db.open_tree("edge_property_values"))?, + metadata: map_err(db.open_tree("metadata"))?, db, }) } @@ -123,6 +126,7 @@ impl Datastore for SledDatastore { &self.holder.vertex_properties, &self.holder.vertex_property_values, ), + meta_data_manager: MetaDataManager::new(&self.holder.metadata).unwrap(), } } } diff --git a/src/errors.rs b/src/errors.rs index 406b339..78b0582 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,6 +1,26 @@ +use std::sync::PoisonError; + use indradb::Error as IndraError; use sled::Error as SledError; pub(crate) fn map_err(result: Result) -> Result { result.map_err(|err| IndraError::Datastore(Box::new(err))) } + +#[derive(Debug, thiserror::Error)] +pub enum DSError { + #[error("Error in locking a RwLock: {0}")] + PoisonError(String), +} + +impl From> for DSError { + fn from(value: PoisonError) -> Self { + DSError::PoisonError(value.to_string()) + } +} + +impl From for IndraError { + fn from(err: DSError) -> Self { + IndraError::Datastore(Box::new(err)) + } +} diff --git a/src/lib.rs b/src/lib.rs index 77f46e0..58240dd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,7 @@ extern crate serde_json; extern crate sled; #[cfg(any(feature = "bench-suite", feature = "test-suite"))] extern crate tempfile; +extern crate thiserror; extern crate uuid; use indradb::Edge; @@ -69,7 +70,7 @@ mod compression_config { fn reverse_edge(edge: &Edge) -> Edge { Edge { outbound_id: edge.inbound_id, - t: edge.t.clone(), + t: edge.t, inbound_id: edge.outbound_id, } } diff --git a/src/managers/edge_manager.rs b/src/managers/edge_manager.rs index 64ada6b..c16bdee 100644 --- a/src/managers/edge_manager.rs +++ b/src/managers/edge_manager.rs @@ -40,7 +40,7 @@ impl<'db, 'tree> EdgeManager<'db, 'tree> { let key = self.key(edge.clone()); map_err(self.tree.insert(key, IVec::default()))?; edge_range_manager.set(edge)?; - reversed_edge_range_manager.set(&reverse_edge(&edge))?; + reversed_edge_range_manager.set(&reverse_edge(edge))?; Ok(()) } @@ -51,7 +51,7 @@ impl<'db, 'tree> EdgeManager<'db, 'tree> { edge_range_manager.delete(edge)?; let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); - reversed_edge_range_manager.delete(&reverse_edge(&edge))?; + reversed_edge_range_manager.delete(&reverse_edge(edge))?; let edge_property_manager = EdgePropertyManager::new(&self.holder.edge_properties, &self.holder.edge_property_values); diff --git a/src/managers/edge_property_manager.rs b/src/managers/edge_property_manager.rs index 0d14dfd..687772d 100644 --- a/src/managers/edge_property_manager.rs +++ b/src/managers/edge_property_manager.rs @@ -148,6 +148,14 @@ impl<'tree> EdgePropertyManager<'tree> { pub fn set(&self, edge: &Edge, name: Identifier, value: &JsonValue) -> indradb::Result<()> { let key = self.key(edge, name); let value_json = serde_json::to_vec(value)?; + + let old_value = map_err(self.tree.get(key.clone()))?; + if let Some(old_value) = old_value { + let old_value: Json = serde_json::from_slice(&old_value)?; + let value_key = Self::key_value_index(edge, &old_value, name); + map_err(self.value_index_tree.remove(value_key.as_slice()))?; + } + map_err(self.tree.insert(key.as_slice(), value_json.as_slice()))?; let value_key = Self::key_value_index(edge, value, name); diff --git a/src/managers/metadata.rs b/src/managers/metadata.rs new file mode 100644 index 0000000..e214db5 --- /dev/null +++ b/src/managers/metadata.rs @@ -0,0 +1,95 @@ +use std::collections::HashSet; +use std::io::Cursor; +use std::sync::{Arc, RwLock}; + +use indradb::{Identifier, util}; +use sled::Tree; + +use errors::{DSError, map_err}; + +const INDEXED_PROPERTIES: &str = "IndexedProperties"; + +pub struct MetaDataManager<'tree> { + pub tree: &'tree Tree, + indexed_properties: Arc>>, + index_key: Identifier, +} + +impl<'tree> MetaDataManager<'tree> { + pub fn new(tree: &'tree Tree) -> indradb::Result { + let manager = MetaDataManager { + tree, + indexed_properties: Arc::new(RwLock::new(HashSet::new())), + index_key: Identifier::new(INDEXED_PROPERTIES)?, + }; + manager.load()?; + Ok(manager) + } + + pub fn is_indexed(&self, prop: &Identifier) -> indradb::Result { + let indexed_properties = self.indexed_properties.read().map_err(DSError::from)?; + eprintln!("Asked for indexing: {}", prop.as_str()); + eprintln!( + "Knowing indexes: {:?}", + indexed_properties.iter().collect::>() + ); + let is_indexed = indexed_properties.contains(prop.as_str()); + Ok(is_indexed) + } + + pub fn add_index(&self, prop: &Identifier) -> indradb::Result<()> { + eprintln!("Adding index: {}", prop.as_str()); + { + let mut indexed_properties = self.indexed_properties.write().map_err(DSError::from)?; + if indexed_properties.contains(prop.as_str()) { + return Ok(()); + } + indexed_properties.insert(prop.to_string()); + } + self.sync()?; + Ok(()) + } + + pub fn remove_index(&self, prop: &Identifier) -> indradb::Result<()> { + { + let mut indexed_properties = self.indexed_properties.write().map_err(DSError::from)?; + if !indexed_properties.contains(prop.as_str()) { + return Ok(()); + } + + indexed_properties.remove(prop.as_str()); + } + self.sync()?; + Ok(()) + } + + fn load(&self) -> indradb::Result<()> { + let mut indexed_properties = self.indexed_properties.write().map_err(DSError::from)?; + let all_indexed_prefix = util::build(&[util::Component::Identifier(self.index_key)]); + for index in self.tree.scan_prefix(all_indexed_prefix) { + let (k, _) = map_err(index)?; + let mut cursor = Cursor::new(k); + let _ = util::read_identifier(&mut cursor); + let prop = util::read_identifier(&mut cursor); + + indexed_properties.insert(prop.to_string()); + } + Ok(()) + } + + pub(crate) fn sync(&self) -> indradb::Result<()> { + let all_indexed_prefix = util::build(&[util::Component::Identifier(self.index_key)]); + for index in self.tree.scan_prefix(all_indexed_prefix) { + let (key, _) = map_err(index)?; + map_err(self.tree.remove(key))?; + } + for index in self.indexed_properties.read().map_err(DSError::from)?.iter() { + let key = util::build(&[ + util::Component::Identifier(self.index_key), + util::Component::Identifier(Identifier::new(index)?), + ]); + map_err(self.tree.insert(key, &[]))?; + } + Ok(()) + } +} diff --git a/src/managers/mod.rs b/src/managers/mod.rs index c620a73..775f518 100644 --- a/src/managers/mod.rs +++ b/src/managers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod edge_manager; pub(crate) mod edge_property_manager; pub(crate) mod edge_range_manager; +pub(crate) mod metadata; pub(crate) mod vertex_manager; pub(crate) mod vertex_property_manager; diff --git a/src/managers/vertex_property_manager.rs b/src/managers/vertex_property_manager.rs index aa6428b..3da1654 100644 --- a/src/managers/vertex_property_manager.rs +++ b/src/managers/vertex_property_manager.rs @@ -104,6 +104,13 @@ impl<'tree> VertexPropertyManager<'tree> { pub fn set(&self, vertex_id: Uuid, name: Identifier, value: &JsonValue) -> indradb::Result<()> { let key = self.key(vertex_id, name); let value_json = serde_json::to_vec(value)?; + + if let Some(old) = map_err(self.tree.get(key.clone()))? { + let old_value = serde_json::from_slice(&old)?; + let value_index_key = Self::key_value_index(&vertex_id, &old_value, name); + map_err(self.value_index_tree.remove(value_index_key))?; + } + map_err(self.tree.insert(key.as_slice(), value_json.as_slice()))?; let value_index_key = Self::key_value_index(&vertex_id, value, name); map_err(self.value_index_tree.insert(value_index_key, value_json.as_slice()))?; diff --git a/src/transaction.rs b/src/transaction.rs index ef0b6a1..5aee525 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -6,6 +6,7 @@ use errors::map_err; use managers::edge_manager::EdgeManager; use managers::edge_property_manager::EdgePropertyManager; use managers::edge_range_manager::EdgeRangeManager; +use managers::metadata::MetaDataManager; use managers::vertex_manager::VertexManager; use managers::vertex_property_manager::VertexPropertyManager; @@ -18,6 +19,7 @@ pub struct SledTransaction<'a> { pub(crate) vertex_property_manager: VertexPropertyManager<'a>, pub(crate) edge_range_manager: EdgeRangeManager<'a>, pub(crate) edge_range_manager_rev: EdgeRangeManager<'a>, + pub(crate) meta_data_manager: MetaDataManager<'a>, } impl<'a> Transaction<'a> for SledTransaction<'a> { @@ -53,6 +55,9 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } fn vertex_ids_with_property(&'a self, name: Identifier) -> indradb::Result>> { + if !self.meta_data_manager.is_indexed(&name)? { + return Ok(None); + } let iter = self.vertex_property_manager.iterate_for_property_name(name)?; let iter = iter.map(|r| r.map(|(id, _)| id.0)); Ok(Some(Box::new(iter))) @@ -63,6 +68,9 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { name: Identifier, value: &Json, ) -> indradb::Result>> { + if !self.meta_data_manager.is_indexed(&name)? { + return Ok(None); + } let iter = self .vertex_property_manager .iterate_for_property_name_and_value(name, value)?; @@ -110,6 +118,9 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } fn edges_with_property(&'a self, name: Identifier) -> indradb::Result>> { + if !self.meta_data_manager.is_indexed(&name)? { + return Ok(None); + } let iter = self.edge_property_manager.iterate_for_property_name(name)?; Ok(Some(Box::new(iter))) } @@ -119,6 +130,9 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { name: Identifier, value: &Json, ) -> indradb::Result>> { + if !self.meta_data_manager.is_indexed(&name)? { + return Ok(None); + } let iter = self .edge_property_manager .iterate_for_property_name_and_value(name, value)?; @@ -179,6 +193,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } fn sync(&self) -> indradb::Result<()> { + self.meta_data_manager.sync()?; let _ = map_err(self.holder.db.flush())?; Ok(()) } @@ -199,8 +214,8 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } } - fn index_property(&mut self, _name: Identifier) -> indradb::Result<()> { - // oh boy we really want this i guess + fn index_property(&mut self, name: Identifier) -> indradb::Result<()> { + self.meta_data_manager.add_index(&name)?; Ok(()) } From 81396dbfb8065f692656aaa0bd1eca23a14d1965 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Tue, 9 Jul 2024 16:06:03 +0200 Subject: [PATCH 23/28] Added batch processing for creating edges, vertices and edge-ranges. Edge properties and vertex properties missing due to complex insertion mechanic. --- README.md | 16 +++++++- src/managers/edge_manager.rs | 18 ++++++++- src/managers/edge_range_manager.rs | 8 +++- src/managers/metadata.rs | 7 +--- src/managers/vertex_manager.rs | 8 +++- src/transaction.rs | 65 +++++++++++++++++++++++++++++- 6 files changed, 111 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index a605476..7b88bdd 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,18 @@ This is an implementation of the IndraDB datastore for sled. -The sled datastore is not production-ready yet. sled itself is pre-1.0, and makes no guarantees about on-disk format stability. Upgrading IndraDB may require you to [manually migrate the sled datastore.](https://docs.rs/sled/0.34.6/sled/struct.Db.html#method.export) Additionally, there is a standing issue that prevents the sled datastore from having the same level of safety as the RocksDB datastore. +## General hints + +The sled datastore is not production-ready yet. sled itself is pre-1.0, and makes no guarantees about on-disk format +stability. +Upgrading IndraDB may require you +to [manually migrate the sled datastore.](https://docs.rs/sled/0.34.6/sled/struct.Db.html#method.export) +Additionally, there is a standing issue that prevents the sled datastore from having the same level of safety as the +RocksDB datastore. +Performance of sled in two-hop-queries is better than rocksdb though. We measured about 40% faster query times. + +## TODO / Notices: + +- This version does index all properties by default (and therefore duplicates the data). + This is adding the benefit of faster queries but also increases the storage requirements. +- Batch operations are not optimized yet, diff --git a/src/managers/edge_manager.rs b/src/managers/edge_manager.rs index c16bdee..4274c2f 100644 --- a/src/managers/edge_manager.rs +++ b/src/managers/edge_manager.rs @@ -1,5 +1,5 @@ use indradb::{Edge, util}; -use sled::{IVec, Tree}; +use sled::{Batch, IVec, Tree}; use datastore::SledHolder; use errors::map_err; @@ -33,6 +33,22 @@ impl<'db, 'tree> EdgeManager<'db, 'tree> { self.tree.iter().count() as u64 } + pub fn set_batch( + &self, + edge: &Edge, + batch: &mut Batch, + range_batch: &mut Batch, + range_rev_batch: &mut Batch, + ) -> indradb::Result<()> { + let key = self.key(edge.clone()); + batch.insert(key, IVec::default()); + let edge_range_manager = EdgeRangeManager::new(self.holder); + edge_range_manager.set_batch(edge, range_batch)?; + let edge_range_manager_rev = EdgeRangeManager::new_reversed(self.holder); + edge_range_manager_rev.set_batch(&reverse_edge(edge), range_rev_batch)?; + Ok(()) + } + pub fn set(&self, edge: &Edge) -> indradb::Result<()> { let edge_range_manager = EdgeRangeManager::new(self.holder); let reversed_edge_range_manager = EdgeRangeManager::new_reversed(self.holder); diff --git a/src/managers/edge_range_manager.rs b/src/managers/edge_range_manager.rs index aa4b694..12f9cb1 100644 --- a/src/managers/edge_range_manager.rs +++ b/src/managers/edge_range_manager.rs @@ -1,7 +1,7 @@ use std::io::Cursor; use indradb::{Edge, util}; -use sled::{Iter as DbIterator, Tree}; +use sled::{Batch, Iter as DbIterator, Tree}; use uuid::Uuid; use datastore::SledHolder; @@ -79,6 +79,12 @@ impl<'tree> EdgeRangeManager<'tree> { Ok(()) } + pub fn set_batch(&self, edge: &Edge, batch: &mut Batch) -> indradb::Result<()> { + let key = self.key(edge); + batch.insert(key, &[]); + Ok(()) + } + pub fn delete(&self, edge: &Edge) -> indradb::Result<()> { map_err(self.tree.remove(self.key(edge)))?; Ok(()) diff --git a/src/managers/metadata.rs b/src/managers/metadata.rs index e214db5..7c6078c 100644 --- a/src/managers/metadata.rs +++ b/src/managers/metadata.rs @@ -28,17 +28,12 @@ impl<'tree> MetaDataManager<'tree> { pub fn is_indexed(&self, prop: &Identifier) -> indradb::Result { let indexed_properties = self.indexed_properties.read().map_err(DSError::from)?; - eprintln!("Asked for indexing: {}", prop.as_str()); - eprintln!( - "Knowing indexes: {:?}", - indexed_properties.iter().collect::>() - ); + let is_indexed = indexed_properties.contains(prop.as_str()); Ok(is_indexed) } pub fn add_index(&self, prop: &Identifier) -> indradb::Result<()> { - eprintln!("Adding index: {}", prop.as_str()); { let mut indexed_properties = self.indexed_properties.write().map_err(DSError::from)?; if indexed_properties.contains(prop.as_str()) { diff --git a/src/managers/vertex_manager.rs b/src/managers/vertex_manager.rs index cefb52e..ba99eb9 100644 --- a/src/managers/vertex_manager.rs +++ b/src/managers/vertex_manager.rs @@ -2,7 +2,7 @@ use std::io::Cursor; use std::ops::Deref; use indradb::{Identifier, util, Vertex}; -use sled::{Iter as DbIterator, Tree}; +use sled::{Batch, Iter as DbIterator, Tree}; use uuid::Uuid; use datastore::SledHolder; @@ -84,6 +84,12 @@ impl<'db: 'tree, 'tree> VertexManager<'db, 'tree> { Ok(true) } + pub fn create_batch(&self, vertex: &Vertex, batch: &mut Batch) -> indradb::Result<()> { + let key = self.key(vertex.id); + batch.insert(key.clone(), util::build(&[util::Component::Identifier(vertex.t)])); + Ok(()) + } + pub fn delete(&self, id: Uuid) -> indradb::Result<()> { map_err(self.tree.remove(self.key(id)))?; diff --git a/src/transaction.rs b/src/transaction.rs index 5aee525..cef8b1a 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -1,4 +1,7 @@ -use indradb::{DynIter, Edge, Error, Identifier, Json, Transaction, Vertex}; +use std::ops::Deref; + +use indradb::{BulkInsertItem, DynIter, Edge, Error, Identifier, Json, Transaction, Vertex}; +use sled::Batch; use uuid::Uuid; use datastore::SledHolder; @@ -10,6 +13,28 @@ use managers::metadata::MetaDataManager; use managers::vertex_manager::VertexManager; use managers::vertex_property_manager::VertexPropertyManager; +#[derive(Default)] +struct IndraSledBatch { + pub(crate) vertex_creation_batch: Batch, + pub(crate) edge_creation_batch: Batch, + pub(crate) edge_range_creation_batch: Batch, + pub(crate) edge_range_rev_creation_batch: Batch, +} + +impl IndraSledBatch { + fn apply(self, holder: &SledHolder) -> indradb::Result<()> { + map_err(holder.db.deref().apply_batch(self.vertex_creation_batch))?; + map_err(holder.edges.apply_batch(self.edge_creation_batch))?; + map_err(holder.edge_ranges.apply_batch(self.edge_range_creation_batch))?; + map_err( + holder + .reversed_edge_ranges + .apply_batch(self.edge_range_rev_creation_batch), + )?; + Ok(()) + } +} + /// A transaction that is backed by Sled. pub struct SledTransaction<'a> { pub(crate) holder: &'a SledHolder, @@ -232,4 +257,42 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } Ok(()) } + + fn bulk_insert(&mut self, items: Vec) -> indradb::Result<()> { + let mut batch = IndraSledBatch::default(); + let mut vertex_props = Vec::new(); + let mut edge_props = Vec::new(); + + for item in items { + match item { + BulkInsertItem::Vertex(v) => { + self.vertex_manager.create_batch(&v, &mut batch.vertex_creation_batch)?; + } + BulkInsertItem::Edge(e) => { + self.edge_manager.set_batch( + &e, + &mut batch.edge_creation_batch, + &mut batch.edge_range_creation_batch, + &mut batch.edge_range_rev_creation_batch, + )?; + } + BulkInsertItem::VertexProperty(id, p, v) => { + vertex_props.push((id, p, v)); + } + BulkInsertItem::EdgeProperty(e, p, v) => { + edge_props.push((e, p, v)); + } + } + } + batch.apply(self.holder)?; + for (id, p, v) in vertex_props { + self.vertex_property_manager.set(id, p, &v)?; + } + + for (e, p, v) in edge_props { + self.edge_property_manager.set(&e, p, &v)?; + } + + Ok(()) + } } From 222412c89bc1e4d9b4deb31c93e75bb765ef7cf2 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Tue, 9 Jul 2024 16:11:58 +0200 Subject: [PATCH 24/28] Bumped rust edition to 2021 --- Cargo.toml | 1 + src/datastore.rs | 11 +++++------ src/managers/edge_manager.rs | 7 +++---- src/managers/edge_property_manager.rs | 2 +- src/managers/edge_range_manager.rs | 4 ++-- src/managers/metadata.rs | 3 ++- src/managers/vertex_manager.rs | 7 +++---- src/managers/vertex_property_manager.rs | 2 +- src/transaction.rs | 18 +++++++++--------- 9 files changed, 27 insertions(+), 28 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 942610b..99be4b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ repository = "https://github.com/indradb/sled" keywords = ["graph", "database"] categories = ["database", "database-implementations"] license = "MPL-2.0" +edition = "2021" [lib] name = "indradb_sled" diff --git a/src/datastore.rs b/src/datastore.rs index f8a4db2..3adeb5c 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -3,14 +3,13 @@ use std::path::Path; use indradb::{Datastore, Result}; use sled::{Config, Db, Tree}; -use managers::edge_manager::EdgeManager; -use managers::edge_range_manager::EdgeRangeManager; -use managers::metadata::MetaDataManager; -use managers::vertex_property_manager::VertexPropertyManager; -use transaction::SledTransaction; - +use crate::managers::edge_manager::EdgeManager; use crate::managers::edge_property_manager::EdgePropertyManager; +use crate::managers::edge_range_manager::EdgeRangeManager; +use crate::managers::metadata::MetaDataManager; use crate::managers::vertex_manager::VertexManager; +use crate::managers::vertex_property_manager::VertexPropertyManager; +use crate::transaction::SledTransaction; use super::errors::map_err; diff --git a/src/managers/edge_manager.rs b/src/managers/edge_manager.rs index 4274c2f..f6b8987 100644 --- a/src/managers/edge_manager.rs +++ b/src/managers/edge_manager.rs @@ -1,12 +1,11 @@ use indradb::{Edge, util}; use sled::{Batch, IVec, Tree}; -use datastore::SledHolder; -use errors::map_err; -use reverse_edge; - +use crate::datastore::SledHolder; +use crate::errors::map_err; use crate::managers::edge_property_manager::EdgePropertyManager; use crate::managers::edge_range_manager::EdgeRangeManager; +use crate::reverse_edge; pub struct EdgeManager<'db: 'tree, 'tree> { pub holder: &'db SledHolder, diff --git a/src/managers/edge_property_manager.rs b/src/managers/edge_property_manager.rs index 687772d..bf09621 100644 --- a/src/managers/edge_property_manager.rs +++ b/src/managers/edge_property_manager.rs @@ -5,7 +5,7 @@ use indradb::{Edge, Identifier, Json, util}; use serde_json::{Value as JsonValue, Value}; use sled::{IVec, Tree}; -use errors::map_err; +use crate::errors::map_err; pub type EdgePropertyItem = ((Edge, Identifier), JsonValue); diff --git a/src/managers/edge_range_manager.rs b/src/managers/edge_range_manager.rs index 12f9cb1..690c23f 100644 --- a/src/managers/edge_range_manager.rs +++ b/src/managers/edge_range_manager.rs @@ -4,8 +4,8 @@ use indradb::{Edge, util}; use sled::{Batch, Iter as DbIterator, Tree}; use uuid::Uuid; -use datastore::SledHolder; -use errors::map_err; +use crate::datastore::SledHolder; +use crate::errors::map_err; pub struct EdgeRangeManager<'tree> { pub tree: &'tree Tree, diff --git a/src/managers/metadata.rs b/src/managers/metadata.rs index 7c6078c..767f373 100644 --- a/src/managers/metadata.rs +++ b/src/managers/metadata.rs @@ -5,7 +5,7 @@ use std::sync::{Arc, RwLock}; use indradb::{Identifier, util}; use sled::Tree; -use errors::{DSError, map_err}; +use crate::errors::{DSError, map_err}; const INDEXED_PROPERTIES: &str = "IndexedProperties"; @@ -45,6 +45,7 @@ impl<'tree> MetaDataManager<'tree> { Ok(()) } + #[allow(dead_code)] pub fn remove_index(&self, prop: &Identifier) -> indradb::Result<()> { { let mut indexed_properties = self.indexed_properties.write().map_err(DSError::from)?; diff --git a/src/managers/vertex_manager.rs b/src/managers/vertex_manager.rs index ba99eb9..c36fbe1 100644 --- a/src/managers/vertex_manager.rs +++ b/src/managers/vertex_manager.rs @@ -5,11 +5,10 @@ use indradb::{Identifier, util, Vertex}; use sled::{Batch, Iter as DbIterator, Tree}; use uuid::Uuid; -use datastore::SledHolder; -use errors::map_err; -use managers::edge_range_manager::EdgeRangeManager; - +use crate::datastore::SledHolder; +use crate::errors::map_err; use crate::managers::edge_manager::EdgeManager; +use crate::managers::edge_range_manager::EdgeRangeManager; use crate::managers::vertex_property_manager::VertexPropertyManager; pub type VertexItem = (Uuid, Identifier); diff --git a/src/managers/vertex_property_manager.rs b/src/managers/vertex_property_manager.rs index 3da1654..7bd94b9 100644 --- a/src/managers/vertex_property_manager.rs +++ b/src/managers/vertex_property_manager.rs @@ -6,7 +6,7 @@ use serde_json::Value as JsonValue; use sled::{IVec, Tree}; use uuid::Uuid; -use errors::map_err; +use crate::errors::map_err; pub type OwnedPropertyItem = ((Uuid, Identifier), JsonValue); diff --git a/src/transaction.rs b/src/transaction.rs index cef8b1a..95ffade 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -4,14 +4,14 @@ use indradb::{BulkInsertItem, DynIter, Edge, Error, Identifier, Json, Transactio use sled::Batch; use uuid::Uuid; -use datastore::SledHolder; -use errors::map_err; -use managers::edge_manager::EdgeManager; -use managers::edge_property_manager::EdgePropertyManager; -use managers::edge_range_manager::EdgeRangeManager; -use managers::metadata::MetaDataManager; -use managers::vertex_manager::VertexManager; -use managers::vertex_property_manager::VertexPropertyManager; +use crate::datastore::SledHolder; +use crate::errors::map_err; +use crate::managers::edge_manager::EdgeManager; +use crate::managers::edge_property_manager::EdgePropertyManager; +use crate::managers::edge_range_manager::EdgeRangeManager; +use crate::managers::metadata::MetaDataManager; +use crate::managers::vertex_manager::VertexManager; +use crate::managers::vertex_property_manager::VertexPropertyManager; #[derive(Default)] struct IndraSledBatch { @@ -292,7 +292,7 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { for (e, p, v) in edge_props { self.edge_property_manager.set(&e, p, &v)?; } - + self.sync()?; Ok(()) } } From 125221daff6339e12794bc49eff0a3662f34cd7a Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Tue, 9 Jul 2024 17:56:09 +0200 Subject: [PATCH 25/28] Improved performance of property/property-value iteration on vertices --- src/managers/edge_property_manager.rs | 32 +++++++------------- src/managers/vertex_property_manager.rs | 27 ++++++++--------- src/transaction.rs | 40 ++++++++++++------------- 3 files changed, 41 insertions(+), 58 deletions(-) diff --git a/src/managers/edge_property_manager.rs b/src/managers/edge_property_manager.rs index bf09621..31d092b 100644 --- a/src/managers/edge_property_manager.rs +++ b/src/managers/edge_property_manager.rs @@ -2,7 +2,7 @@ use std::io::Cursor; use ecow::EcoVec; use indradb::{Edge, Identifier, Json, util}; -use serde_json::{Value as JsonValue, Value}; +use serde_json::Value as JsonValue; use sled::{IVec, Tree}; use crate::errors::map_err; @@ -62,29 +62,17 @@ impl<'tree> EdgePropertyManager<'tree> { value: &JsonValue, ) -> indradb::Result> + 'tree> { let value = value.clone(); - let prefix = util::build(&[util::Component::Identifier(name)]); + let prefix = util::build(&[ + util::Component::Identifier(name), + util::Component::Json(&Json::new(value)), + ]); let iterator = self.value_index_tree.scan_prefix(prefix); - Ok(iterator - .map(move |item| -> indradb::Result<(Edge, Value)> { - let (k, v) = map_err(item)?; - - let (_p, _, edge) = Self::read_key_value_index(k); - let val = serde_json::from_slice(&v)?; - Ok((edge, val)) - }) - .filter_map(move |item| { - item.map_or_else( - |e| Some(Err(e)), - |(edge, val)| { - if val == value { - Some(Ok(edge)) - } else { - None - } - }, - ) - })) + Ok(iterator.map(move |item| -> indradb::Result { + let (k, _) = map_err(item)?; + let (_p, _, edge) = Self::read_key_value_index(k); + Ok(edge) + })) } pub fn iterate_for_owner<'a>( diff --git a/src/managers/vertex_property_manager.rs b/src/managers/vertex_property_manager.rs index 7bd94b9..8ffc586 100644 --- a/src/managers/vertex_property_manager.rs +++ b/src/managers/vertex_property_manager.rs @@ -40,38 +40,35 @@ impl<'tree> VertexPropertyManager<'tree> { (name, value, uuid) } + fn value_iterate_uuids(&self, iterator: sled::Iter) -> impl Iterator> + '_ { + iterator.map(move |item| -> indradb::Result { + let (k, _) = map_err(item)?; + let (_, _, vid) = Self::read_key_value_index(k); + Ok(vid) + }) + } + pub fn iterate_for_property_name( &self, name: Identifier, - ) -> indradb::Result> + '_> { + ) -> indradb::Result> + '_> { let prefix = util::build(&[util::Component::Identifier(name)]); let iterator = self.value_index_tree.scan_prefix(prefix); - - Ok(iterator.map(move |item| -> indradb::Result { - let (k, v) = map_err(item)?; - let (n, _, vid) = Self::read_key_value_index(k); - let value = serde_json::from_slice(&v)?; - Ok(((vid, n), value)) - })) + Ok(self.value_iterate_uuids(iterator)) } pub fn iterate_for_property_name_and_value( &self, name: Identifier, value: &JsonValue, - ) -> indradb::Result> + '_> { + ) -> indradb::Result> + '_> { let prefix = util::build(&[ util::Component::Identifier(name), util::Component::Json(&Json::new(value.clone())), ]); let iterator = self.value_index_tree.scan_prefix(prefix); - Ok(iterator.map(move |item| -> indradb::Result { - let (k, v) = map_err(item)?; - let (n, _, vid) = Self::read_key_value_index(k); - let value = serde_json::from_slice(&v)?; - Ok(((vid, n), value)) - })) + Ok(self.value_iterate_uuids(iterator)) } pub fn iterate_for_owner( diff --git a/src/transaction.rs b/src/transaction.rs index 95ffade..8c1a2a8 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -84,7 +84,6 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { return Ok(None); } let iter = self.vertex_property_manager.iterate_for_property_name(name)?; - let iter = iter.map(|r| r.map(|(id, _)| id.0)); Ok(Some(Box::new(iter))) } @@ -99,7 +98,6 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { let iter = self .vertex_property_manager .iterate_for_property_name_and_value(name, value)?; - let iter = iter.map(|r| r.map(|(id, _)| id.0)); Ok(Some(Box::new(iter))) } @@ -239,25 +237,6 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { } } - fn index_property(&mut self, name: Identifier) -> indradb::Result<()> { - self.meta_data_manager.add_index(&name)?; - Ok(()) - } - - fn set_vertex_properties(&mut self, vertices: Vec, name: Identifier, value: &Json) -> indradb::Result<()> { - for v in vertices { - self.vertex_property_manager.set(v, name, value)?; - } - Ok(()) - } - - fn set_edge_properties(&mut self, edges: Vec, name: Identifier, value: &Json) -> indradb::Result<()> { - for edge in edges { - self.edge_property_manager.set(&edge, name, value)?; - } - Ok(()) - } - fn bulk_insert(&mut self, items: Vec) -> indradb::Result<()> { let mut batch = IndraSledBatch::default(); let mut vertex_props = Vec::new(); @@ -295,4 +274,23 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { self.sync()?; Ok(()) } + + fn index_property(&mut self, name: Identifier) -> indradb::Result<()> { + self.meta_data_manager.add_index(&name)?; + Ok(()) + } + + fn set_vertex_properties(&mut self, vertices: Vec, name: Identifier, value: &Json) -> indradb::Result<()> { + for v in vertices { + self.vertex_property_manager.set(v, name, value)?; + } + Ok(()) + } + + fn set_edge_properties(&mut self, edges: Vec, name: Identifier, value: &Json) -> indradb::Result<()> { + for edge in edges { + self.edge_property_manager.set(&edge, name, value)?; + } + Ok(()) + } } From 70220091e71c3a71f303d6718f3d911b2c83f3c2 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Tue, 9 Jul 2024 18:37:22 +0200 Subject: [PATCH 26/28] Cargo fmt --- src/managers/edge_manager.rs | 2 +- src/managers/edge_property_manager.rs | 2 +- src/managers/edge_range_manager.rs | 2 +- src/managers/metadata.rs | 4 ++-- src/managers/vertex_manager.rs | 2 +- src/managers/vertex_property_manager.rs | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/managers/edge_manager.rs b/src/managers/edge_manager.rs index f6b8987..427ddee 100644 --- a/src/managers/edge_manager.rs +++ b/src/managers/edge_manager.rs @@ -1,4 +1,4 @@ -use indradb::{Edge, util}; +use indradb::{util, Edge}; use sled::{Batch, IVec, Tree}; use crate::datastore::SledHolder; diff --git a/src/managers/edge_property_manager.rs b/src/managers/edge_property_manager.rs index 31d092b..9604490 100644 --- a/src/managers/edge_property_manager.rs +++ b/src/managers/edge_property_manager.rs @@ -1,7 +1,7 @@ use std::io::Cursor; use ecow::EcoVec; -use indradb::{Edge, Identifier, Json, util}; +use indradb::{util, Edge, Identifier, Json}; use serde_json::Value as JsonValue; use sled::{IVec, Tree}; diff --git a/src/managers/edge_range_manager.rs b/src/managers/edge_range_manager.rs index 690c23f..0ee1b61 100644 --- a/src/managers/edge_range_manager.rs +++ b/src/managers/edge_range_manager.rs @@ -1,6 +1,6 @@ use std::io::Cursor; -use indradb::{Edge, util}; +use indradb::{util, Edge}; use sled::{Batch, Iter as DbIterator, Tree}; use uuid::Uuid; diff --git a/src/managers/metadata.rs b/src/managers/metadata.rs index 767f373..82b989e 100644 --- a/src/managers/metadata.rs +++ b/src/managers/metadata.rs @@ -2,10 +2,10 @@ use std::collections::HashSet; use std::io::Cursor; use std::sync::{Arc, RwLock}; -use indradb::{Identifier, util}; +use indradb::{util, Identifier}; use sled::Tree; -use crate::errors::{DSError, map_err}; +use crate::errors::{map_err, DSError}; const INDEXED_PROPERTIES: &str = "IndexedProperties"; diff --git a/src/managers/vertex_manager.rs b/src/managers/vertex_manager.rs index c36fbe1..c760239 100644 --- a/src/managers/vertex_manager.rs +++ b/src/managers/vertex_manager.rs @@ -1,7 +1,7 @@ use std::io::Cursor; use std::ops::Deref; -use indradb::{Identifier, util, Vertex}; +use indradb::{util, Identifier, Vertex}; use sled::{Batch, Iter as DbIterator, Tree}; use uuid::Uuid; diff --git a/src/managers/vertex_property_manager.rs b/src/managers/vertex_property_manager.rs index 8ffc586..a5363f2 100644 --- a/src/managers/vertex_property_manager.rs +++ b/src/managers/vertex_property_manager.rs @@ -1,7 +1,7 @@ use std::io::Cursor; use ecow::EcoVec; -use indradb::{Identifier, Json, util}; +use indradb::{util, Identifier, Json}; use serde_json::Value as JsonValue; use sled::{IVec, Tree}; use uuid::Uuid; From 7453073d208ee4896a7681468c37cd6a5a8eb21d Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Tue, 9 Jul 2024 18:47:02 +0200 Subject: [PATCH 27/28] Bump version to match versioning of indradb. Makes using them from a separate repo easier. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 99be4b1..251b499 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "indradb-sled" -version = "0.1.0" +version = "4.0.0-RC1" authors = ["Yusuf Simonson "] description = "A sled-backed datastore for IndraDB" homepage = "https://indradb.github.io" From 5e4032f37297b59ee714f1b83aed2eef002a356e Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Mon, 15 Jul 2024 17:09:11 +0200 Subject: [PATCH 28/28] Speedup the bulk import process --- src/managers/edge_property_manager.rs | 42 ++++++++++++----- src/managers/vertex_property_manager.rs | 42 ++++++++++++----- src/transaction.rs | 60 ++++++++++++++++++++----- 3 files changed, 108 insertions(+), 36 deletions(-) diff --git a/src/managers/edge_property_manager.rs b/src/managers/edge_property_manager.rs index 31d092b..32379a2 100644 --- a/src/managers/edge_property_manager.rs +++ b/src/managers/edge_property_manager.rs @@ -1,6 +1,6 @@ +use std::collections::HashMap; use std::io::Cursor; -use ecow::EcoVec; use indradb::{Edge, Identifier, Json, util}; use serde_json::Value as JsonValue; use sled::{IVec, Tree}; @@ -133,6 +133,29 @@ impl<'tree> EdgePropertyManager<'tree> { ) } + pub fn set_batch( + &self, + edge: &Edge, + batch: &mut sled::Batch, + batch_value: &mut sled::Batch, + property_creation_set: &mut HashMap<(Edge, Identifier), Vec>, + name: Identifier, + value: &JsonValue, + ) -> indradb::Result<()> { + let key = self.key(edge, name); + let value_json = serde_json::to_vec(value)?; + batch.insert(key.clone(), value_json); + let old_value = map_err(self.tree.get(key.clone()))?; + if let Some(old_value) = old_value { + let old_value: Json = serde_json::from_slice(&old_value)?; + let value_key = Self::key_value_index(edge, &old_value, name); + batch_value.remove(value_key.as_slice()); + } + let value_key = Self::key_value_index(edge, value, name); + property_creation_set.insert((edge.clone(), name), value_key); + Ok(()) + } + pub fn set(&self, edge: &Edge, name: Identifier, value: &JsonValue) -> indradb::Result<()> { let key = self.key(edge, name); let value_json = serde_json::to_vec(value)?; @@ -155,19 +178,14 @@ impl<'tree> EdgePropertyManager<'tree> { } pub fn delete(&self, edge: &Edge, name: Identifier) -> indradb::Result<()> { + let old_value = map_err(self.tree.get(self.key(edge, name)))?; map_err(self.tree.remove(self.key(edge, name)))?; - let prefix = util::build(&[util::Component::Identifier(name)]); - let mut edges_to_delete = EcoVec::new(); - for current_edge in self.value_index_tree.scan_prefix(prefix) { - let (k, _) = map_err(current_edge)?; - let (_n, _, vid) = Self::read_key_value_index(k.clone()); - if vid == *edge { - edges_to_delete.push(k); - } - } - for edge in edges_to_delete { - map_err(self.value_index_tree.remove(&edge))?; + if let Some(old_value) = old_value { + let old_value: Json = serde_json::from_slice(&old_value)?; + let value_key = Self::key_value_index(edge, &old_value, name); + map_err(self.value_index_tree.remove(value_key.as_slice()))?; } + Ok(()) } } diff --git a/src/managers/vertex_property_manager.rs b/src/managers/vertex_property_manager.rs index 8ffc586..7e1c5a1 100644 --- a/src/managers/vertex_property_manager.rs +++ b/src/managers/vertex_property_manager.rs @@ -1,6 +1,6 @@ +use std::collections::HashMap; use std::io::Cursor; -use ecow::EcoVec; use indradb::{Identifier, Json, util}; use serde_json::Value as JsonValue; use sled::{IVec, Tree}; @@ -98,6 +98,29 @@ impl<'tree> VertexPropertyManager<'tree> { } } + pub fn set_batch( + &self, + vertex_id: Uuid, + batch: &mut sled::Batch, + batch_value: &mut sled::Batch, + property_creation_set: &mut HashMap<(Uuid, Identifier), Vec>, + name: Identifier, + value: &JsonValue, + ) -> indradb::Result<()> { + let key = self.key(vertex_id, name); + let value_json = serde_json::to_vec(value)?; + batch.insert(key.clone(), value_json); + let old_value = map_err(self.tree.get(key.clone()))?; + if let Some(old_value) = old_value { + let old_value: Json = serde_json::from_slice(&old_value)?; + let value_key = Self::key_value_index(&vertex_id, &old_value, name); + batch_value.remove(value_key.as_slice()); + } + let value_key = Self::key_value_index(&vertex_id, value, name); + property_creation_set.insert((vertex_id, name), value_key); + Ok(()) + } + pub fn set(&self, vertex_id: Uuid, name: Identifier, value: &JsonValue) -> indradb::Result<()> { let key = self.key(vertex_id, name); let value_json = serde_json::to_vec(value)?; @@ -115,19 +138,14 @@ impl<'tree> VertexPropertyManager<'tree> { } pub fn delete(&self, vertex_id: Uuid, name: Identifier) -> indradb::Result<()> { + let old_value = map_err(self.tree.get(self.key(vertex_id, name)))?; map_err(self.tree.remove(self.key(vertex_id, name)))?; - let prefix = util::build(&[util::Component::Identifier(name)]); - let items = self.value_index_tree.scan_prefix(prefix); - let mut keys_to_remove = EcoVec::new(); - for (key, _) in items.flatten() { - let (_n, _v, vid) = Self::read_key_value_index(key.clone()); - if vertex_id == vid { - keys_to_remove.push(key); - } - } - for key in keys_to_remove { - map_err(self.value_index_tree.remove(key))?; + if let Some(old_value) = old_value { + let old_value = serde_json::from_slice(&old_value)?; + let value_index_key = Self::key_value_index(&vertex_id, &old_value, name); + map_err(self.value_index_tree.remove(value_index_key))?; } + Ok(()) } } diff --git a/src/transaction.rs b/src/transaction.rs index 8c1a2a8..d4d694c 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -1,7 +1,8 @@ +use std::collections::HashMap; use std::ops::Deref; use indradb::{BulkInsertItem, DynIter, Edge, Error, Identifier, Json, Transaction, Vertex}; -use sled::Batch; +use sled::{Batch, IVec}; use uuid::Uuid; use crate::datastore::SledHolder; @@ -19,10 +20,16 @@ struct IndraSledBatch { pub(crate) edge_creation_batch: Batch, pub(crate) edge_range_creation_batch: Batch, pub(crate) edge_range_rev_creation_batch: Batch, + pub(crate) vertex_property_creation_batch: Batch, + pub(crate) vertex_property_value_creation_batch: Batch, + pub(crate) vertex_property_creation_set: HashMap<(Uuid, Identifier), Vec>, + pub(crate) edge_property_creation_batch: Batch, + pub(crate) edge_property_value_creation_batch: Batch, + pub(crate) edge_property_creation_set: HashMap<(Edge, Identifier), Vec>, } impl IndraSledBatch { - fn apply(self, holder: &SledHolder) -> indradb::Result<()> { + fn apply(mut self, holder: &SledHolder) -> indradb::Result<()> { map_err(holder.db.deref().apply_batch(self.vertex_creation_batch))?; map_err(holder.edges.apply_batch(self.edge_creation_batch))?; map_err(holder.edge_ranges.apply_batch(self.edge_range_creation_batch))?; @@ -31,6 +38,29 @@ impl IndraSledBatch { .reversed_edge_ranges .apply_batch(self.edge_range_rev_creation_batch), )?; + map_err(holder.edge_properties.apply_batch(self.edge_property_creation_batch))?; + map_err( + holder + .vertex_properties + .apply_batch(self.vertex_property_creation_batch), + )?; + + for (_, key) in self.edge_property_creation_set { + self.edge_property_value_creation_batch.insert(key, IVec::default()); + } + for (_, key) in self.vertex_property_creation_set { + self.vertex_property_value_creation_batch.insert(key, IVec::default()); + } + map_err( + holder + .vertex_property_values + .apply_batch(self.vertex_property_value_creation_batch), + )?; + map_err( + holder + .edge_property_values + .apply_batch(self.edge_property_value_creation_batch), + )?; Ok(()) } } @@ -239,8 +269,6 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { fn bulk_insert(&mut self, items: Vec) -> indradb::Result<()> { let mut batch = IndraSledBatch::default(); - let mut vertex_props = Vec::new(); - let mut edge_props = Vec::new(); for item in items { match item { @@ -256,21 +284,29 @@ impl<'a> Transaction<'a> for SledTransaction<'a> { )?; } BulkInsertItem::VertexProperty(id, p, v) => { - vertex_props.push((id, p, v)); + self.vertex_property_manager.set_batch( + id, + &mut batch.vertex_property_creation_batch, + &mut batch.vertex_property_value_creation_batch, + &mut batch.vertex_property_creation_set, + p, + &v, + )?; } BulkInsertItem::EdgeProperty(e, p, v) => { - edge_props.push((e, p, v)); + self.edge_property_manager.set_batch( + &e, + &mut batch.edge_property_creation_batch, + &mut batch.edge_property_value_creation_batch, + &mut batch.edge_property_creation_set, + p, + &v, + )?; } } } batch.apply(self.holder)?; - for (id, p, v) in vertex_props { - self.vertex_property_manager.set(id, p, &v)?; - } - for (e, p, v) in edge_props { - self.edge_property_manager.set(&e, p, &v)?; - } self.sync()?; Ok(()) }