GraphQL improvements for v4

Makefile

@@ -12,17 +12,13 @@ build-all: rust-build
 
 test-all: rust-test-all python-test
 
-test-all-public: rust-test-all-public python-test-public
-
 # Tidying
 
 tidy: rust-fmt build-python stubs python-fmt
 
-tidy-public: rust-fmt build-python-public stubs python-fmt
-
 python-tidy: stubs python-fmt test-graphql-schema
 
-check-pr: tidy-public test-all
+check-pr: tidy test-all
 
 gen-graphql-schema:
 	raphtory schema > raphtory-graphql/schema.graphql
@@ -31,7 +27,6 @@ test-graphql-schema: install-node-tools
 	npx graphql-schema-linter --rules fields-have-descriptions,types-have-descriptions raphtory-graphql/schema.graphql
 
 # Utilities
-
 activate-storage:
 	./scripts/activate_private_storage.py
 
@@ -71,13 +66,12 @@ run-graphql:
 rust-test:
 	cargo test -q
 
-rust-test-all: activate-storage
-	cargo nextest run --all --features=storage
+rust-check:
 	cargo hack check --workspace --all-targets --each-feature  --skip extension-module,default
 
-rust-test-all-public:
+rust-test-all: rust-check
 	cargo nextest run --all
-	cargo hack check --workspace --all-targets --each-feature  --skip extension-module,default,storage
+
 
 ##########
 # Python #
@@ -86,32 +80,23 @@ rust-test-all-public:
 install-python:
 	cd python && maturin build && pip install ../target/wheels/*.whl
 
-build-python-public: deactivate-storage
+build-python:
 	cd python && maturin develop -r --extras=dev
 
-build-python: activate-storage
-	cd python && maturin develop -r --features=storage --extras=dev
+debug-python:
+	cd python && maturin develop --profile=debug --extras=dev
 
 # Testing
-
-python-test: activate-storage
-	cd python && tox run && tox run -e storage
-
-python-test-public:
+python-test:
 	cd python && tox run
 
 python-fmt:
 	cd python && black .
 
-debug-python-public: deactivate-storage
-	cd python && maturin develop --profile=debug
 
 build-python-rtd:
 	cd python && maturin build --profile=build-fast && pip install ../target/wheels/*.whl
 
-debug-python: activate-storage
-	cd python && maturin develop --features=storage,extension-module --extras=dev
-
 ########
 # Docs #
 ########

db4-graph/src/lib.rs

@@ -212,8 +212,9 @@ impl<EXT: PersistentStrategy<NS = NS<EXT>, ES = ES<EXT>, GS = GS<EXT>>> Temporal
         })
     }
 
-    pub fn disk_storage_enabled(&self) -> bool {
-        self.graph_dir().is_some() && Extension::disk_storage_enabled()
+    pub fn disk_storage_enabled(&self) -> Option<&Path> {
+        self.graph_dir()
+            .filter(|_| Extension::disk_storage_enabled())
     }
 
     pub fn extension(&self) -> &EXT {

db4-storage/src/pages/edge_page/writer.rs

@@ -2,12 +2,8 @@ use crate::{
     LocalPOS, api::edges::EdgeSegmentOps, error::StorageError, pages::layer_counter::GraphStats,
     segments::edge::segment::MemEdgeSegment,
 };
-use arrow_array::{ArrayRef, BooleanArray};
 use raphtory_api::core::entities::{VID, properties::prop::Prop};
-use raphtory_core::{
-    entities::EID,
-    storage::timeindex::{AsTime, TimeIndexEntry},
-};
+use raphtory_core::storage::timeindex::{AsTime, TimeIndexEntry};
 use std::ops::DerefMut;
 
 pub struct EdgeWriter<

db4-storage/src/pages/layer_counter.rs

@@ -96,8 +96,6 @@ impl GraphStats {
             }
         } else {
             // we need to create the layer
-            self.layers.reserve(layer_id + 1 - self.layers.count());
-
             loop {
                 let new_layer_id = self.layers.push_with(|_| Default::default());
                 if new_layer_id >= layer_id {

db4-storage/src/pages/mod.rs

@@ -484,7 +484,7 @@ mod test {
             make_nodes,
         },
     };
-    use chrono::{DateTime, NaiveDateTime, Utc};
+    use chrono::DateTime;
     use proptest::prelude::*;
     use raphtory_api::core::entities::properties::prop::Prop;
     use raphtory_core::{entities::VID, storage::timeindex::TimeIndexOps};
@@ -766,14 +766,11 @@ mod test {
                     ("857".to_owned(), Prop::F64(2.56)),
                     (
                         "296".to_owned(),
-                        Prop::NDTime(NaiveDateTime::from_timestamp(1334043671, 0)),
+                        Prop::NDTime(DateTime::from_timestamp(1334043671, 0).unwrap().naive_utc()),
                     ),
                     (
                         "92".to_owned(),
-                        Prop::DTime(DateTime::<Utc>::from_utc(
-                            NaiveDateTime::from_timestamp(994032315, 0),
-                            Utc,
-                        )),
+                        Prop::DTime(DateTime::from_timestamp(994032315, 0).unwrap()),
                     ),
                 ],
             )],

db4-storage/src/properties/mod.rs

@@ -8,8 +8,8 @@ use bigdecimal::ToPrimitive;
 use raphtory_api::core::entities::properties::{
     meta::PropMapper,
     prop::{
-        Prop, PropType, SerdeList, SerdeMap, arrow_dtype_from_prop_type, list_array_from_props,
-        struct_array_from_props,
+        Prop, PropType, SerdeArrowList, SerdeArrowMap, arrow_dtype_from_prop_type,
+        list_array_from_props, struct_array_from_props,
     },
 };
 use raphtory_core::{
@@ -206,7 +206,7 @@ impl Properties {
                     .unwrap();
                 let array_iter = indices
                     .map(|i| lazy_vec.get_opt(i))
-                    .map(|e| e.map(|m| SerdeMap(m)));
+                    .map(|e| e.map(|m| SerdeArrowMap(m)));
 
                 let struct_array = struct_array_from_props(&dt, array_iter);
 
@@ -221,7 +221,7 @@ impl Properties {
 
                 let array_iter = indices
                     .map(|i| lazy_vec.get_opt(i))
-                    .map(|opt_list| opt_list.map(SerdeList));
+                    .map(|opt_list| opt_list.map(SerdeArrowList));
 
                 let list_array = list_array_from_props(&dt, array_iter);
 

db4-storage/src/segments/graph_prop/segment.rs

@@ -1,5 +1,4 @@
 use crate::{
-    LocalPOS,
     error::StorageError,
     segments::{HasRow, SegmentContainer},
 };

db4-storage/src/segments/node/segment.rs

@@ -550,21 +550,19 @@ impl<P: PersistentStrategy<NS = NodeSegmentView<P>>> NodeSegmentOps for NodeSegm
 
 #[cfg(test)]
 mod test {
-    use std::sync::Arc;
-
-    use raphtory_api::core::entities::properties::{
-        meta::Meta,
-        prop::{Prop, PropType},
-    };
-    use raphtory_core::entities::{EID, ELID, VID};
-    use tempfile::tempdir;
-
     use crate::{
         LocalPOS, NodeSegmentView,
         api::nodes::NodeSegmentOps,
         pages::{layer_counter::GraphStats, node_page::writer::NodeWriter},
         persist::strategy::NoOpStrategy,
     };
+    use raphtory_api::core::entities::properties::{
+        meta::Meta,
+        prop::{Prop, PropType},
+    };
+    use raphtory_core::entities::{EID, ELID, VID};
+    use std::sync::Arc;
+    use tempfile::tempdir;
 
     #[test]
     fn est_size_changes() {

examples/rust/src/bin/bench/main.rs

@@ -39,7 +39,7 @@ fn main() {
     info!("Loading data");
     let graph = if encoded_data_dir.exists() {
         let now = Instant::now();
-        let g = Graph::decode(encoded_data_dir.as_path(), None)
+        let g = Graph::decode(encoded_data_dir.as_path())
             .expect("Failed to load graph from encoded data files");
 
         info!(

examples/rust/src/bin/btc/main.rs

@@ -64,7 +64,7 @@ fn main() {
 
     let graph = if encoded_data_dir.exists() {
         let now = Instant::now();
-        let g = Graph::decode(encoded_data_dir.as_path(), None)
+        let g = Graph::decode(encoded_data_dir.as_path())
             .expect("Failed to load graph from encoded data files");
 
         info!(

examples/rust/src/bin/hulongbay/main.rs

@@ -65,7 +65,7 @@ pub fn loader(data_dir: &Path) -> Result<Graph, Box<dyn Error>> {
     let encoded_data_dir = data_dir.join("graphdb.bincode");
     if encoded_data_dir.exists() {
         let now = Instant::now();
-        let g = Graph::decode(encoded_data_dir.as_path(), None)?;
+        let g = Graph::decode(encoded_data_dir.as_path())?;
 
         info!(
             "Loaded graph from path {} with {} nodes, {} edges, took {} seconds",

examples/rust/src/bin/lotr/main.rs

@@ -38,7 +38,7 @@ fn main() {
 
     let graph = if encoded_data_dir.exists() {
         let now = Instant::now();
-        let g = Graph::decode(encoded_data_dir.as_path(), None)
+        let g = Graph::decode(encoded_data_dir.as_path())
             .expect("Failed to load graph from encoded data files");
 
         info!(

examples/rust/src/bin/pokec/main.rs

@@ -24,7 +24,7 @@ fn main() {
     let data_dir = Path::new(args.get(1).expect("No data directory provided"));
 
     let g = if std::path::Path::new("/tmp/pokec").exists() {
-        Graph::decode("/tmp/pokec", None).unwrap()
+        Graph::decode("/tmp/pokec").unwrap()
     } else {
         let g = Graph::new();
         CsvLoader::new(data_dir)