Vacuum

.github/workflows/test_during_pr.yml

@@ -19,25 +19,13 @@ jobs:
     uses: ./.github/workflows/test_rust_workflow.yml
     secrets: inherit
     needs: rust-format-check
-  call-test-rust-storage-workflow-in-local-repo:
-    name: Run Rust storage tests
-    uses: ./.github/workflows/test_rust_disk_storage_workflow.yml
-    secrets: inherit
-    needs: rust-format-check
   call-test-python-workflow-in-local-repo:
     name: Run Python tests
     uses: ./.github/workflows/test_python_workflow.yml
     with:
       test_python_lower: false
     secrets: inherit
     needs: rust-format-check
-  call-test-python-disk-storage-workflow-in-local-repo:
-    name: Run Python storage tests
-    uses: ./.github/workflows/test_python_disk_storage_workflow.yml
-    with:
-      test_python_lower: false
-    secrets: inherit
-    needs: rust-format-check
   call-benchmark-workflow-in-local-repo:
     name: Run benchmarks
     uses: ./.github/workflows/benchmark.yml

.github/workflows/test_rust_workflow.yml

@@ -49,9 +49,9 @@ jobs:
         with:
           cache-all-crates: true
       - name: Install nextest
-        uses: taiki-e/install-action@v2
-        with:
-          tool: nextest@0.9.99
+        uses: taiki-e/install-action@nextest
+      - name: Install cargo-hack
+        uses: taiki-e/install-action@cargo-hack
       - uses: actions/setup-python@v5
         with:
           python-version: '3.12'
@@ -60,7 +60,12 @@ jobs:
           RUSTFLAGS: -Awarnings
           TEMPDIR: ${{ runner.temp }}
         run: |
-          cargo nextest run --all --no-default-features --cargo-profile build-fast
+          cargo nextest run --workspace --no-default-features --cargo-profile build-fast
+      - name: Check all features
+        env:
+          RUSTFLAGS: -Awarnings
+        run: |
+          cargo hack check --workspace --all-targets --each-feature  --skip extension-module,default
   doc-test:
     if: ${{ !inputs.skip_tests }}
     name: "Doc tests"

Cargo.toml

@@ -24,7 +24,7 @@ readme = "README.md"
 homepage = "https://github.com/Raphtory/raphtory/"
 keywords = ["graph", "temporal-graph", "temporal"]
 authors = ["Pometry"]
-rust-version = "1.86.0"
+rust-version = "1.88.0"
 edition = "2021"
 
 # debug symbols are using a lot of resources
@@ -60,6 +60,7 @@ async-graphql = { version = "7.0.16", features = ["dynamic-schema"] }
 bincode = "1.3.3"
 async-graphql-poem = "7.0.16"
 dynamic-graphql = "0.10.1"
+derive_more = "2.0.1"
 reqwest = { version = "0.12.8", default-features = false, features = [
     "rustls-tls",
     "multipart",
@@ -159,6 +160,7 @@ parquet = { version = "56.2.0" }
 arrow-json = { version = "56.2.0" }
 arrow-buffer = { version = "56.2.0" }
 arrow-schema = { version = "56.2.0" }
+serde_arrow = {version = "0.13.6", features = ["arrow-56"]}
 arrow-array = { version = "56.2.0", features = ["chrono-tz"] }
 arrow-ipc = { version = "56.2.0" }
 arrow-csv = { version = "56.2.0" }

db4-graph/src/lib.rs

@@ -207,7 +207,7 @@ impl<EXT: PersistentStrategy<NS = NS<EXT>, ES = ES<EXT>>> TemporalGraph<EXT> {
     }
 
     pub fn disk_storage_enabled(&self) -> bool {
-        Extension::disk_storage_enabled()
+        self.graph_dir().is_some() && Extension::disk_storage_enabled()
     }
     pub fn extension(&self) -> &EXT {
         self.storage().extension()

db4-storage/Cargo.toml

@@ -13,7 +13,7 @@ edition = "2024"
 [dependencies]
 raphtory-api.workspace = true
 raphtory-api-macros.workspace = true
-raphtory-core = { workspace = true, features = ["arrow"] }
+raphtory-core = { workspace = true }
 # db4-common = {path = "../db4-common"}
 
 bitvec = { workspace = true, features = ["serde"] }
@@ -28,6 +28,7 @@ arrow.workspace = true
 arrow-array.workspace = true
 arrow-csv.workspace = true
 arrow-schema.workspace = true
+serde_arrow.workspace = true
 parquet.workspace = true
 bytemuck.workspace = true
 rayon.workspace = true

db4-storage/src/api/edges.rs

@@ -53,6 +53,10 @@ pub trait EdgeSegmentOps: Send + Sync + std::fmt::Debug + 'static {
 
     fn try_head_mut(&self) -> Option<RwLockWriteGuard<'_, MemEdgeSegment>>;
 
+    /// mark segment as dirty without triggering a write
+    fn mark_dirty(&self);
+
+    /// notify that an edge was added (might need to write to disk)
     fn notify_write(
         &self,
         head_lock: impl DerefMut<Target = MemEdgeSegment>,
@@ -84,6 +88,11 @@ pub trait EdgeSegmentOps: Send + Sync + std::fmt::Debug + 'static {
     ) -> Option<Self::Entry<'a>>;
 
     fn locked(self: &Arc<Self>) -> Self::ArcLockedSegment;
+
+    fn vacuum(
+        &self,
+        locked_head: impl DerefMut<Target = MemEdgeSegment>,
+    ) -> Result<(), StorageError>;
 }
 
 pub trait LockedESegment: Send + Sync + std::fmt::Debug {