redesign, refactor property filters (#2311)

* redesign, refactor property filters

* takes a reference

* rework validation

* fix arbitrary list, fix tests

* merge python traits, fix tests, add more validations

* rework gql filtering apis, fix tests

* add review suggestions

* Features/gql apis (#2350)

* impl nodes select filtering in gql

* change semantics of filters in gql, add missing filter apis in edges, fix all tests

* add more edge filter tests

* add filtering to path from node, add tests

* fix apply views

* rename filter-iter to select

* add select as args

* impl window filter (#2359)

* impl window filter

* impl window filter in python, add tests

* impl gql window filter, add tests

* ref

* impl review suggestions

* fixes

* fix py and gql

* add review suggestions

Author: shivamka1

python/python/raphtory/__init__.pyi

@@ -49,7 +49,6 @@ __all__ = [
     "IndexSpec",
     "Prop",
     "version",
-    "DiskGraphStorage",
     "graphql",
     "algorithms",
     "graph_loader",
@@ -1376,17 +1375,6 @@ class Graph(GraphView):
           MutableNode: The node object with the specified id, or None if the node does not exist
         """
 
-    def persist_as_disk_graph(self, graph_dir: str | PathLike) -> DiskGraphStorage:
-        """
-        save graph in disk_graph format and memory map the result
-
-        Arguments:
-            graph_dir (str | PathLike): folder where the graph will be saved
-
-        Returns:
-            DiskGraphStorage: the persisted graph storage
-        """
-
     def persistent_graph(self) -> PersistentGraph:
         """
         View graph with persistent semantics
@@ -1424,17 +1412,6 @@ class Graph(GraphView):
           bytes:
         """
 
-    def to_disk_graph(self, graph_dir: str | PathLike) -> Graph:
-        """
-        Persist graph on disk
-
-        Arguments:
-            graph_dir (str | PathLike): the folder where the graph will be persisted
-
-        Returns:
-            Graph: a view of the persisted graph
-        """
-
     def to_parquet(self, graph_dir: str | PathLike):
         """
         Persist graph to parquet files
@@ -2209,7 +2186,6 @@ class PersistentGraph(GraphView):
           bytes:
         """
 
-    def to_disk_graph(self, graph_dir): ...
     def update_metadata(self, metadata: dict) -> None:
         """
         Updates metadata of the graph.
@@ -6142,35 +6118,3 @@ class Prop(object):
     def u8(value): ...
 
 def version(): ...
-
-class DiskGraphStorage(object):
-    def __repr__(self):
-        """Return repr(self)."""
-
-    def append_node_temporal_properties(self, location, chunk_size=20000000): ...
-    def graph_dir(self): ...
-    @staticmethod
-    def load_from_dir(graph_dir): ...
-    @staticmethod
-    def load_from_pandas(graph_dir, edge_df, time_col, src_col, dst_col): ...
-    @staticmethod
-    def load_from_parquets(
-        graph_dir,
-        layer_parquet_cols,
-        node_properties=None,
-        chunk_size=10000000,
-        t_props_chunk_size=10000000,
-        num_threads=4,
-        node_type_col=None,
-        node_id_col=None,
-    ): ...
-    def load_node_metadata(self, location, col_names=None, chunk_size=None): ...
-    def load_node_types(self, location, col_name, chunk_size=None): ...
-    def merge_by_sorted_gids(self, other, graph_dir):
-        """
-        Merge this graph with another `DiskGraph`. Note that both graphs should have nodes that are
-        sorted by their global ids or the resulting graph will be nonsense!
-        """
-
-    def to_events(self): ...
-    def to_persistent(self): ...

python/python/raphtory/algorithms/__init__.pyi

@@ -70,7 +70,6 @@ __all__ = [
     "max_weight_matching",
     "Matching",
     "Infected",
-    "connected_components",
 ]
 
 def dijkstra_single_source_shortest_paths(
@@ -894,5 +893,3 @@ class Infected(object):
         Returns:
             int:
         """
-
-def connected_components(graph): ...

python/python/raphtory/filter/__init__.pyi

@@ -30,9 +30,11 @@ __all__ = [
     "EdgeEndpoint",
     "Edge",
     "ExplodedEdge",
-    "Property",
-    "Metadata",
-    "TemporalPropertyFilterBuilder",
+    "FilterOps",
+    "PropertyFilterOps",
+    "NodeWindow",
+    "EdgeWindow",
+    "ExplodedEdgeWindow",
 ]
 
 class FilterExpr(object):
@@ -51,41 +53,8 @@ class FilterExpr(object):
     def __ror__(self, value):
         """Return value|self."""
 
-class PropertyFilterOps(object):
-    def __eq__(self, value):
-        """Return self==value."""
-
-    def __ge__(self, value):
-        """Return self>=value."""
-
-    def __gt__(self, value):
-        """Return self>value."""
-
-    def __le__(self, value):
-        """Return self<=value."""
-
-    def __lt__(self, value):
-        """Return self<value."""
-
-    def __ne__(self, value):
-        """Return self!=value."""
-
-    def all(self): ...
-    def any(self): ...
-    def avg(self): ...
-    def contains(self, value): ...
-    def ends_with(self, value): ...
-    def fuzzy_search(self, prop_value, levenshtein_distance, prefix_match): ...
-    def is_in(self, values): ...
-    def is_none(self): ...
-    def is_not_in(self, values): ...
-    def is_some(self): ...
-    def len(self): ...
-    def max(self): ...
-    def min(self): ...
-    def not_contains(self, value): ...
-    def starts_with(self, value): ...
-    def sum(self): ...
+class PropertyFilterOps(FilterOps):
+    def temporal(self): ...
 
 class Node(object):
     @staticmethod
@@ -119,6 +88,8 @@ class Node(object):
 
     @staticmethod
     def property(name): ...
+    @staticmethod
+    def window(py_start, py_end): ...
 
 class EdgeFilterOp(object):
     def __eq__(self, value):
@@ -160,32 +131,18 @@ class Edge(object):
     def property(name): ...
     @staticmethod
     def src(): ...
+    @staticmethod
+    def window(py_start, py_end): ...
 
 class ExplodedEdge(object):
     @staticmethod
     def metadata(name): ...
     @staticmethod
     def property(name): ...
+    @staticmethod
+    def window(py_start, py_end): ...
 
-class Property(PropertyFilterOps):
-    """
-    Construct a property filter
-
-    Arguments:
-        name (str): the name of the property to filter
-    """
-
-    def temporal(self): ...
-
-class Metadata(PropertyFilterOps):
-    """
-    Construct a metadata filter
-
-    Arguments:
-        name (str): the name of the property to filter
-    """
-
-class TemporalPropertyFilterBuilder(object):
+class FilterOps(object):
     def __eq__(self, value):
         """Return self==value."""
 
@@ -207,9 +164,33 @@ class TemporalPropertyFilterBuilder(object):
     def all(self): ...
     def any(self): ...
     def avg(self): ...
+    def contains(self, value): ...
+    def ends_with(self, value): ...
     def first(self): ...
-    def latest(self): ...
+    def fuzzy_search(self, prop_value, levenshtein_distance, prefix_match): ...
+    def is_in(self, values): ...
+    def is_none(self): ...
+    def is_not_in(self, values): ...
+    def is_some(self): ...
+    def last(self): ...
     def len(self): ...
     def max(self): ...
     def min(self): ...
+    def not_contains(self, value): ...
+    def starts_with(self, value): ...
     def sum(self): ...
+
+class PropertyFilterOps(FilterOps):
+    def temporal(self): ...
+
+class NodeWindow(object):
+    def metadata(self, name): ...
+    def property(self, name): ...
+
+class EdgeWindow(object):
+    def metadata(self, name): ...
+    def property(self, name): ...
+
+class ExplodedEdgeWindow(object):
+    def metadata(self, name): ...
+    def property(self, name): ...

python/tests/test_base_install/test_filters/semantics/test_edge_property_filter_semantics.py

@@ -96,9 +96,9 @@ def check(graph):
 
 
 @with_disk_variants(init_edges_graph, variants=["graph", "event_disk_graph"])
-def test_temporal_latest_semantics():
+def test_temporal_last_semantics():
     def check(graph):
-        filter_expr = filter.Edge.property("p1").temporal().latest() == 1
+        filter_expr = filter.Edge.property("p1").temporal().last() == 1
         result_ids = sorted(graph.filter(filter_expr).edges.id)
         expected_ids = sorted(
             [("N1", "N2"), ("N3", "N4"), ("N4", "N5"), ("N6", "N7"), ("N7", "N8")]
@@ -112,9 +112,9 @@ def check(graph):
     init_fn=combined([init_edges_graph, init_graph_for_secondary_indexes]),
     variants=["graph", "event_disk_graph"],
 )
-def test_temporal_latest_semantics_for_secondary_indexes3():
+def test_temporal_last_semantics_for_secondary_indexes3():
     def check(graph):
-        filter_expr = filter.Edge.property("p1").temporal().latest() == 1
+        filter_expr = filter.Edge.property("p1").temporal().last() == 1
         result_ids = sorted(graph.filter(filter_expr).edges.id)
         expected_ids = sorted(
             [

python/tests/test_base_install/test_filters/semantics/test_node_property_filter_semantics.py

@@ -56,9 +56,9 @@ def check(graph):
 
 
 @with_disk_variants(init_nodes_graph)
-def test_temporal_latest_semantics():
+def test_temporal_last_semantics():
     def check(graph):
-        filter_expr = filter.Node.property("p1").temporal().latest() == 1
+        filter_expr = filter.Node.property("p1").temporal().last() == 1
         result_ids = sorted(graph.filter(filter_expr).nodes.id)
         expected_ids = sorted(["N1", "N3", "N4", "N6", "N7"])
         assert result_ids == expected_ids
@@ -69,9 +69,9 @@ def check(graph):
 @with_disk_variants(
     init_fn=combined([init_nodes_graph, init_graph_for_secondary_indexes]),
 )
-def test_temporal_latest_semantics_for_secondary_indexes():
+def test_temporal_last_semantics_for_secondary_indexes():
     def check(graph):
-        filter_expr = filter.Node.property("p1").temporal().latest() == 1
+        filter_expr = filter.Node.property("p1").temporal().last() == 1
         result_ids = sorted(graph.filter(filter_expr).nodes.id)
         expected_ids = sorted(["N1", "N16", "N3", "N4", "N6", "N7"])
         assert result_ids == expected_ids