@@ -25,6 +25,47 @@ template isDisposed(txFrame: AristoTxRef): bool =
2525proc isKeyframe (txFrame: AristoTxRef ): bool =
2626 txFrame == txFrame.db.txRef
2727
28+ proc clearSnapshot * (txFrame: AristoTxRef ) {.raises : [], gcsafe .}
29+
30+ proc removeSnapshotFrame (db: AristoDbRef , txFrame: AristoTxRef ) =
31+ let index = db.snapshots.find (txFrame)
32+ if index != - 1 :
33+ db.snapshots.del (index)
34+
35+ proc addSnapshotFrame (db: AristoDbRef , txFrame: AristoTxRef ) =
36+ doAssert txFrame.snapshot.level.isSome ()
37+ if db.snapshots.contains (txFrame):
38+ # No-op if the queue already contains the snapshot
39+ return
40+
41+ if db.snapshots.len () == db.maxSnapshots:
42+ let frame = db.snapshots.pop ()
43+ frame.clearSnapshot ()
44+
45+ db.snapshots.push (txFrame)
46+
47+ proc cleanSnapshots (db: AristoDbRef ) =
48+ let minLevel = db.txRef.level
49+
50+ # Some of the snapshot content may have already been persisted to disk
51+ # (since they were made based on the in-memory frames at the time of creation).
52+ # Annoyingly, there's no way to remove items while iterating but even
53+ # with the extra seq, move + remove turns out to be faster than
54+ # creating a new table - specially when the ratio between old and
55+ # and current items favors current items.
56+ template delIfIt (tbl: var Table , body: untyped ) =
57+ var toRemove = newSeqOfCap [typeof (tbl).A](tbl.len div 2 )
58+ for k, it {.inject .} in tbl:
59+ if body:
60+ toRemove.add k
61+ for k in toRemove:
62+ tbl.del (k)
63+
64+ for frame in db.snapshots:
65+ frame.snapshot.vtx.delIfIt (it[2 ] < minLevel)
66+ frame.snapshot.acc.delIfIt (it[1 ] < minLevel)
67+ frame.snapshot.sto.delIfIt (it[1 ] < minLevel)
68+
2869proc buildSnapshot (txFrame: AristoTxRef , minLevel: int ) =
2970 # Starting from the previous snapshot, build a snapshot that includes all
3071 # ancestor changes as well as the changes in txFrame itself
@@ -41,30 +82,11 @@ proc buildSnapshot(txFrame: AristoTxRef, minLevel: int) =
4182 if frame.snapshot.level.isSome () and not isKeyframe:
4283 # `frame` has a snapshot only in the first iteration of the for loop
4384 txFrame.snapshot = move (frame.snapshot)
85+ txFrame.db.removeSnapshotFrame (frame)
4486
4587 # Verify that https://github.com/nim-lang/Nim/issues/23759 is not present
4688 assert frame.snapshot.vtx.len == 0 and frame.snapshot.level.isNone ()
4789
48- if txFrame.snapshot.level != Opt .some (minLevel):
49- # When recycling an existing snapshot, some of its content may have
50- # already been persisted to disk (since it was made base on the
51- # in-memory frames at the time of its creation).
52- # Annoyingly, there's no way to remove items while iterating but even
53- # with the extra seq, move + remove turns out to be faster than
54- # creating a new table - specially when the ratio between old and
55- # and current items favors current items.
56- template delIfIt (tbl: var Table , body: untyped ) =
57- var toRemove = newSeqOfCap [typeof (tbl).A](tbl.len div 2 )
58- for k, it {.inject .} in tbl:
59- if body:
60- toRemove.add k
61- for k in toRemove:
62- tbl.del (k)
63-
64- txFrame.snapshot.vtx.delIfIt (it[2 ] < minLevel)
65- txFrame.snapshot.acc.delIfIt (it[1 ] < minLevel)
66- txFrame.snapshot.sto.delIfIt (it[1 ] < minLevel)
67-
6890 if frame.snapshot.level.isSome () and isKeyframe:
6991 txFrame.snapshot.vtx = initTable [RootedVertexID , VtxSnapshot ](
7092 max (1024 , max (frame.sTab.len, frame.snapshot.vtx.len))
@@ -96,6 +118,9 @@ proc buildSnapshot(txFrame: AristoTxRef, minLevel: int) =
96118
97119 txFrame.snapshot.level = Opt .some (minLevel)
98120
121+ if not txFrame.isKeyframe ():
122+ txFrame.db.addSnapshotFrame (txFrame)
123+
99124# ------------------------------------------------------------------------------
100125# Public functions
101126# ------------------------------------------------------------------------------
@@ -115,6 +140,8 @@ proc txFrameBegin*(
115140 level: parent.level + 1 )
116141
117142proc dispose * (txFrame: AristoTxRef ) =
143+ if not txFrame.db.isNil ():
144+ txFrame.db.removeSnapshotFrame (txFrame)
118145 txFrame[].reset ()
119146 txFrame.level = disposedLevel
120147
@@ -246,6 +273,11 @@ proc persist*(db: AristoDbRef, batch: PutHdlRef, txFrame: AristoTxRef) =
246273 else :
247274 discard db.stoLeaves.update (mixPath, vtx)
248275
276+ # Remove snapshot data that has been persisted to disk to save memory.
277+ # All snapshot records with a level lower than the current base level
278+ # are deleted.
279+ db.cleanSnapshots ()
280+
249281 txFrame.snapshot.vtx.clear ()
250282 txFrame.snapshot.acc.clear ()
251283 txFrame.snapshot.sto.clear ()
0 commit comments