Skip to content

Commit a09980a

Browse files
authored
Implement snapshots cache in aristo database (#3760)
1 parent bd09787 commit a09980a

File tree

6 files changed

+80
-58
lines changed

6 files changed

+80
-58
lines changed

execution_chain/core/chain/forked_chain.nim

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -506,20 +506,19 @@ proc validateBlock(c: ForkedChainRef,
506506
parentTxFrame=cast[uint](parentFrame),
507507
txFrame=cast[uint](txFrame)
508508

509-
# Checkpoint creates a snapshot of ancestor changes in txFrame - it is an
510-
# expensive operation, specially when creating a new branch (ie when blk
511-
# is being applied to a block that is currently not a head).
512-
# Create the snapshot before processing the block so that any vertexes in snapshots
513-
# from lower levels than the baseTxFrame are removed from the snapshot before running
514-
# the stateroot computation.
515-
parentFrame.checkpoint(parent.blk.header.number, skipSnapshot = false)
509+
516510

517511
var receipts = c.processBlock(parent, txFrame, blk, blkHash, finalized).valueOr:
518512
txFrame.dispose()
519513
return err(error)
520514

521515
c.writeBaggage(blk, blkHash, txFrame, receipts)
522516

517+
# Checkpoint creates a snapshot of ancestor changes in txFrame - it is an
518+
# expensive operation, specially when creating a new branch (ie when blk
519+
# is being applied to a block that is currently not a head).
520+
parentFrame.checkpoint(blk.header.number, skipSnapshot = false)
521+
523522
let newBlock = c.appendBlock(parent, blk, blkHash, txFrame, move(receipts))
524523

525524
for i, tx in blk.transactions:

execution_chain/core/chain/forked_chain/chain_serialize.nim

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -126,14 +126,6 @@ proc replayBlock(fc: ForkedChainRef;
126126
parentFrame = parent.txFrame
127127
txFrame = parentFrame.txFrameBegin()
128128

129-
# Checkpoint creates a snapshot of ancestor changes in txFrame - it is an
130-
# expensive operation, specially when creating a new branch (ie when blk
131-
# is being applied to a block that is currently not a head).
132-
# Create the snapshot before processing the block so that any vertexes in snapshots
133-
# from lower levels than the baseTxFrame are removed from the snapshot before running
134-
# the stateroot computation.
135-
parentFrame.checkpoint(parent.blk.header.number, skipSnapshot = false)
136-
137129
# Set finalized to true in order to skip the stateroot check when replaying the
138130
# block because the blocks should have already been checked previously during
139131
# the initial block execution.
@@ -143,6 +135,11 @@ proc replayBlock(fc: ForkedChainRef;
143135

144136
fc.writeBaggage(blk.blk, blk.hash, txFrame, receipts)
145137

138+
# Checkpoint creates a snapshot of ancestor changes in txFrame - it is an
139+
# expensive operation, specially when creating a new branch (ie when blk
140+
# is being applied to a block that is currently not a head).
141+
parentFrame.checkpoint(blk.header.number, skipSnapshot = false)
142+
146143
blk.txFrame = txFrame
147144
blk.receipts = move(receipts)
148145

execution_chain/db/aristo/aristo_desc.nim

Lines changed: 14 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
{.push raises: [].}
2323

2424
import
25-
std/[hashes, sequtils, sets, tables],
25+
std/[hashes, sequtils, sets, tables, heapqueue],
2626
eth/common/hashes, eth/trie/nibbles,
2727
results,
2828
./aristo_constants,
@@ -34,7 +34,7 @@ import
3434
# Not auto-exporting backend
3535
export
3636
tables, aristo_constants, desc_error, desc_identifiers, nibbles,
37-
desc_structural, minilru, hashes, PutHdlRef
37+
desc_structural, minilru, hashes, heapqueue, PutHdlRef
3838

3939
type
4040
AristoTxRef* = ref object
@@ -124,6 +124,17 @@ type
124124
## MPT level where "most" leaves can be found, for static vid lookups
125125
lookups*: tuple[lower, hits, higher: int]
126126

127+
snapshots*: HeapQueue[AristoTxRef]
128+
## A priority queue of txFrames holding snapshots. Used to limit the number
129+
## of snapshots that can be taken and therefore limit memory usage and also
130+
## to support cleaning old values out of snapshots after persisting to the
131+
## database. txFrames in the queue are sorted by their level in ascending order.
132+
133+
maxSnapshots*: int
134+
## The maximum number of snapshots to hold in the snapshots queue. When the queue
135+
## is full (queue.len == maxSnapshots) then the oldest snapshot is removed from
136+
## the queue and cleaned up.
137+
127138
Leg* = object
128139
## For constructing a `VertexPath`
129140
wp*: VidVtxPair ## Vertex ID and data ref
@@ -139,9 +150,7 @@ const
139150
dbLevel* = -1
140151
disposedLevel* = int.low
141152

142-
# ------------------------------------------------------------------------------
143-
# Public helpers
144-
# ------------------------------------------------------------------------------
153+
proc `<`*(a, b: AristoTxRef): bool = a.level < b.level
145154

146155
template mixUp*(accPath, stoPath: Hash32): Hash32 =
147156
# Insecure but fast way of mixing the values of two hashes, for the purpose
@@ -170,8 +179,6 @@ func getOrVoid*[W](tab: Table[W,RootedVertexID]; w: W): RootedVertexID =
170179
func getOrVoid*[W](tab: Table[W,HashSet[RootedVertexID]]; w: W): HashSet[RootedVertexID] =
171180
tab.getOrDefault(w, default(HashSet[RootedVertexID]))
172181

173-
# --------
174-
175182
func isValid*(vtx: VertexRef): bool =
176183
not isNil(vtx)
177184

@@ -197,20 +204,12 @@ func isValid*(rvid: RootedVertexID): bool =
197204
func isValid*(sqv: HashSet[RootedVertexID]): bool =
198205
sqv.len > 0
199206

200-
# ------------------------------------------------------------------------------
201-
# Public functions, miscellaneous
202-
# ------------------------------------------------------------------------------
203-
204207
func hash*(db: AristoDbRef): Hash {.error.}
205208
func hash*(db: AristoTxRef): Hash {.error.}
206209

207210
proc baseTxFrame*(db: AristoDbRef): AristoTxRef =
208211
db.txRef
209212

210-
# ------------------------------------------------------------------------------
211-
# Public helpers
212-
# ------------------------------------------------------------------------------
213-
214213
iterator rstack*(tx: AristoTxRef, stopAtSnapshot = false): AristoTxRef =
215214
# Stack in reverse order, ie going from tx to base
216215
var tx = tx
@@ -254,8 +253,3 @@ func getStaticLevel*(db: AristoDbRef): int =
254253
db.staticLevel = 1
255254

256255
db.staticLevel
257-
258-
259-
# ------------------------------------------------------------------------------
260-
# End
261-
# ------------------------------------------------------------------------------

execution_chain/db/aristo/aristo_init/init_common.nim

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ proc initInstance*(db: AristoDbRef): Result[void, AristoError] =
8686
db.txRef = AristoTxRef(db: db, vTop: vTop, snapshot: Snapshot(level: Opt.some(0)))
8787
db.accLeaves = LruCache[Hash32, AccLeafRef].init(ACC_LRU_SIZE)
8888
db.stoLeaves = LruCache[Hash32, StoLeafRef].init(ACC_LRU_SIZE)
89+
db.maxSnapshots = 10
8990
ok()
9091

9192
proc finish*(db: AristoDbRef; eradicate = false) =

execution_chain/db/aristo/aristo_tx_frame.nim

Lines changed: 52 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,47 @@ template isDisposed(txFrame: AristoTxRef): bool =
2525
proc isKeyframe(txFrame: AristoTxRef): bool =
2626
txFrame == txFrame.db.txRef
2727

28+
proc clearSnapshot*(txFrame: AristoTxRef) {.raises: [], gcsafe.}
29+
30+
proc removeSnapshotFrame(db: AristoDbRef, txFrame: AristoTxRef) =
31+
let index = db.snapshots.find(txFrame)
32+
if index != -1:
33+
db.snapshots.del(index)
34+
35+
proc addSnapshotFrame(db: AristoDbRef, txFrame: AristoTxRef) =
36+
doAssert txFrame.snapshot.level.isSome()
37+
if db.snapshots.contains(txFrame):
38+
# No-op if the queue already contains the snapshot
39+
return
40+
41+
if db.snapshots.len() == db.maxSnapshots:
42+
let frame = db.snapshots.pop()
43+
frame.clearSnapshot()
44+
45+
db.snapshots.push(txFrame)
46+
47+
proc cleanSnapshots(db: AristoDbRef) =
48+
let minLevel = db.txRef.level
49+
50+
# Some of the snapshot content may have already been persisted to disk
51+
# (since they were made based on the in-memory frames at the time of creation).
52+
# Annoyingly, there's no way to remove items while iterating but even
53+
# with the extra seq, move + remove turns out to be faster than
54+
# creating a new table - specially when the ratio between old and
55+
# and current items favors current items.
56+
template delIfIt(tbl: var Table, body: untyped) =
57+
var toRemove = newSeqOfCap[typeof(tbl).A](tbl.len div 2)
58+
for k, it {.inject.} in tbl:
59+
if body:
60+
toRemove.add k
61+
for k in toRemove:
62+
tbl.del(k)
63+
64+
for frame in db.snapshots:
65+
frame.snapshot.vtx.delIfIt(it[2] < minLevel)
66+
frame.snapshot.acc.delIfIt(it[1] < minLevel)
67+
frame.snapshot.sto.delIfIt(it[1] < minLevel)
68+
2869
proc buildSnapshot(txFrame: AristoTxRef, minLevel: int) =
2970
# Starting from the previous snapshot, build a snapshot that includes all
3071
# ancestor changes as well as the changes in txFrame itself
@@ -41,30 +82,11 @@ proc buildSnapshot(txFrame: AristoTxRef, minLevel: int) =
4182
if frame.snapshot.level.isSome() and not isKeyframe:
4283
# `frame` has a snapshot only in the first iteration of the for loop
4384
txFrame.snapshot = move(frame.snapshot)
85+
txFrame.db.removeSnapshotFrame(frame)
4486

4587
# Verify that https://github.com/nim-lang/Nim/issues/23759 is not present
4688
assert frame.snapshot.vtx.len == 0 and frame.snapshot.level.isNone()
4789

48-
if txFrame.snapshot.level != Opt.some(minLevel):
49-
# When recycling an existing snapshot, some of its content may have
50-
# already been persisted to disk (since it was made base on the
51-
# in-memory frames at the time of its creation).
52-
# Annoyingly, there's no way to remove items while iterating but even
53-
# with the extra seq, move + remove turns out to be faster than
54-
# creating a new table - specially when the ratio between old and
55-
# and current items favors current items.
56-
template delIfIt(tbl: var Table, body: untyped) =
57-
var toRemove = newSeqOfCap[typeof(tbl).A](tbl.len div 2)
58-
for k, it {.inject.} in tbl:
59-
if body:
60-
toRemove.add k
61-
for k in toRemove:
62-
tbl.del(k)
63-
64-
txFrame.snapshot.vtx.delIfIt(it[2] < minLevel)
65-
txFrame.snapshot.acc.delIfIt(it[1] < minLevel)
66-
txFrame.snapshot.sto.delIfIt(it[1] < minLevel)
67-
6890
if frame.snapshot.level.isSome() and isKeyframe:
6991
txFrame.snapshot.vtx = initTable[RootedVertexID, VtxSnapshot](
7092
max(1024, max(frame.sTab.len, frame.snapshot.vtx.len))
@@ -96,6 +118,9 @@ proc buildSnapshot(txFrame: AristoTxRef, minLevel: int) =
96118

97119
txFrame.snapshot.level = Opt.some(minLevel)
98120

121+
if not txFrame.isKeyframe():
122+
txFrame.db.addSnapshotFrame(txFrame)
123+
99124
# ------------------------------------------------------------------------------
100125
# Public functions
101126
# ------------------------------------------------------------------------------
@@ -115,6 +140,8 @@ proc txFrameBegin*(
115140
level: parent.level + 1)
116141

117142
proc dispose*(txFrame: AristoTxRef) =
143+
if not txFrame.db.isNil():
144+
txFrame.db.removeSnapshotFrame(txFrame)
118145
txFrame[].reset()
119146
txFrame.level = disposedLevel
120147

@@ -246,6 +273,11 @@ proc persist*(db: AristoDbRef, batch: PutHdlRef, txFrame: AristoTxRef) =
246273
else:
247274
discard db.stoLeaves.update(mixPath, vtx)
248275

276+
# Remove snapshot data that has been persisted to disk to save memory.
277+
# All snapshot records with a level lower than the current base level
278+
# are deleted.
279+
db.cleanSnapshots()
280+
249281
txFrame.snapshot.vtx.clear()
250282
txFrame.snapshot.acc.clear()
251283
txFrame.snapshot.sto.clear()

tests/test_aristo/test_tx_frame.nim

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -315,8 +315,7 @@ suite "Aristo TxFrame":
315315
# Verify that getting the state root of the level 3 txFrame does not impact
316316
# the persisted state in the database.
317317
let stateRootBefore = tx3.fetchStateRoot().get()
318-
expect(Defect):
319-
discard tx4.fetchStateRoot()
318+
discard tx4.fetchStateRoot()
320319
let stateRootAfter = tx3.fetchStateRoot().get()
321320
check stateRootBefore == stateRootAfter
322321

@@ -346,7 +345,7 @@ suite "Aristo TxFrame":
346345
for i in 1..<10:
347346
let acc = makeAccount(i.uint64)
348347
check tx1.mergeAccountRecord(acc[0], acc[1]).isOk()
349-
348+
350349
tx1.checkpoint(1, skipSnapshot = false)
351350

352351
let

0 commit comments

Comments
 (0)