Skip to content

Commit e91dcef

Browse files
julienrbrtjulienrbrt
committed
perf: improve restore perf
1 parent d20b1ac commit e91dcef

File tree

8 files changed

+191
-67
lines changed

8 files changed

+191
-67
lines changed

‎block/internal/cache/generic_cache.go‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"encoding/binary"
66
"fmt"
7+
"strings"
78
"sync"
89

910
"sync/atomic"
@@ -215,20 +216,31 @@ func (c *Cache[T]) deleteAllForHeight(height uint64) {
215216

216217
// RestoreFromStore loads DA inclusion data from the store into the in-memory cache.
217218
// This should be called during initialization to restore persisted state.
218-
// It iterates through store metadata keys with the cache's prefix and populates the LRU cache.
219-
func (c *Cache[T]) RestoreFromStore(ctx context.Context, hashes []string) error {
220-
if c.store == nil {
221-
return nil // No store configured, nothing to restore
219+
// It directly queries store metadata keys with the cache's prefix, avoiding iteration through all blocks.
220+
func (c *Cache[T]) RestoreFromStore(ctx context.Context) error {
221+
if c.store == nil || c.storeKeyPrefix == "" {
222+
return nil // No store configured or no prefix, nothing to restore
222223
}
223224

224-
for _, hash := range hashes {
225-
value, err := c.store.GetMetadata(ctx, c.storeKey(hash))
226-
if err != nil {
227-
// Key not found is not an error - the hash may not have been DA included yet
228-
continue
225+
// Query all metadata entries with our prefix directly
226+
entries, err := c.store.GetMetadataByPrefix(ctx, c.storeKeyPrefix)
227+
if err != nil {
228+
return fmt.Errorf("failed to query metadata by prefix %q: %w", c.storeKeyPrefix, err)
229+
}
230+
231+
for _, entry := range entries {
232+
// Extract the hash from the key by removing the prefix
233+
// The key may have a leading "/" so we try both formats
234+
hash := strings.TrimPrefix(entry.Key, c.storeKeyPrefix)
235+
if hash == entry.Key {
236+
// Try with leading slash
237+
hash = strings.TrimPrefix(entry.Key, "/"+c.storeKeyPrefix)
238+
}
239+
if hash == "" || hash == entry.Key {
240+
continue // Invalid key format
229241
}
230242

231-
daHeight, blockHeight, ok := decodeDAInclusion(value)
243+
daHeight, blockHeight, ok := decodeDAInclusion(entry.Value)
232244
if !ok {
233245
continue // Invalid data, skip
234246
}

‎block/internal/cache/generic_cache_test.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,7 @@ func TestCache_MaxDAHeight_WithStore(t *testing.T) {
6767
// Create new cache and restore from store
6868
c2 := NewCache[testItem](st, "test/da-included/")
6969

70-
// Restore with the known hashes
71-
err = c2.RestoreFromStore(ctx, []string{"hash1", "hash2", "hash3"})
70+
err = c2.RestoreFromStore(ctx)
7271
require.NoError(t, err)
7372

7473
if got := c2.daHeight(); got != 200 {
@@ -106,7 +105,7 @@ func TestCache_WithStorePersistence(t *testing.T) {
106105
// Create new cache with same store and restore
107106
c2 := NewCache[testItem](st, "test/")
108107

109-
err = c2.RestoreFromStore(ctx, []string{"hash1", "hash2", "hash3"})
108+
err = c2.RestoreFromStore(ctx)
110109
require.NoError(t, err)
111110

112111
// hash1 and hash2 should be restored, hash3 should not exist
@@ -263,7 +262,7 @@ func TestCache_SaveToStore(t *testing.T) {
263262
// Verify data is in store by creating new cache and restoring
264263
c2 := NewCache[testItem](st, "save-test/")
265264

266-
err = c2.RestoreFromStore(ctx, []string{"hash1", "hash2"})
265+
err = c2.RestoreFromStore(ctx)
267266
require.NoError(t, err)
268267

269268
daHeight, ok := c2.getDAIncluded("hash1")

‎block/internal/cache/manager.go‎

Lines changed: 6 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -342,54 +342,26 @@ func (m *implementation) SaveToStore() error {
342342
}
343343

344344
// RestoreFromStore restores the DA inclusion cache from the store.
345-
// This iterates through blocks in the store and checks for persisted DA inclusion data.
345+
// This uses prefix-based queries to directly load persisted DA inclusion data,
346+
// avoiding expensive iteration through all blocks.
346347
func (m *implementation) RestoreFromStore() error {
347348
ctx := context.Background()
348349

349-
// Get current store height to know how many blocks to check
350-
height, err := m.store.Height(ctx)
351-
if err != nil {
352-
return fmt.Errorf("failed to get store height: %w", err)
353-
}
354-
355-
if height == 0 {
356-
return nil // No blocks to restore
357-
}
358-
359-
// Collect hashes from stored blocks
360-
var headerHashes []string
361-
var dataHashes []string
362-
363-
for h := uint64(1); h <= height; h++ {
364-
header, data, err := m.store.GetBlockData(ctx, h)
365-
if err != nil {
366-
m.logger.Warn().Uint64("height", h).Err(err).Msg("failed to get block data during cache restore")
367-
continue
368-
}
369-
370-
if header != nil {
371-
headerHashes = append(headerHashes, header.Hash().String())
372-
}
373-
if data != nil {
374-
dataHashes = append(dataHashes, data.DACommitment().String())
375-
}
376-
}
377-
378350
// Restore DA inclusion data from store
379-
if err := m.headerCache.RestoreFromStore(ctx, headerHashes); err != nil {
351+
if err := m.headerCache.RestoreFromStore(ctx); err != nil {
380352
return fmt.Errorf("failed to restore header cache from store: %w", err)
381353
}
382354

383-
if err := m.dataCache.RestoreFromStore(ctx, dataHashes); err != nil {
355+
if err := m.dataCache.RestoreFromStore(ctx); err != nil {
384356
return fmt.Errorf("failed to restore data cache from store: %w", err)
385357
}
386358

387359
// Initialize DA height from store metadata to ensure DaHeight() is never 0.
388360
m.initDAHeightFromStore(ctx)
389361

390362
m.logger.Info().
391-
Int("header_hashes", len(headerHashes)).
392-
Int("data_hashes", len(dataHashes)).
363+
Int("header_entries", m.headerCache.daIncluded.Len()).
364+
Int("data_entries", m.dataCache.daIncluded.Len()).
393365
Uint64("da_height", m.DaHeight()).
394366
Msg("restored DA inclusion cache from store")
395367

‎pkg/store/store.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ import (
66
"encoding/binary"
77
"errors"
88
"fmt"
9+
"strings"
910

1011
ds "github.com/ipfs/go-datastore"
12+
dsq "github.com/ipfs/go-datastore/query"
1113
"google.golang.org/protobuf/proto"
1214

1315
"github.com/evstack/ev-node/types"
@@ -190,6 +192,39 @@ func (s *DefaultStore) GetMetadata(ctx context.Context, key string) ([]byte, err
190192
return data, nil
191193
}
192194

195+
// GetMetadataByPrefix returns all metadata entries whose keys have the given prefix.
196+
// This is more efficient than iterating through known keys when the set of keys is unknown.
197+
func (s *DefaultStore) GetMetadataByPrefix(ctx context.Context, prefix string) ([]MetadataEntry, error) {
198+
// The full key in the datastore includes the meta prefix
199+
fullPrefix := getMetaKey(prefix)
200+
201+
results, err := s.db.Query(ctx, dsq.Query{Prefix: fullPrefix})
202+
if err != nil {
203+
return nil, fmt.Errorf("failed to query metadata with prefix '%s': %w", prefix, err)
204+
}
205+
defer results.Close()
206+
207+
var entries []MetadataEntry
208+
for result := range results.Next() {
209+
if result.Error != nil {
210+
return nil, fmt.Errorf("error iterating metadata results: %w", result.Error)
211+
}
212+
213+
// Extract the original key by removing the meta prefix
214+
// The key from datastore is like "/m/cache/header-da-included/hash"
215+
// We want to return "cache/header-da-included/hash"
216+
metaKeyPrefix := getMetaKey("")
217+
key := strings.TrimPrefix(result.Key, metaKeyPrefix)
218+
219+
entries = append(entries, MetadataEntry{
220+
Key: key,
221+
Value: result.Value,
222+
})
223+
}
224+
225+
return entries, nil
226+
}
227+
193228
// DeleteMetadata removes a metadata key from the store.
194229
func (s *DefaultStore) DeleteMetadata(ctx context.Context, key string) error {
195230
err := s.db.Delete(ctx, ds.NewKey(getMetaKey(key)))

‎pkg/store/tracing.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,23 @@ func (t *tracedStore) GetMetadata(ctx context.Context, key string) ([]byte, erro
174174
return data, nil
175175
}
176176

177+
func (t *tracedStore) GetMetadataByPrefix(ctx context.Context, prefix string) ([]MetadataEntry, error) {
178+
ctx, span := t.tracer.Start(ctx, "Store.GetMetadataByPrefix",
179+
trace.WithAttributes(attribute.String("prefix", prefix)),
180+
)
181+
defer span.End()
182+
183+
entries, err := t.inner.GetMetadataByPrefix(ctx, prefix)
184+
if err != nil {
185+
span.RecordError(err)
186+
span.SetStatus(codes.Error, err.Error())
187+
return entries, err
188+
}
189+
190+
span.SetAttributes(attribute.Int("entries.count", len(entries)))
191+
return entries, nil
192+
}
193+
177194
func (t *tracedStore) SetMetadata(ctx context.Context, key string, value []byte) error {
178195
ctx, span := t.tracer.Start(ctx, "Store.SetMetadata",
179196
trace.WithAttributes(

‎pkg/store/tracing_test.go‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,20 @@ import (
1717
)
1818

1919
type tracingMockStore struct {
20-
heightFn func(ctx context.Context) (uint64, error)
21-
getBlockDataFn func(ctx context.Context, height uint64) (*types.SignedHeader, *types.Data, error)
22-
getBlockByHashFn func(ctx context.Context, hash []byte) (*types.SignedHeader, *types.Data, error)
23-
getSignatureFn func(ctx context.Context, height uint64) (*types.Signature, error)
24-
getSignatureByHash func(ctx context.Context, hash []byte) (*types.Signature, error)
25-
getHeaderFn func(ctx context.Context, height uint64) (*types.SignedHeader, error)
26-
getStateFn func(ctx context.Context) (types.State, error)
27-
getStateAtHeightFn func(ctx context.Context, height uint64) (types.State, error)
28-
getMetadataFn func(ctx context.Context, key string) ([]byte, error)
29-
setMetadataFn func(ctx context.Context, key string, value []byte) error
30-
deleteMetadataFn func(ctx context.Context, key string) error
31-
rollbackFn func(ctx context.Context, height uint64, aggregator bool) error
32-
newBatchFn func(ctx context.Context) (Batch, error)
20+
heightFn func(ctx context.Context) (uint64, error)
21+
getBlockDataFn func(ctx context.Context, height uint64) (*types.SignedHeader, *types.Data, error)
22+
getBlockByHashFn func(ctx context.Context, hash []byte) (*types.SignedHeader, *types.Data, error)
23+
getSignatureFn func(ctx context.Context, height uint64) (*types.Signature, error)
24+
getSignatureByHash func(ctx context.Context, hash []byte) (*types.Signature, error)
25+
getHeaderFn func(ctx context.Context, height uint64) (*types.SignedHeader, error)
26+
getStateFn func(ctx context.Context) (types.State, error)
27+
getStateAtHeightFn func(ctx context.Context, height uint64) (types.State, error)
28+
getMetadataFn func(ctx context.Context, key string) ([]byte, error)
29+
getMetadataByPrefixFn func(ctx context.Context, prefix string) ([]MetadataEntry, error)
30+
setMetadataFn func(ctx context.Context, key string, value []byte) error
31+
deleteMetadataFn func(ctx context.Context, key string) error
32+
rollbackFn func(ctx context.Context, height uint64, aggregator bool) error
33+
newBatchFn func(ctx context.Context) (Batch, error)
3334
}
3435

3536
func (m *tracingMockStore) Height(ctx context.Context) (uint64, error) {
@@ -95,6 +96,13 @@ func (m *tracingMockStore) GetMetadata(ctx context.Context, key string) ([]byte,
9596
return nil, nil
9697
}
9798

99+
func (m *tracingMockStore) GetMetadataByPrefix(ctx context.Context, prefix string) ([]MetadataEntry, error) {
100+
if m.getMetadataByPrefixFn != nil {
101+
return m.getMetadataByPrefixFn(ctx, prefix)
102+
}
103+
return nil, nil
104+
}
105+
98106
func (m *tracingMockStore) SetMetadata(ctx context.Context, key string, value []byte) error {
99107
if m.setMetadataFn != nil {
100108
return m.setMetadataFn(ctx, key, value)

‎pkg/store/types.go‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,20 +33,33 @@ type Batch interface {
3333
type Store interface {
3434
Rollback
3535
Reader
36+
Metadata
3637

38+
// Close safely closes underlying data storage, to ensure that data is actually saved.
39+
Close() error
40+
41+
// NewBatch creates a new batch for atomic operations.
42+
NewBatch(ctx context.Context) (Batch, error)
43+
}
44+
45+
// MetadataEntry represents a key-value pair from metadata storage.
46+
type MetadataEntry struct {
47+
Key string
48+
Value []byte
49+
}
50+
51+
type Metadata interface {
3752
// SetMetadata saves arbitrary value in the store.
3853
//
3954
// This method enables evolve to safely persist any information.
4055
SetMetadata(ctx context.Context, key string, value []byte) error
4156

57+
// GetMetadataByPrefix returns all metadata entries whose keys have the given prefix.
58+
// This is more efficient than iterating through known keys when the set of keys is unknown.
59+
GetMetadataByPrefix(ctx context.Context, prefix string) ([]MetadataEntry, error)
60+
4261
// DeleteMetadata removes a metadata key from the store.
4362
DeleteMetadata(ctx context.Context, key string) error
44-
45-
// Close safely closes underlying data storage, to ensure that data is actually saved.
46-
Close() error
47-
48-
// NewBatch creates a new batch for atomic operations.
49-
NewBatch(ctx context.Context) (Batch, error)
5063
}
5164

5265
type Reader interface {

‎test/mocks/store.go‎

Lines changed: 68 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)