diff --git a/apps/evm/cmd/run.go b/apps/evm/cmd/run.go index 60730eac52..0136e1eb6c 100644 --- a/apps/evm/cmd/run.go +++ b/apps/evm/cmd/run.go @@ -60,10 +60,11 @@ var RunCmd = &cobra.Command{ return err } - blobClient, err := blobrpc.NewClient(context.Background(), nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") + blobClient, err := blobrpc.NewWSClient(cmd.Context(), nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") if err != nil { return fmt.Errorf("failed to create blob client: %w", err) } + defer blobClient.Close() daClient := block.NewDAClient(blobClient, nodeConfig, logger) diff --git a/apps/evm/server/force_inclusion_test.go b/apps/evm/server/force_inclusion_test.go index d04a356622..a107a10d11 100644 --- a/apps/evm/server/force_inclusion_test.go +++ b/apps/evm/server/force_inclusion_test.go @@ -50,6 +50,13 @@ func (m *mockDA) Get(ctx context.Context, ids []da.ID, namespace []byte) ([]da.B return nil, nil } +func (m *mockDA) Subscribe(_ context.Context, _ []byte, _ bool) (<-chan da.SubscriptionEvent, error) { + // Not needed in these tests; return a closed channel. + ch := make(chan da.SubscriptionEvent) + close(ch) + return ch, nil +} + func (m *mockDA) Validate(ctx context.Context, ids []da.ID, proofs []da.Proof, namespace []byte) ([]bool, error) { return nil, nil } diff --git a/apps/grpc/cmd/run.go b/apps/grpc/cmd/run.go index 41278999cb..6db5a409da 100644 --- a/apps/grpc/cmd/run.go +++ b/apps/grpc/cmd/run.go @@ -108,7 +108,7 @@ func createSequencer( genesis genesis.Genesis, executor execution.Executor, ) (coresequencer.Sequencer, error) { - blobClient, err := blobrpc.NewClient(ctx, nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") + blobClient, err := blobrpc.NewWSClient(ctx, nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") if err != nil { return nil, fmt.Errorf("failed to create blob client: %w", err) } diff --git a/apps/testapp/cmd/run.go b/apps/testapp/cmd/run.go index fe026cdfcc..fff3a9e064 100644 --- a/apps/testapp/cmd/run.go +++ b/apps/testapp/cmd/run.go @@ -111,7 +111,7 @@ func createSequencer( genesis genesis.Genesis, executor execution.Executor, ) (coresequencer.Sequencer, error) { - blobClient, err := blobrpc.NewClient(ctx, nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") + blobClient, err := blobrpc.NewWSClient(ctx, nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") if err != nil { return nil, fmt.Errorf("failed to create blob client: %w", err) } diff --git a/block/components.go b/block/components.go index c54f8d554d..71b6f60523 100644 --- a/block/components.go +++ b/block/components.go @@ -114,7 +114,7 @@ func (bc *Components) Stop() error { } } if bc.Syncer != nil { - if err := bc.Syncer.Stop(); err != nil { + if err := bc.Syncer.Stop(context.Background()); err != nil { errs = errors.Join(errs, fmt.Errorf("failed to stop syncer: %w", err)) } } diff --git a/block/internal/common/event.go b/block/internal/common/event.go index f02a181de8..3cc18f4641 100644 --- a/block/internal/common/event.go +++ b/block/internal/common/event.go @@ -1,6 +1,10 @@ package common -import "github.com/evstack/ev-node/types" +import ( + "context" + + "github.com/evstack/ev-node/types" +) // EventSource represents the origin of a block event type EventSource string @@ -24,3 +28,18 @@ type DAHeightEvent struct { // Optional DA height hints from P2P. first is the DA height hint for the header, second is the DA height hint for the data DaHeightHints [2]uint64 } + +// EventSink receives parsed DA events with backpressure support. +type EventSink interface { + PipeEvent(ctx context.Context, event DAHeightEvent) error +} + +// EventSinkFunc adapts a plain function to the EventSink interface. +// Useful in tests: +// +// sink := common.EventSinkFunc(func(ctx context.Context, ev common.DAHeightEvent) error { return nil }) +type EventSinkFunc func(ctx context.Context, event DAHeightEvent) error + +func (f EventSinkFunc) PipeEvent(ctx context.Context, event DAHeightEvent) error { + return f(ctx, event) +} diff --git a/block/internal/da/async_block_retriever.go b/block/internal/da/async_block_retriever.go index 4fd81d3e9c..6789051514 100644 --- a/block/internal/da/async_block_retriever.go +++ b/block/internal/da/async_block_retriever.go @@ -4,8 +4,6 @@ import ( "context" "errors" "fmt" - "sync" - "sync/atomic" "time" ds "github.com/ipfs/go-datastore" @@ -14,14 +12,13 @@ import ( "github.com/rs/zerolog" "google.golang.org/protobuf/proto" - "github.com/evstack/ev-node/pkg/config" datypes "github.com/evstack/ev-node/pkg/da/types" pb "github.com/evstack/ev-node/types/pb/evnode/v1" ) // AsyncBlockRetriever provides background prefetching of DA blocks type AsyncBlockRetriever interface { - Start() + Start(ctx context.Context) Stop() GetCachedBlock(ctx context.Context, daHeight uint64) (*BlockData, error) UpdateCurrentHeight(height uint64) @@ -35,29 +32,23 @@ type BlockData struct { } // asyncBlockRetriever handles background prefetching of individual DA blocks -// from a specific namespace. +// from a specific namespace. Wraps a da.Subscriber for the subscription +// plumbing and implements SubscriberHandler for caching. type asyncBlockRetriever struct { - client Client - logger zerolog.Logger - namespace []byte - daStartHeight uint64 + subscriber *Subscriber + client Client + logger zerolog.Logger + namespace []byte // In-memory cache for prefetched block data cache ds.Batching - // Background fetcher control - ctx context.Context - cancel context.CancelFunc - wg sync.WaitGroup - - // Current DA height tracking (accessed atomically) - currentDAHeight atomic.Uint64 + // Current DA height tracking (accessed atomically via subscriber). + // Updated externally via UpdateCurrentHeight. + daStartHeight uint64 - // Prefetch window - how many blocks ahead to prefetch + // Prefetch window - how many blocks ahead to speculatively fetch. prefetchWindow uint64 - - // Polling interval for checking new DA heights - pollInterval time.Duration } // NewAsyncBlockRetriever creates a new async block retriever with in-memory cache. @@ -65,61 +56,70 @@ func NewAsyncBlockRetriever( client Client, logger zerolog.Logger, namespace []byte, - config config.Config, + daBlockTime time.Duration, daStartHeight uint64, prefetchWindow uint64, ) AsyncBlockRetriever { if prefetchWindow == 0 { - prefetchWindow = 10 // Default: prefetch next 10 blocks + prefetchWindow = 10 } - ctx, cancel := context.WithCancel(context.Background()) - - fetcher := &asyncBlockRetriever{ + f := &asyncBlockRetriever{ client: client, logger: logger.With().Str("component", "async_block_retriever").Logger(), namespace: namespace, daStartHeight: daStartHeight, cache: dsync.MutexWrap(ds.NewMapDatastore()), - ctx: ctx, - cancel: cancel, prefetchWindow: prefetchWindow, - pollInterval: config.DA.BlockTime.Duration, } - fetcher.currentDAHeight.Store(daStartHeight) - return fetcher + + var namespaces [][]byte + if len(namespace) > 0 { + namespaces = [][]byte{namespace} + } + + f.subscriber = NewSubscriber(SubscriberConfig{ + Client: client, + Logger: logger, + Namespaces: namespaces, + DABlockTime: daBlockTime, + Handler: f, + FetchBlockTimestamp: true, + }) + f.subscriber.SetStartHeight(daStartHeight) + + return f } -// Start begins the background prefetching process. -func (f *asyncBlockRetriever) Start() { - f.wg.Add(1) - go f.backgroundFetchLoop() +// Start begins the subscription follow loop and catchup loop. +func (f *asyncBlockRetriever) Start(ctx context.Context) { + if err := f.subscriber.Start(ctx); err != nil { + f.logger.Warn().Err(err).Msg("failed to start subscriber") + } f.logger.Debug(). Uint64("da_start_height", f.daStartHeight). Uint64("prefetch_window", f.prefetchWindow). - Dur("poll_interval", f.pollInterval). Msg("async block retriever started") } -// Stop gracefully stops the background prefetching process. +// Stop gracefully stops the background goroutines. func (f *asyncBlockRetriever) Stop() { f.logger.Debug().Msg("stopping async block retriever") - f.cancel() - f.wg.Wait() + f.subscriber.Stop() } -// UpdateCurrentHeight updates the current DA height for prefetching. +// UpdateCurrentHeight updates the current DA height. func (f *asyncBlockRetriever) UpdateCurrentHeight(height uint64) { - // Use atomic compare-and-swap to update only if the new height is greater for { - current := f.currentDAHeight.Load() + current := f.subscriber.LocalDAHeight() if height <= current { return } - if f.currentDAHeight.CompareAndSwap(current, height) { + if f.subscriber.CompareAndSwapLocalHeight(current, height) { f.logger.Debug(). Uint64("new_height", height). Msg("updated current DA height") + f.cleanupOldBlocks(context.Background(), height) return } } @@ -149,7 +149,6 @@ func (f *asyncBlockRetriever) GetCachedBlock(ctx context.Context, daHeight uint6 return nil, fmt.Errorf("failed to get cached block: %w", err) } - // Deserialize the cached block var pbBlock pb.BlockData if err := proto.Unmarshal(data, &pbBlock); err != nil { return nil, fmt.Errorf("failed to unmarshal cached block: %w", err) @@ -169,138 +168,103 @@ func (f *asyncBlockRetriever) GetCachedBlock(ctx context.Context, daHeight uint6 return block, nil } -// backgroundFetchLoop runs in the background and prefetches blocks ahead of time. -func (f *asyncBlockRetriever) backgroundFetchLoop() { - defer f.wg.Done() - - ticker := time.NewTicker(f.pollInterval) - defer ticker.Stop() - - for { - select { - case <-f.ctx.Done(): - return - case <-ticker.C: - f.prefetchBlocks() - } +// HandleEvent caches blobs from the subscription inline. +func (f *asyncBlockRetriever) HandleEvent(ctx context.Context, ev datypes.SubscriptionEvent) { + if len(ev.Blobs) > 0 { + f.cacheBlock(ctx, ev.Height, ev.Timestamp, ev.Blobs) } } -// prefetchBlocks prefetches blocks within the prefetch window. -func (f *asyncBlockRetriever) prefetchBlocks() { - if len(f.namespace) == 0 { - return - } - - currentHeight := f.currentDAHeight.Load() - - // Prefetch upcoming blocks - for i := uint64(0); i < f.prefetchWindow; i++ { - targetHeight := currentHeight + i - - // Check if already cached - key := newBlockDataKey(targetHeight) - _, err := f.cache.Get(f.ctx, key) - if err == nil { - // Already cached - continue +// HandleCatchup fetches a single height via Retrieve and caches it. +// Also applies the prefetch window for speculative forward fetching. +func (f *asyncBlockRetriever) HandleCatchup(ctx context.Context, height uint64) error { + f.fetchAndCacheBlock(ctx, height) + + // Speculatively prefetch ahead. + highest := f.subscriber.HighestSeenDAHeight() + target := highest + f.prefetchWindow + for h := height + 1; h <= target; h++ { + if err := ctx.Err(); err != nil { + return err } - - // Fetch and cache the block - f.fetchAndCacheBlock(targetHeight) + key := newBlockDataKey(h) + if _, err := f.cache.Get(ctx, key); err == nil { + continue // Already cached. + } + f.fetchAndCacheBlock(ctx, h) } - // Clean up old blocks from cache to prevent memory growth - f.cleanupOldBlocks(currentHeight) + return nil } -// fetchAndCacheBlock fetches a block and stores it in the cache. -func (f *asyncBlockRetriever) fetchAndCacheBlock(height uint64) { - f.logger.Debug(). - Uint64("height", height). - Msg("prefetching block") +// --------------------------------------------------------------------------- +// Cache helpers +// --------------------------------------------------------------------------- - result := f.client.Retrieve(f.ctx, height, f.namespace) +// fetchAndCacheBlock fetches a block via Retrieve and caches it. +func (f *asyncBlockRetriever) fetchAndCacheBlock(ctx context.Context, height uint64) { + f.logger.Debug().Uint64("height", height).Msg("prefetching block") - block := &BlockData{ - Height: height, - Timestamp: result.Timestamp, - Blobs: [][]byte{}, - } + result := f.client.Retrieve(ctx, height, f.namespace) switch result.Code { case datypes.StatusHeightFromFuture: - f.logger.Debug(). - Uint64("height", height). - Msg("block height not yet available - will retry") + f.logger.Debug().Uint64("height", height).Msg("block height not yet available - will retry") return case datypes.StatusNotFound: - f.logger.Debug(). - Uint64("height", height). - Msg("no blobs at height") - // Cache empty result to avoid re-fetching + f.cacheBlock(ctx, height, result.Timestamp, nil) case datypes.StatusSuccess: - // Process each blob + blobs := make([][]byte, 0, len(result.Data)) for _, blob := range result.Data { if len(blob) > 0 { - block.Blobs = append(block.Blobs, blob) + blobs = append(blobs, blob) } } - f.logger.Debug(). - Uint64("height", height). - Int("blob_count", len(result.Data)). - Msg("processed blobs for prefetch") + f.cacheBlock(ctx, height, result.Timestamp, blobs) default: f.logger.Debug(). Uint64("height", height). Str("status", result.Message). Msg("failed to retrieve block - will retry") - return + } +} + +// cacheBlock serializes and stores a block in the in-memory cache. +func (f *asyncBlockRetriever) cacheBlock(ctx context.Context, daHeight uint64, daTimestamp time.Time, blobs [][]byte) { + if blobs == nil { + blobs = [][]byte{} } - // Serialize and cache the block pbBlock := &pb.BlockData{ - Height: block.Height, - Timestamp: block.Timestamp.UnixNano(), - Blobs: block.Blobs, + Height: daHeight, + Timestamp: daTimestamp.UnixNano(), + Blobs: blobs, } data, err := proto.Marshal(pbBlock) if err != nil { - f.logger.Error(). - Err(err). - Uint64("height", height). - Msg("failed to marshal block for caching") + f.logger.Error().Err(err).Uint64("height", daHeight).Msg("failed to marshal block for caching") return } - key := newBlockDataKey(height) - err = f.cache.Put(f.ctx, key, data) - if err != nil { - f.logger.Error(). - Err(err). - Uint64("height", height). - Msg("failed to cache block") + key := newBlockDataKey(daHeight) + if err := f.cache.Put(ctx, key, data); err != nil { + f.logger.Error().Err(err).Uint64("height", daHeight).Msg("failed to cache block") return } - f.logger.Debug(). - Uint64("height", height). - Int("blob_count", len(block.Blobs)). - Msg("successfully prefetched and cached block") + f.logger.Debug().Uint64("height", daHeight).Int("blob_count", len(blobs)).Msg("cached block") } -// cleanupOldBlocks removes blocks older than a threshold from cache. -func (f *asyncBlockRetriever) cleanupOldBlocks(currentHeight uint64) { - // Remove blocks older than current - prefetchWindow +// cleanupOldBlocks removes blocks older than currentHeight − prefetchWindow. +func (f *asyncBlockRetriever) cleanupOldBlocks(ctx context.Context, currentHeight uint64) { if currentHeight < f.prefetchWindow { return } cleanupThreshold := currentHeight - f.prefetchWindow - // Query all keys query := dsq.Query{Prefix: "/block/"} - results, err := f.cache.Query(f.ctx, query) + results, err := f.cache.Query(ctx, query) if err != nil { f.logger.Debug().Err(err).Msg("failed to query cache for cleanup") return @@ -313,7 +277,6 @@ func (f *asyncBlockRetriever) cleanupOldBlocks(currentHeight uint64) { } key := ds.NewKey(result.Key) - // Extract height from key var height uint64 _, err := fmt.Sscanf(key.String(), "/block/%d", &height) if err != nil { @@ -321,11 +284,8 @@ func (f *asyncBlockRetriever) cleanupOldBlocks(currentHeight uint64) { } if height < cleanupThreshold { - if err := f.cache.Delete(f.ctx, key); err != nil { - f.logger.Debug(). - Err(err). - Uint64("height", height). - Msg("failed to delete old block from cache") + if err := f.cache.Delete(ctx, key); err != nil { + f.logger.Debug().Err(err).Uint64("height", height).Msg("failed to delete old block from cache") } } } diff --git a/block/internal/da/async_block_retriever_test.go b/block/internal/da/async_block_retriever_test.go index dcfbce84d1..1ecaca9a88 100644 --- a/block/internal/da/async_block_retriever_test.go +++ b/block/internal/da/async_block_retriever_test.go @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" - "github.com/evstack/ev-node/pkg/config" datypes "github.com/evstack/ev-node/pkg/da/types" mocks "github.com/evstack/ev-node/test/mocks" pb "github.com/evstack/ev-node/types/pb/evnode/v1" @@ -21,7 +20,7 @@ func TestAsyncBlockRetriever_GetCachedBlock_NoNamespace(t *testing.T) { client := &mocks.MockClient{} logger := zerolog.Nop() - fetcher := NewAsyncBlockRetriever(client, logger, nil, config.DefaultConfig(), 100, 10) + fetcher := NewAsyncBlockRetriever(client, logger, nil, 100*time.Millisecond, 100, 10) ctx := context.Background() block, err := fetcher.GetCachedBlock(ctx, 100) @@ -34,7 +33,7 @@ func TestAsyncBlockRetriever_GetCachedBlock_CacheMiss(t *testing.T) { fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() logger := zerolog.Nop() - fetcher := NewAsyncBlockRetriever(client, logger, fiNs, config.DefaultConfig(), 100, 10) + fetcher := NewAsyncBlockRetriever(client, logger, fiNs, 100*time.Millisecond, 100, 10) ctx := context.Background() @@ -44,141 +43,143 @@ func TestAsyncBlockRetriever_GetCachedBlock_CacheMiss(t *testing.T) { assert.Nil(t, block) // Cache miss } -func TestAsyncBlockRetriever_FetchAndCache(t *testing.T) { +func TestAsyncBlockRetriever_SubscriptionDrivenCaching(t *testing.T) { + // Test that blobs arriving via subscription are cached inline. testBlobs := [][]byte{ []byte("tx1"), []byte("tx2"), - []byte("tx3"), } client := &mocks.MockClient{} fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() - // Mock Retrieve call for height 100 - client.On("Retrieve", mock.Anything, uint64(100), fiNs).Return(datypes.ResultRetrieve{ - BaseResult: datypes.BaseResult{ - Code: datypes.StatusSuccess, - Timestamp: time.Unix(1000, 0), - }, - Data: testBlobs, - }).Once() - - // Mock other heights that will be prefetched - for height := uint64(101); height <= 109; height++ { - client.On("Retrieve", mock.Anything, height, fiNs).Return(datypes.ResultRetrieve{ - BaseResult: datypes.BaseResult{Code: datypes.StatusNotFound}, - }).Maybe() + // Create a subscription channel that delivers one event then blocks. + subCh := make(chan datypes.SubscriptionEvent, 1) + subCh <- datypes.SubscriptionEvent{ + Height: 100, + Blobs: testBlobs, } + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() + // Catchup loop may call Retrieve for heights beyond 100 — stub those. + client.On("Retrieve", mock.Anything, mock.Anything, fiNs).Return(datypes.ResultRetrieve{ + BaseResult: datypes.BaseResult{Code: datypes.StatusHeightFromFuture}, + }).Maybe() + + // On second subscribe (after watchdog timeout) just block forever. + blockCh := make(chan datypes.SubscriptionEvent) + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() + logger := zerolog.Nop() - // Use a short poll interval for faster test execution - cfg := config.DefaultConfig() - cfg.DA.BlockTime.Duration = 100 * time.Millisecond - fetcher := NewAsyncBlockRetriever(client, logger, fiNs, cfg, 100, 10) - fetcher.Start() - defer fetcher.Stop() + fetcher := NewAsyncBlockRetriever(client, logger, fiNs, 200*time.Millisecond, 100, 5) - // Update current height to trigger prefetch - fetcher.UpdateCurrentHeight(100) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + fetcher.Start(ctx) + defer fetcher.Stop() - // Wait for the background fetch to complete by polling the cache - ctx := context.Background() + // Wait for the subscription event to be processed. var block *BlockData var err error - - // Poll for up to 2 seconds for the block to be cached for range 40 { block, err = fetcher.GetCachedBlock(ctx, 100) require.NoError(t, err) if block != nil { break } - time.Sleep(50 * time.Millisecond) + time.Sleep(25 * time.Millisecond) } - require.NotNil(t, block, "block should be cached after background fetch") + require.NotNil(t, block, "block should be cached after subscription event") assert.Equal(t, uint64(100), block.Height) - assert.Equal(t, 3, len(block.Blobs)) - for i, tb := range testBlobs { - assert.Equal(t, tb, block.Blobs[i]) - } + assert.Equal(t, 2, len(block.Blobs)) + assert.Equal(t, []byte("tx1"), block.Blobs[0]) + assert.Equal(t, []byte("tx2"), block.Blobs[1]) } -func TestAsyncBlockRetriever_BackgroundPrefetch(t *testing.T) { - testBlobs := [][]byte{ - []byte("tx1"), - []byte("tx2"), - } +func TestAsyncBlockRetriever_CatchupFillsGaps(t *testing.T) { + // When subscription reports height 105 but current is 100, + // catchup loop should Retrieve heights 100-114 (100 + prefetch window). + testBlobs := [][]byte{[]byte("gap-tx")} client := &mocks.MockClient{} fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() - // Mock for heights 100-110 (current + prefetch window) - for height := uint64(100); height <= 110; height++ { - if height == 105 { - client.On("Retrieve", mock.Anything, height, fiNs).Return(datypes.ResultRetrieve{ - BaseResult: datypes.BaseResult{ - Code: datypes.StatusSuccess, - Timestamp: time.Unix(2000, 0), - }, - Data: testBlobs, - }).Maybe() - } else { - client.On("Retrieve", mock.Anything, height, fiNs).Return(datypes.ResultRetrieve{ - BaseResult: datypes.BaseResult{Code: datypes.StatusNotFound}, - }).Maybe() - } - } + // Subscription delivers height 105 (no blobs — just a signal). + subCh := make(chan datypes.SubscriptionEvent, 1) + subCh <- datypes.SubscriptionEvent{Height: 105} - logger := zerolog.Nop() - cfg := config.DefaultConfig() - cfg.DA.BlockTime.Duration = 100 * time.Millisecond - fetcher := NewAsyncBlockRetriever(client, logger, fiNs, cfg, 100, 10) + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() + blockCh := make(chan datypes.SubscriptionEvent) + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() - fetcher.Start() - defer fetcher.Stop() + // Height 102 has blobs; rest return not found or future. + client.On("Retrieve", mock.Anything, uint64(102), fiNs).Return(datypes.ResultRetrieve{ + BaseResult: datypes.BaseResult{ + Code: datypes.StatusSuccess, + Timestamp: time.Unix(2000, 0), + }, + Data: testBlobs, + }).Maybe() + client.On("Retrieve", mock.Anything, mock.Anything, fiNs).Return(datypes.ResultRetrieve{ + BaseResult: datypes.BaseResult{Code: datypes.StatusNotFound}, + }).Maybe() - // Update current height to trigger prefetch - fetcher.UpdateCurrentHeight(100) + logger := zerolog.Nop() + fetcher := NewAsyncBlockRetriever(client, logger, fiNs, 100*time.Millisecond, 100, 10) - // Wait for background prefetch to happen (wait for at least one poll cycle) - time.Sleep(250 * time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + fetcher.Start(ctx) + defer fetcher.Stop() - // Check if block was prefetched - ctx := context.Background() - block, err := fetcher.GetCachedBlock(ctx, 105) - require.NoError(t, err) - assert.NotNil(t, block) - assert.Equal(t, uint64(105), block.Height) - assert.Equal(t, 2, len(block.Blobs)) + // Wait for catchup to fill the gap. + var block *BlockData + var err error + for range 40 { + block, err = fetcher.GetCachedBlock(ctx, 102) + require.NoError(t, err) + if block != nil { + break + } + time.Sleep(50 * time.Millisecond) + } + require.NotNil(t, block, "block at 102 should be cached via catchup") + assert.Equal(t, uint64(102), block.Height) + assert.Equal(t, 1, len(block.Blobs)) + assert.Equal(t, []byte("gap-tx"), block.Blobs[0]) } func TestAsyncBlockRetriever_HeightFromFuture(t *testing.T) { client := &mocks.MockClient{} fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() - // All heights in prefetch window not available yet - for height := uint64(100); height <= 109; height++ { - client.On("Retrieve", mock.Anything, height, fiNs).Return(datypes.ResultRetrieve{ - BaseResult: datypes.BaseResult{Code: datypes.StatusHeightFromFuture}, - }).Maybe() - } + // Subscription delivers height 100 with no blobs. + subCh := make(chan datypes.SubscriptionEvent, 1) + subCh <- datypes.SubscriptionEvent{Height: 100} + + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() + blockCh := make(chan datypes.SubscriptionEvent) + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() + + // All Retrieve calls return HeightFromFuture. + client.On("Retrieve", mock.Anything, mock.Anything, fiNs).Return(datypes.ResultRetrieve{ + BaseResult: datypes.BaseResult{Code: datypes.StatusHeightFromFuture}, + }).Maybe() logger := zerolog.Nop() - cfg := config.DefaultConfig() - cfg.DA.BlockTime.Duration = 100 * time.Millisecond - fetcher := NewAsyncBlockRetriever(client, logger, fiNs, cfg, 100, 10) - fetcher.Start() - defer fetcher.Stop() + fetcher := NewAsyncBlockRetriever(client, logger, fiNs, 100*time.Millisecond, 100, 10) - fetcher.UpdateCurrentHeight(100) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + fetcher.Start(ctx) + defer fetcher.Stop() - // Wait for at least one poll cycle + // Wait a bit for catchup to attempt fetches. time.Sleep(250 * time.Millisecond) - // Cache should be empty - ctx := context.Background() + // Cache should be empty since all heights are from the future. block, err := fetcher.GetCachedBlock(ctx, 100) require.NoError(t, err) assert.Nil(t, block) @@ -188,16 +189,76 @@ func TestAsyncBlockRetriever_StopGracefully(t *testing.T) { client := &mocks.MockClient{} fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() + blockCh := make(chan datypes.SubscriptionEvent) + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() + logger := zerolog.Nop() - fetcher := NewAsyncBlockRetriever(client, logger, fiNs, config.DefaultConfig(), 100, 10) + fetcher := NewAsyncBlockRetriever(client, logger, fiNs, 100*time.Millisecond, 100, 10) - fetcher.Start() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + fetcher.Start(ctx) time.Sleep(100 * time.Millisecond) // Should stop gracefully without panic fetcher.Stop() } +func TestAsyncBlockRetriever_ReconnectOnSubscriptionError(t *testing.T) { + // Verify that the follow loop reconnects after a subscription channel closes. + testBlobs := [][]byte{[]byte("reconnect-tx")} + + client := &mocks.MockClient{} + fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() + + // First subscription closes immediately (simulating error). + closedCh := make(chan datypes.SubscriptionEvent) + close(closedCh) + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(closedCh), nil).Once() + + // Second subscription delivers a blob. + subCh := make(chan datypes.SubscriptionEvent, 1) + subCh <- datypes.SubscriptionEvent{ + Height: 100, + Blobs: testBlobs, + } + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() + + // Third+ subscribe returns a blocking channel so it doesn't loop forever. + blockCh := make(chan datypes.SubscriptionEvent) + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() + + // Stub Retrieve for catchup. + client.On("Retrieve", mock.Anything, mock.Anything, fiNs).Return(datypes.ResultRetrieve{ + BaseResult: datypes.BaseResult{Code: datypes.StatusHeightFromFuture}, + }).Maybe() + + logger := zerolog.Nop() + // Very short backoff so reconnect is fast in tests. + fetcher := NewAsyncBlockRetriever(client, logger, fiNs, 50*time.Millisecond, 100, 5) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + fetcher.Start(ctx) + defer fetcher.Stop() + + // Wait for reconnect + event processing. + var block *BlockData + var err error + for range 60 { + block, err = fetcher.GetCachedBlock(ctx, 100) + require.NoError(t, err) + if block != nil { + break + } + time.Sleep(50 * time.Millisecond) + } + + require.NotNil(t, block, "block should be cached after reconnect") + assert.Equal(t, 1, len(block.Blobs)) + assert.Equal(t, []byte("reconnect-tx"), block.Blobs[0]) +} + func TestBlockData_Serialization(t *testing.T) { block := &BlockData{ Height: 100, diff --git a/block/internal/da/client.go b/block/internal/da/client.go index a92a9eef28..0b30ed6466 100644 --- a/block/internal/da/client.go +++ b/block/internal/da/client.go @@ -350,6 +350,88 @@ func (c *client) HasForcedInclusionNamespace() bool { return c.hasForcedNamespace } +// Subscribe subscribes to blobs in the given namespace via the celestia-node +// Subscribe API. It returns a channel that emits a SubscriptionEvent for every +// DA block containing a matching blob. The channel is closed when ctx is +// cancelled. The caller must drain the channel after cancellation to avoid +// goroutine leaks. +// Timestamps are included from the header if available (celestia-node v0.29.1+§), otherwise +// fetched via a separate call when includeTimestamp is true. Be aware that fetching timestamps +// separately is an additional call to the celestia node for each event. +func (c *client) Subscribe(ctx context.Context, namespace []byte, includeTimestamp bool) (<-chan datypes.SubscriptionEvent, error) { + ns, err := share.NewNamespaceFromBytes(namespace) + if err != nil { + return nil, fmt.Errorf("invalid namespace: %w", err) + } + + rawCh, err := c.blobAPI.Subscribe(ctx, ns) + if err != nil { + return nil, fmt.Errorf("blob subscribe: %w", err) + } + + out := make(chan datypes.SubscriptionEvent, 16) + go func() { + defer close(out) + for { + select { + case <-ctx.Done(): + return + case resp, ok := <-rawCh: + if !ok { + return + } + if resp == nil { + continue + } + var blockTime time.Time + // Use header time if available (celestia-node v0.21.0+) + if resp.Header != nil && !resp.Header.Time.IsZero() { + blockTime = resp.Header.Time + } else if includeTimestamp { + // Fallback to fetching timestamp for older nodes + blockTime, err = c.getBlockTimestamp(ctx, resp.Height) + if err != nil { + c.logger.Error().Uint64("height", resp.Height).Err(err).Msg("failed to get DA block timestamp for subscription event") + blockTime = time.Now() + // TODO: we should retry fetching the timestamp. Current time may mess block time consistency for based sequencers. + } + } + select { + case out <- datypes.SubscriptionEvent{ + Height: resp.Height, + Timestamp: blockTime, + Blobs: extractBlobData(resp), + }: + case <-ctx.Done(): + return + } + } + } + }() + + return out, nil +} + +// extractBlobData extracts raw byte slices from a subscription response, +// filtering out nil blobs, empty data, and blobs exceeding DefaultMaxBlobSize. +func extractBlobData(resp *blobrpc.SubscriptionResponse) [][]byte { + if resp == nil || len(resp.Blobs) == 0 { + return nil + } + blobs := make([][]byte, 0, len(resp.Blobs)) + for _, blob := range resp.Blobs { + if blob == nil { + continue + } + data := blob.Data() + if len(data) == 0 || len(data) > common.DefaultMaxBlobSize { + continue + } + blobs = append(blobs, data) + } + return blobs +} + // Get fetches blobs by their IDs. Used for visualization and fetching specific blobs. func (c *client) Get(ctx context.Context, ids []datypes.ID, namespace []byte) ([]datypes.Blob, error) { if len(ids) == 0 { diff --git a/block/internal/da/forced_inclusion_retriever.go b/block/internal/da/forced_inclusion_retriever.go index 2ec299333a..a8051a5758 100644 --- a/block/internal/da/forced_inclusion_retriever.go +++ b/block/internal/da/forced_inclusion_retriever.go @@ -8,7 +8,6 @@ import ( "github.com/rs/zerolog" - "github.com/evstack/ev-node/pkg/config" datypes "github.com/evstack/ev-node/pkg/da/types" "github.com/evstack/ev-node/types" ) @@ -42,9 +41,11 @@ type ForcedInclusionEvent struct { // NewForcedInclusionRetriever creates a new forced inclusion retriever. // It internally creates and manages an AsyncBlockRetriever for background prefetching. func NewForcedInclusionRetriever( + ctx context.Context, client Client, logger zerolog.Logger, - cfg config.Config, + daBlockTime time.Duration, + tracingEnabled bool, daStartHeight, daEpochSize uint64, ) ForcedInclusionRetriever { retrieverLogger := logger.With().Str("component", "forced_inclusion_retriever").Logger() @@ -54,11 +55,11 @@ func NewForcedInclusionRetriever( client, logger, client.GetForcedInclusionNamespace(), - cfg, + daBlockTime, daStartHeight, daEpochSize*2, // prefetch window: 2x epoch size ) - asyncFetcher.Start() + asyncFetcher.Start(ctx) base := &forcedInclusionRetriever{ client: client, @@ -67,7 +68,7 @@ func NewForcedInclusionRetriever( daEpochSize: daEpochSize, asyncFetcher: asyncFetcher, } - if cfg.Instrumentation.IsTracingEnabled() { + if tracingEnabled { return withTracingForcedInclusionRetriever(base) } return base diff --git a/block/internal/da/forced_inclusion_retriever_test.go b/block/internal/da/forced_inclusion_retriever_test.go index 446b655bec..09a5e5b077 100644 --- a/block/internal/da/forced_inclusion_retriever_test.go +++ b/block/internal/da/forced_inclusion_retriever_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/mock" "gotest.tools/v3/assert" - "github.com/evstack/ev-node/pkg/config" datypes "github.com/evstack/ev-node/pkg/da/types" "github.com/evstack/ev-node/pkg/genesis" "github.com/evstack/ev-node/test/mocks" @@ -18,13 +17,14 @@ import ( func TestNewForcedInclusionRetriever(t *testing.T) { client := mocks.NewMockClient(t) client.On("GetForcedInclusionNamespace").Return(datypes.NamespaceFromString("test-fi-ns").Bytes()).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() gen := genesis.Genesis{ DAStartHeight: 100, DAEpochForcedInclusion: 10, } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) assert.Assert(t, retriever != nil) retriever.Stop() } @@ -39,7 +39,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_NoNamespace(t *testi DAEpochForcedInclusion: 10, } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -53,13 +53,14 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_NotAtEpochStart(t *t fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() gen := genesis.Genesis{ DAStartHeight: 100, DAEpochForcedInclusion: 10, } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -83,6 +84,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EpochStartSuccess(t fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, mock.Anything, fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusSuccess, IDs: []datypes.ID{[]byte("id1"), []byte("id2"), []byte("id3")}, Timestamp: time.Now()}, Data: testBlobs, @@ -93,7 +95,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EpochStartSuccess(t DAEpochForcedInclusion: 1, // Single height epoch } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -112,6 +114,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EpochStartNotAvailab fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // Mock the first height in epoch as not available client.On("Retrieve", mock.Anything, uint64(100), fiNs).Return(datypes.ResultRetrieve{ @@ -123,7 +126,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EpochStartNotAvailab DAEpochForcedInclusion: 10, } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -138,6 +141,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_NoBlobsAtHeight(t *t fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, uint64(100), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusNotFound}, }).Once() @@ -147,7 +151,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_NoBlobsAtHeight(t *t DAEpochForcedInclusion: 1, // Single height epoch } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -168,6 +172,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_MultiHeightEpoch(t * fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, uint64(102), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusSuccess, Timestamp: time.Now()}, Data: testBlobsByHeight[102], @@ -186,7 +191,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_MultiHeightEpoch(t * DAEpochForcedInclusion: 3, // Epoch: 100-102 } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -208,6 +213,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_ErrorHandling(t *tes fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, uint64(100), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{ Code: datypes.StatusError, @@ -220,7 +226,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_ErrorHandling(t *tes DAEpochForcedInclusion: 1, // Single height epoch } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -236,6 +242,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EmptyBlobsSkipped(t fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, uint64(100), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusSuccess, Timestamp: time.Now()}, Data: [][]byte{[]byte("tx1"), {}, []byte("tx2"), nil, []byte("tx3")}, @@ -246,7 +253,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EmptyBlobsSkipped(t DAEpochForcedInclusion: 1, // Single height epoch } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -272,6 +279,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_OrderPreserved(t *te fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // Return heights out of order to test ordering is preserved client.On("Retrieve", mock.Anything, uint64(102), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusSuccess, Timestamp: time.Now()}, @@ -291,7 +299,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_OrderPreserved(t *te DAEpochForcedInclusion: 3, // Epoch: 100-102 } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() diff --git a/block/internal/da/interface.go b/block/internal/da/interface.go index dd7a15d8f9..812d12847b 100644 --- a/block/internal/da/interface.go +++ b/block/internal/da/interface.go @@ -17,6 +17,12 @@ type Client interface { // Get retrieves blobs by their IDs. Used for visualization and fetching specific blobs. Get(ctx context.Context, ids []datypes.ID, namespace []byte) ([]datypes.Blob, error) + // Subscribe returns a channel that emits one SubscriptionEvent per DA block + // that contains a blob in the given namespace. The channel is closed when ctx + // is cancelled. Callers MUST drain the channel after cancellation. + // The fetchTimestamp param is going to be removed with https://github.com/evstack/ev-node/issues/3142 as the timestamp is going to be included by default + Subscribe(ctx context.Context, namespace []byte, fetchTimestamp bool) (<-chan datypes.SubscriptionEvent, error) + // GetLatestDAHeight returns the latest height available on the DA layer. GetLatestDAHeight(ctx context.Context) (uint64, error) diff --git a/block/internal/da/subscriber.go b/block/internal/da/subscriber.go new file mode 100644 index 0000000000..779ff559f3 --- /dev/null +++ b/block/internal/da/subscriber.go @@ -0,0 +1,389 @@ +package da + +import ( + "bytes" + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/rs/zerolog" + + datypes "github.com/evstack/ev-node/pkg/da/types" +) + +// SubscriberHandler is the callback interface for subscription consumers. +// Implementations drive the consumer-specific logic (caching, piping events, etc.). +type SubscriberHandler interface { + // HandleEvent processes a subscription event inline (fast path). + // Called on the followLoop goroutine for each subscription event. + HandleEvent(ctx context.Context, ev datypes.SubscriptionEvent) + + // HandleCatchup is called for each height during sequential catchup. + // The subscriber advances localDAHeight only after this returns nil. + // Returning an error triggers a backoff retry. + HandleCatchup(ctx context.Context, height uint64) error +} + +// SubscriberConfig holds configuration for creating a Subscriber. +type SubscriberConfig struct { + Client Client + Logger zerolog.Logger + Namespaces [][]byte // subscribe to all, merge into one channel + DABlockTime time.Duration + Handler SubscriberHandler + // Deprecated: Remove with https://github.com/evstack/ev-node/issues/3142 + FetchBlockTimestamp bool // the timestamp comes with an extra api call before Celestia v0.29.1-mocha. +} + +// Subscriber is a shared DA subscription primitive that encapsulates the +// follow/catchup lifecycle. It subscribes to one or more DA namespaces, +// tracks the highest seen DA height, and drives sequential catchup via +// callbacks on SubscriberHandler. +// +// Used by both DAFollower (syncing) and asyncBlockRetriever (forced inclusion). +type Subscriber struct { + client Client + logger zerolog.Logger + handler SubscriberHandler + + // namespaces to subscribe on. When multiple, they are merged. + namespaces [][]byte + + // localDAHeight is only written by catchupLoop (via CAS) and read by + // followLoop to determine whether inline processing is possible. + localDAHeight atomic.Uint64 + + // highestSeenDAHeight is written by followLoop and read by catchupLoop. + highestSeenDAHeight atomic.Uint64 + + // headReached tracks whether the subscriber has caught up to DA head. + headReached atomic.Bool + + // catchupSignal wakes catchupLoop when a new height is seen above localDAHeight. + catchupSignal chan struct{} + + // daBlockTime used as backoff and watchdog base. + daBlockTime time.Duration + + // Deprecated: Remove with https://github.com/evstack/ev-node/issues/3142 + fetchBlockTimestamp bool + + // lifecycle + cancel context.CancelFunc + wg sync.WaitGroup +} + +// NewSubscriber creates a new Subscriber. +func NewSubscriber(cfg SubscriberConfig) *Subscriber { + s := &Subscriber{ + client: cfg.Client, + logger: cfg.Logger, + handler: cfg.Handler, + namespaces: cfg.Namespaces, + catchupSignal: make(chan struct{}, 1), + daBlockTime: cfg.DABlockTime, + fetchBlockTimestamp: cfg.FetchBlockTimestamp, + } + return s +} + +// SetStartHeight sets the initial local DA height before Start is called. +func (s *Subscriber) SetStartHeight(height uint64) { + s.localDAHeight.Store(height) +} + +// Start begins the follow and catchup goroutines. +func (s *Subscriber) Start(ctx context.Context) error { + if len(s.namespaces) == 0 { + return errors.New("no namespaces configured") + } + + ctx, s.cancel = context.WithCancel(ctx) + s.wg.Add(2) + go s.followLoop(ctx) + go s.catchupLoop(ctx) + return nil +} + +// Stop gracefully stops the background goroutines. +func (s *Subscriber) Stop() { + if s.cancel != nil { + s.cancel() + } + s.wg.Wait() +} + +// LocalDAHeight returns the current local DA height. +func (s *Subscriber) LocalDAHeight() uint64 { + return s.localDAHeight.Load() +} + +// HighestSeenDAHeight returns the highest DA height seen from the subscription. +func (s *Subscriber) HighestSeenDAHeight() uint64 { + return s.highestSeenDAHeight.Load() +} + +// HasReachedHead returns whether the subscriber has caught up to DA head. +func (s *Subscriber) HasReachedHead() bool { + return s.headReached.Load() +} + +// SetHeadReached marks the subscriber as having reached DA head. +func (s *Subscriber) SetHeadReached() { + s.headReached.Store(true) +} + +// CompareAndSwapLocalHeight attempts a CAS on localDAHeight. +// Used by handlers that want to claim exclusive processing of a height. +func (s *Subscriber) CompareAndSwapLocalHeight(old, new uint64) bool { + return s.localDAHeight.CompareAndSwap(old, new) +} + +// SetLocalHeight stores a new localDAHeight value. +func (s *Subscriber) SetLocalHeight(height uint64) { + s.localDAHeight.Store(height) +} + +// UpdateHighestForTest directly sets the highest seen DA height and signals catchup. +// Only for use in tests that bypass the subscription loop. +func (s *Subscriber) UpdateHighestForTest(height uint64) { + s.updateHighest(height) +} + +// RunCatchupForTest runs a single catchup pass. Only for use in tests that +// bypass the catchup loop's signal-wait mechanism. +func (s *Subscriber) RunCatchupForTest(ctx context.Context) { + s.runCatchup(ctx) +} + +// --------------------------------------------------------------------------- +// Follow loop +// --------------------------------------------------------------------------- + +// signalCatchup sends a non-blocking signal to wake catchupLoop. +func (s *Subscriber) signalCatchup() { + select { + case s.catchupSignal <- struct{}{}: + default: + } +} + +// followLoop subscribes to DA blob events and keeps highestSeenDAHeight up to date. +func (s *Subscriber) followLoop(ctx context.Context) { + defer s.wg.Done() + + s.logger.Info().Msg("starting follow loop") + defer s.logger.Info().Msg("follow loop stopped") + + for { + if err := s.runSubscription(ctx); err != nil { + if ctx.Err() != nil { + return + } + s.logger.Warn().Err(err).Msg("DA subscription failed, reconnecting") + select { + case <-ctx.Done(): + return + case <-time.After(s.backoff()): + } + } + } +} + +// runSubscription opens subscriptions on all namespaces (merging if more than one) +// and processes events until a channel is closed or the watchdog times out. +func (s *Subscriber) runSubscription(ctx context.Context) error { + subCtx, subCancel := context.WithCancel(ctx) + defer subCancel() + + ch, err := s.subscribe(subCtx) + if err != nil { + return err + } + + watchdogTimeout := s.watchdogTimeout() + watchdog := time.NewTimer(watchdogTimeout) + defer watchdog.Stop() + + for { + select { + case <-subCtx.Done(): + return subCtx.Err() + case ev, ok := <-ch: + if !ok { + return errors.New("subscription channel closed") + } + s.updateHighest(ev.Height) + s.handler.HandleEvent(ctx, ev) + watchdog.Reset(watchdogTimeout) + case <-watchdog.C: + return errors.New("subscription watchdog: no events received, reconnecting") + } + } +} + +// subscribe opens subscriptions on all configured namespaces. When there are +// multiple distinct namespaces, channels are merged via mergeSubscriptions. +func (s *Subscriber) subscribe(ctx context.Context) (<-chan datypes.SubscriptionEvent, error) { + if len(s.namespaces) == 0 { + return nil, errors.New("no namespaces configured") + } + + // Subscribe to the first namespace. + ch, err := s.client.Subscribe(ctx, s.namespaces[0], s.fetchBlockTimestamp) + if err != nil { + return nil, fmt.Errorf("subscribe namespace 0: %w", err) + } + + // Subscribe to additional namespaces and merge. + for i := 1; i < len(s.namespaces); i++ { + if bytes.Equal(s.namespaces[i], s.namespaces[0]) { + continue // Same namespace, skip duplicate. + } + ch2, err := s.client.Subscribe(ctx, s.namespaces[i], s.fetchBlockTimestamp) + if err != nil { + return nil, fmt.Errorf("subscribe namespace %d: %w", i, err) + } + ch = mergeSubscriptions(ctx, ch, ch2) + } + + return ch, nil +} + +// mergeSubscriptions fans two subscription channels into one. +func mergeSubscriptions( + ctx context.Context, + ch1, ch2 <-chan datypes.SubscriptionEvent, +) <-chan datypes.SubscriptionEvent { + out := make(chan datypes.SubscriptionEvent, 16) + go func() { + defer close(out) + for ch1 != nil || ch2 != nil { + var ev datypes.SubscriptionEvent + var ok bool + select { + case <-ctx.Done(): + return + case ev, ok = <-ch1: + if !ok { + ch1 = nil + continue + } + case ev, ok = <-ch2: + if !ok { + ch2 = nil + continue + } + } + select { + case out <- ev: + case <-ctx.Done(): + return + } + } + }() + return out +} + +// updateHighest atomically bumps highestSeenDAHeight and signals catchup if needed. +func (s *Subscriber) updateHighest(height uint64) { + for { + cur := s.highestSeenDAHeight.Load() + if height <= cur { + return + } + if s.highestSeenDAHeight.CompareAndSwap(cur, height) { + s.signalCatchup() + return + } + } +} + +// catchupLoop waits for signals and sequentially catches up. +func (s *Subscriber) catchupLoop(ctx context.Context) { + defer s.wg.Done() + + s.logger.Info().Msg("starting catchup loop") + defer s.logger.Info().Msg("catchup loop stopped") + + for { + select { + case <-ctx.Done(): + return + case <-s.catchupSignal: + s.runCatchup(ctx) + } + } +} + +// runCatchup sequentially calls HandleCatchup from localDAHeight to highestSeenDAHeight. +func (s *Subscriber) runCatchup(ctx context.Context) { + for { + if ctx.Err() != nil { + return + } + + local := s.localDAHeight.Load() + highest := s.highestSeenDAHeight.Load() + + if local > highest { + s.headReached.Store(true) + return + } + + // CAS claims this height — prevents followLoop from inline-processing. + if !s.localDAHeight.CompareAndSwap(local, local+1) { + // followLoop already advanced past this height via inline processing. + continue + } + + if err := s.handler.HandleCatchup(ctx, local); err != nil { + // Roll back so we can retry after backoff. + s.localDAHeight.Store(local) + if !s.waitOnCatchupError(ctx, err, local) { + return + } + continue + } + } +} + +// ErrCaughtUp is a sentinel used to signal that the catchup loop has reached DA head. +var ErrCaughtUp = errors.New("caught up with DA head") + +// waitOnCatchupError logs the error and backs off before retrying. +func (s *Subscriber) waitOnCatchupError(ctx context.Context, err error, daHeight uint64) bool { + if errors.Is(err, ErrCaughtUp) { + s.logger.Debug().Uint64("da_height", daHeight).Msg("DA catchup reached head, waiting for subscription signal") + return false + } + if ctx.Err() != nil { + return false + } + s.logger.Warn().Err(err).Uint64("da_height", daHeight).Msg("catchup error, backing off") + select { + case <-ctx.Done(): + return false + case <-time.After(s.backoff()): + return true + } +} + +func (s *Subscriber) backoff() time.Duration { + if s.daBlockTime > 0 { + return s.daBlockTime + } + return 2 * time.Second +} + +const subscriberWatchdogMultiplier = 3 + +func (s *Subscriber) watchdogTimeout() time.Duration { + if s.daBlockTime > 0 { + return s.daBlockTime * subscriberWatchdogMultiplier + } + return 30 * time.Second +} diff --git a/block/internal/da/tracing.go b/block/internal/da/tracing.go index 4d946a8b74..3da79feddb 100644 --- a/block/internal/da/tracing.go +++ b/block/internal/da/tracing.go @@ -145,6 +145,9 @@ func (t *tracedClient) GetForcedInclusionNamespace() []byte { func (t *tracedClient) HasForcedInclusionNamespace() bool { return t.inner.HasForcedInclusionNamespace() } +func (t *tracedClient) Subscribe(ctx context.Context, namespace []byte, includeTimestamp bool) (<-chan datypes.SubscriptionEvent, error) { + return t.inner.Subscribe(ctx, namespace, includeTimestamp) +} type submitError struct{ msg string } diff --git a/block/internal/da/tracing_test.go b/block/internal/da/tracing_test.go index de32532a31..fc1ae72f96 100644 --- a/block/internal/da/tracing_test.go +++ b/block/internal/da/tracing_test.go @@ -22,6 +22,14 @@ type mockFullClient struct { getFn func(ctx context.Context, ids []datypes.ID, namespace []byte) ([]datypes.Blob, error) getProofsFn func(ctx context.Context, ids []datypes.ID, namespace []byte) ([]datypes.Proof, error) validateFn func(ctx context.Context, ids []datypes.ID, proofs []datypes.Proof, namespace []byte) ([]bool, error) + subscribeFn func(ctx context.Context, namespace []byte, ts bool) (<-chan datypes.SubscriptionEvent, error) +} + +func (m *mockFullClient) Subscribe(ctx context.Context, namespace []byte, ts bool) (<-chan datypes.SubscriptionEvent, error) { + if m.subscribeFn == nil { + panic("not expected to be called") + } + return m.subscribeFn(ctx, namespace, ts) } func (m *mockFullClient) Submit(ctx context.Context, data [][]byte, gasPrice float64, namespace []byte, options []byte) datypes.ResultSubmit { diff --git a/block/internal/syncing/da_follower.go b/block/internal/syncing/da_follower.go new file mode 100644 index 0000000000..a31ffc43ca --- /dev/null +++ b/block/internal/syncing/da_follower.go @@ -0,0 +1,204 @@ +package syncing + +import ( + "context" + "errors" + "slices" + "sync" + "time" + + "github.com/rs/zerolog" + + "github.com/evstack/ev-node/block/internal/common" + "github.com/evstack/ev-node/block/internal/da" + datypes "github.com/evstack/ev-node/pkg/da/types" +) + +// DAFollower follows DA blob events and drives sequential catchup +// using a shared da.Subscriber for the subscription plumbing. +type DAFollower interface { + Start(ctx context.Context) error + Stop() + HasReachedHead() bool + // QueuePriorityHeight queues a DA height for priority retrieval (from P2P hints). + QueuePriorityHeight(daHeight uint64) +} + +// daFollower is the concrete implementation of DAFollower. +type daFollower struct { + subscriber *da.Subscriber + retriever DARetriever + eventSink common.EventSink + logger zerolog.Logger + + // Priority queue for P2P hint heights (absorbed from DARetriever refactoring #2). + priorityMu sync.Mutex + priorityHeights []uint64 +} + +// DAFollowerConfig holds configuration for creating a DAFollower. +type DAFollowerConfig struct { + Client da.Client + Retriever DARetriever + Logger zerolog.Logger + EventSink common.EventSink + Namespace []byte + DataNamespace []byte // may be nil or equal to Namespace + StartDAHeight uint64 + DABlockTime time.Duration +} + +// NewDAFollower creates a new daFollower. +func NewDAFollower(cfg DAFollowerConfig) DAFollower { + dataNs := cfg.DataNamespace + if len(dataNs) == 0 { + dataNs = cfg.Namespace + } + + f := &daFollower{ + retriever: cfg.Retriever, + eventSink: cfg.EventSink, + logger: cfg.Logger.With().Str("component", "da_follower").Logger(), + priorityHeights: make([]uint64, 0), + } + + f.subscriber = da.NewSubscriber(da.SubscriberConfig{ + Client: cfg.Client, + Logger: cfg.Logger, + Namespaces: [][]byte{cfg.Namespace, dataNs}, + DABlockTime: cfg.DABlockTime, + Handler: f, + }) + f.subscriber.SetStartHeight(cfg.StartDAHeight) + + return f +} + +// Start begins the follow and catchup goroutines. +func (f *daFollower) Start(ctx context.Context) error { + return f.subscriber.Start(ctx) +} + +// Stop gracefully stops the background goroutines. +func (f *daFollower) Stop() { + f.subscriber.Stop() +} + +// HasReachedHead returns whether the follower has caught up to DA head. +func (f *daFollower) HasReachedHead() bool { + return f.subscriber.HasReachedHead() +} + +// --------------------------------------------------------------------------- +// SubscriberHandler implementation +// --------------------------------------------------------------------------- + +// HandleEvent processes a subscription event. When the follower is +// caught up (ev.Height == localDAHeight) and blobs are available, it processes +// them inline — avoiding a DA re-fetch round trip. Otherwise, it just lets +// the catchup loop handle retrieval. +// +// Uses CAS on localDAHeight to claim exclusive access to processBlobs, +// preventing concurrent map access with catchupLoop. +func (f *daFollower) HandleEvent(ctx context.Context, ev datypes.SubscriptionEvent) { + // Fast path: try to claim this height for inline processing. + // CAS(N, N+1) ensures only one goroutine (followLoop or catchupLoop) + // can enter processBlobs for height N. + if len(ev.Blobs) > 0 && f.subscriber.CompareAndSwapLocalHeight(ev.Height, ev.Height+1) { + events := f.retriever.ProcessBlobs(ctx, ev.Blobs, ev.Height) + for _, event := range events { + if err := f.eventSink.PipeEvent(ctx, event); err != nil { + // Roll back so catchupLoop can retry this height. + f.subscriber.SetLocalHeight(ev.Height) + f.logger.Warn().Err(err).Uint64("da_height", ev.Height). + Msg("failed to pipe inline event, catchup will retry") + return + } + } + if len(events) != 0 { + f.subscriber.SetHeadReached() + f.logger.Debug().Uint64("da_height", ev.Height).Int("events", len(events)). + Msg("processed subscription blobs inline (fast path)") + } else { + // No complete events (split namespace, waiting for other half). + f.subscriber.SetLocalHeight(ev.Height) + } + return + } + + // Slow path: behind, no blobs, or catchupLoop claimed this height. +} + +// HandleCatchup retrieves events at a single DA height and pipes them +// to the event sink. Checks priority heights first. +func (f *daFollower) HandleCatchup(ctx context.Context, daHeight uint64) error { + // Check for priority heights from P2P hints first. + if priorityHeight := f.popPriorityHeight(); priorityHeight > 0 { + if priorityHeight >= daHeight { + f.logger.Debug(). + Uint64("da_height", priorityHeight). + Msg("fetching priority DA height from P2P hint") + if err := f.fetchAndPipeHeight(ctx, priorityHeight); err != nil { + return err + } + } + // Re-queue the current height by rolling back (the subscriber already advanced). + f.subscriber.SetLocalHeight(daHeight) + return nil + } + + return f.fetchAndPipeHeight(ctx, daHeight) +} + +// fetchAndPipeHeight retrieves events at a single DA height and pipes them. +func (f *daFollower) fetchAndPipeHeight(ctx context.Context, daHeight uint64) error { + events, err := f.retriever.RetrieveFromDA(ctx, daHeight) + if err != nil { + switch { + case errors.Is(err, datypes.ErrBlobNotFound): + return nil + case errors.Is(err, datypes.ErrHeightFromFuture): + f.subscriber.SetHeadReached() + return err + default: + return err + } + } + + for _, event := range events { + if err := f.eventSink.PipeEvent(ctx, event); err != nil { + return err + } + } + + return nil +} + +// --------------------------------------------------------------------------- +// Priority queue (absorbed from DARetriever — refactoring #2) +// --------------------------------------------------------------------------- + +// QueuePriorityHeight queues a DA height for priority retrieval. +func (f *daFollower) QueuePriorityHeight(daHeight uint64) { + f.priorityMu.Lock() + defer f.priorityMu.Unlock() + + idx, found := slices.BinarySearch(f.priorityHeights, daHeight) + if found { + return + } + f.priorityHeights = slices.Insert(f.priorityHeights, idx, daHeight) +} + +// popPriorityHeight returns the next priority height to fetch, or 0 if none. +func (f *daFollower) popPriorityHeight() uint64 { + f.priorityMu.Lock() + defer f.priorityMu.Unlock() + + if len(f.priorityHeights) == 0 { + return 0 + } + height := f.priorityHeights[0] + f.priorityHeights = f.priorityHeights[1:] + return height +} diff --git a/block/internal/syncing/da_retriever.go b/block/internal/syncing/da_retriever.go index a6c9d43c7c..691b2d1bb6 100644 --- a/block/internal/syncing/da_retriever.go +++ b/block/internal/syncing/da_retriever.go @@ -5,8 +5,6 @@ import ( "context" "errors" "fmt" - "slices" - "sync" "github.com/rs/zerolog" "google.golang.org/protobuf/proto" @@ -24,11 +22,9 @@ import ( type DARetriever interface { // RetrieveFromDA retrieves blocks from the specified DA height and returns height events RetrieveFromDA(ctx context.Context, daHeight uint64) ([]common.DAHeightEvent, error) - // QueuePriorityHeight queues a DA height for priority retrieval (from P2P hints). - // These heights take precedence over sequential fetching. - QueuePriorityHeight(daHeight uint64) - // PopPriorityHeight returns the next priority height to fetch, or 0 if none. - PopPriorityHeight() uint64 + // ProcessBlobs parses raw blob bytes at a given DA height into height events. + // Used by the DAFollower to process subscription blobs inline without re-fetching. + ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent } // daRetriever handles DA retrieval operations for syncing @@ -46,12 +42,6 @@ type daRetriever struct { // strictMode indicates if the node has seen a valid DAHeaderEnvelope // and should now reject all legacy/unsigned headers. strictMode bool - - // priorityMu protects priorityHeights from concurrent access - priorityMu sync.Mutex - // priorityHeights holds DA heights from P2P hints that should be fetched - // before continuing sequential retrieval. Sorted in ascending order. - priorityHeights []uint64 } // NewDARetriever creates a new DA retriever @@ -62,45 +52,16 @@ func NewDARetriever( logger zerolog.Logger, ) *daRetriever { return &daRetriever{ - client: client, - cache: cache, - genesis: genesis, - logger: logger.With().Str("component", "da_retriever").Logger(), - pendingHeaders: make(map[uint64]*types.SignedHeader), - pendingData: make(map[uint64]*types.Data), - strictMode: false, - priorityHeights: make([]uint64, 0), + client: client, + cache: cache, + genesis: genesis, + logger: logger.With().Str("component", "da_retriever").Logger(), + pendingHeaders: make(map[uint64]*types.SignedHeader), + pendingData: make(map[uint64]*types.Data), + strictMode: false, } } -// QueuePriorityHeight queues a DA height for priority retrieval. -// Heights from P2P hints take precedence over sequential fetching. -func (r *daRetriever) QueuePriorityHeight(daHeight uint64) { - r.priorityMu.Lock() - defer r.priorityMu.Unlock() - - idx, found := slices.BinarySearch(r.priorityHeights, daHeight) - if found { - return // Already queued - } - r.priorityHeights = slices.Insert(r.priorityHeights, idx, daHeight) -} - -// PopPriorityHeight returns the next priority height to fetch, or 0 if none. -func (r *daRetriever) PopPriorityHeight() uint64 { - r.priorityMu.Lock() - defer r.priorityMu.Unlock() - - if len(r.priorityHeights) == 0 { - return 0 - } - - height := r.priorityHeights[0] - r.priorityHeights = r.priorityHeights[1:] - - return height -} - // RetrieveFromDA retrieves blocks from the specified DA height and returns height events func (r *daRetriever) RetrieveFromDA(ctx context.Context, daHeight uint64) ([]common.DAHeightEvent, error) { r.logger.Debug().Uint64("da_height", daHeight).Msg("retrieving from DA") @@ -191,6 +152,15 @@ func (r *daRetriever) validateBlobResponse(res datypes.ResultRetrieve, daHeight } } +// ProcessBlobs processes raw blob bytes to extract headers and data and returns height events. +// This is the public interface used by the DAFollower for inline subscription processing. +// +// NOT thread-safe: the caller (DAFollower) must ensure exclusive access via CAS +// on localDAHeight before calling this method. +func (r *daRetriever) ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { + return r.processBlobs(ctx, blobs, daHeight) +} + // processBlobs processes retrieved blobs to extract headers and data and returns height events func (r *daRetriever) processBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { // Decode all blobs diff --git a/block/internal/syncing/da_retriever_mock.go b/block/internal/syncing/da_retriever_mock.go index 2e191c8851..10e08bbd90 100644 --- a/block/internal/syncing/da_retriever_mock.go +++ b/block/internal/syncing/da_retriever_mock.go @@ -38,87 +38,68 @@ func (_m *MockDARetriever) EXPECT() *MockDARetriever_Expecter { return &MockDARetriever_Expecter{mock: &_m.Mock} } -// PopPriorityHeight provides a mock function for the type MockDARetriever -func (_mock *MockDARetriever) PopPriorityHeight() uint64 { - ret := _mock.Called() +// ProcessBlobs provides a mock function for the type MockDARetriever +func (_mock *MockDARetriever) ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { + ret := _mock.Called(ctx, blobs, daHeight) if len(ret) == 0 { - panic("no return value specified for PopPriorityHeight") + panic("no return value specified for ProcessBlobs") } - var r0 uint64 - if returnFunc, ok := ret.Get(0).(func() uint64); ok { - r0 = returnFunc() + var r0 []common.DAHeightEvent + if returnFunc, ok := ret.Get(0).(func(context.Context, [][]byte, uint64) []common.DAHeightEvent); ok { + r0 = returnFunc(ctx, blobs, daHeight) } else { - r0 = ret.Get(0).(uint64) + if ret.Get(0) != nil { + r0 = ret.Get(0).([]common.DAHeightEvent) + } } return r0 } -// MockDARetriever_PopPriorityHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PopPriorityHeight' -type MockDARetriever_PopPriorityHeight_Call struct { +// MockDARetriever_ProcessBlobs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ProcessBlobs' +type MockDARetriever_ProcessBlobs_Call struct { *mock.Call } -// PopPriorityHeight is a helper method to define mock.On call -func (_e *MockDARetriever_Expecter) PopPriorityHeight() *MockDARetriever_PopPriorityHeight_Call { - return &MockDARetriever_PopPriorityHeight_Call{Call: _e.mock.On("PopPriorityHeight")} -} - -func (_c *MockDARetriever_PopPriorityHeight_Call) Run(run func()) *MockDARetriever_PopPriorityHeight_Call { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *MockDARetriever_PopPriorityHeight_Call) Return(v uint64) *MockDARetriever_PopPriorityHeight_Call { - _c.Call.Return(v) - return _c -} - -func (_c *MockDARetriever_PopPriorityHeight_Call) RunAndReturn(run func() uint64) *MockDARetriever_PopPriorityHeight_Call { - _c.Call.Return(run) - return _c -} - -// QueuePriorityHeight provides a mock function for the type MockDARetriever -func (_mock *MockDARetriever) QueuePriorityHeight(daHeight uint64) { - _mock.Called(daHeight) - return -} - -// MockDARetriever_QueuePriorityHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'QueuePriorityHeight' -type MockDARetriever_QueuePriorityHeight_Call struct { - *mock.Call -} - -// QueuePriorityHeight is a helper method to define mock.On call +// ProcessBlobs is a helper method to define mock.On call +// - ctx context.Context +// - blobs [][]byte // - daHeight uint64 -func (_e *MockDARetriever_Expecter) QueuePriorityHeight(daHeight interface{}) *MockDARetriever_QueuePriorityHeight_Call { - return &MockDARetriever_QueuePriorityHeight_Call{Call: _e.mock.On("QueuePriorityHeight", daHeight)} +func (_e *MockDARetriever_Expecter) ProcessBlobs(ctx interface{}, blobs interface{}, daHeight interface{}) *MockDARetriever_ProcessBlobs_Call { + return &MockDARetriever_ProcessBlobs_Call{Call: _e.mock.On("ProcessBlobs", ctx, blobs, daHeight)} } -func (_c *MockDARetriever_QueuePriorityHeight_Call) Run(run func(daHeight uint64)) *MockDARetriever_QueuePriorityHeight_Call { +func (_c *MockDARetriever_ProcessBlobs_Call) Run(run func(ctx context.Context, blobs [][]byte, daHeight uint64)) *MockDARetriever_ProcessBlobs_Call { _c.Call.Run(func(args mock.Arguments) { - var arg0 uint64 + var arg0 context.Context if args[0] != nil { - arg0 = args[0].(uint64) + arg0 = args[0].(context.Context) + } + var arg1 [][]byte + if args[1] != nil { + arg1 = args[1].([][]byte) + } + var arg2 uint64 + if args[2] != nil { + arg2 = args[2].(uint64) } run( arg0, + arg1, + arg2, ) }) return _c } -func (_c *MockDARetriever_QueuePriorityHeight_Call) Return() *MockDARetriever_QueuePriorityHeight_Call { - _c.Call.Return() +func (_c *MockDARetriever_ProcessBlobs_Call) Return(dAHeightEvents []common.DAHeightEvent) *MockDARetriever_ProcessBlobs_Call { + _c.Call.Return(dAHeightEvents) return _c } -func (_c *MockDARetriever_QueuePriorityHeight_Call) RunAndReturn(run func(daHeight uint64)) *MockDARetriever_QueuePriorityHeight_Call { - _c.Run(run) +func (_c *MockDARetriever_ProcessBlobs_Call) RunAndReturn(run func(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent) *MockDARetriever_ProcessBlobs_Call { + _c.Call.Return(run) return _c } diff --git a/block/internal/syncing/da_retriever_tracing.go b/block/internal/syncing/da_retriever_tracing.go index 2bc7a4094d..d41418a1d8 100644 --- a/block/internal/syncing/da_retriever_tracing.go +++ b/block/internal/syncing/da_retriever_tracing.go @@ -56,10 +56,6 @@ func (t *tracedDARetriever) RetrieveFromDA(ctx context.Context, daHeight uint64) return events, nil } -func (t *tracedDARetriever) QueuePriorityHeight(daHeight uint64) { - t.inner.QueuePriorityHeight(daHeight) -} - -func (t *tracedDARetriever) PopPriorityHeight() uint64 { - return t.inner.PopPriorityHeight() +func (t *tracedDARetriever) ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { + return t.inner.ProcessBlobs(ctx, blobs, daHeight) } diff --git a/block/internal/syncing/da_retriever_tracing_test.go b/block/internal/syncing/da_retriever_tracing_test.go index 99ce1eb639..83bd864bfa 100644 --- a/block/internal/syncing/da_retriever_tracing_test.go +++ b/block/internal/syncing/da_retriever_tracing_test.go @@ -27,9 +27,9 @@ func (m *mockDARetriever) RetrieveFromDA(ctx context.Context, daHeight uint64) ( return nil, nil } -func (m *mockDARetriever) QueuePriorityHeight(daHeight uint64) {} - -func (m *mockDARetriever) PopPriorityHeight() uint64 { return 0 } +func (m *mockDARetriever) ProcessBlobs(_ context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { + return nil +} func setupDARetrieverTrace(t *testing.T, inner DARetriever) (DARetriever, *tracetest.SpanRecorder) { t.Helper() diff --git a/block/internal/syncing/raft_retriever.go b/block/internal/syncing/raft_retriever.go index 0b976acab9..cfa55662bd 100644 --- a/block/internal/syncing/raft_retriever.go +++ b/block/internal/syncing/raft_retriever.go @@ -14,20 +14,6 @@ import ( "github.com/evstack/ev-node/types" ) -// eventProcessor handles DA height events. Used for syncing. -type eventProcessor interface { - // handle processes a single DA height event. - handle(ctx context.Context, event common.DAHeightEvent) error -} - -// eventProcessorFn adapts a function to an eventProcessor. -type eventProcessorFn func(ctx context.Context, event common.DAHeightEvent) error - -// handle calls the wrapped function. -func (e eventProcessorFn) handle(ctx context.Context, event common.DAHeightEvent) error { - return e(ctx, event) -} - // raftStatePreProcessor is called before processing a raft block state type raftStatePreProcessor func(ctx context.Context, state *raft.RaftBlockState) error @@ -37,7 +23,7 @@ type raftRetriever struct { wg sync.WaitGroup logger zerolog.Logger genesis genesis.Genesis - eventProcessor eventProcessor + eventSink common.EventSink raftBlockPreProcessor raftStatePreProcessor mtx sync.Mutex @@ -49,14 +35,14 @@ func newRaftRetriever( raftNode common.RaftNode, genesis genesis.Genesis, logger zerolog.Logger, - eventProcessor eventProcessor, + eventSink common.EventSink, raftBlockPreProcessor raftStatePreProcessor, ) *raftRetriever { return &raftRetriever{ raftNode: raftNode, genesis: genesis, logger: logger, - eventProcessor: eventProcessor, + eventSink: eventSink, raftBlockPreProcessor: raftBlockPreProcessor, } } @@ -153,7 +139,7 @@ func (r *raftRetriever) consumeRaftBlock(ctx context.Context, state *raft.RaftBl Data: &data, DaHeight: 0, // raft events don't have DA height context, yet as DA submission is asynchronous } - return r.eventProcessor.handle(ctx, event) + return r.eventSink.PipeEvent(ctx, event) } // Height returns the current height of the raft node's state. diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index 27c533f89a..91d231527e 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -21,7 +21,6 @@ import ( "github.com/evstack/ev-node/block/internal/da" coreexecutor "github.com/evstack/ev-node/core/execution" "github.com/evstack/ev-node/pkg/config" - datypes "github.com/evstack/ev-node/pkg/da/types" "github.com/evstack/ev-node/pkg/genesis" "github.com/evstack/ev-node/pkg/raft" "github.com/evstack/ev-node/pkg/store" @@ -80,6 +79,8 @@ type Syncer struct { p2pHandler p2pHandler raftRetriever *raftRetriever + daFollower DAFollower + // Forced inclusion tracking forcedInclusionMu sync.RWMutex seenBlockTxs map[string]struct{} // SHA-256 hex of every tx seen in a DA-sourced block @@ -96,9 +97,6 @@ type Syncer struct { // P2P wait coordination p2pWaitState atomic.Value // stores p2pWaitState - // DA head-reached signal for recovery mode (stays true once DA head is seen) - daHeadReached atomic.Bool - // blockSyncer is the interface used for block sync operations. // defaults to self, but can be wrapped with tracing. blockSyncer BlockSyncer @@ -145,7 +143,7 @@ func NewSyncer( } s.blockSyncer = s if raftNode != nil && !reflect.ValueOf(raftNode).IsNil() { - s.raftRetriever = newRaftRetriever(raftNode, genesis, logger, eventProcessorFn(s.pipeEvent), + s.raftRetriever = newRaftRetriever(raftNode, genesis, logger, s, func(ctx context.Context, state *raft.RaftBlockState) error { s.logger.Debug().Uint64("header_height", state.LastSubmittedDaHeaderHeight).Uint64("data_height", state.LastSubmittedDaDataHeight).Msg("received raft block state") cache.SetLastSubmittedHeaderHeight(ctx, state.LastSubmittedDaHeaderHeight) @@ -163,11 +161,17 @@ func (s *Syncer) SetBlockSyncer(bs BlockSyncer) { } // Start begins the syncing component -func (s *Syncer) Start(ctx context.Context) error { +func (s *Syncer) Start(ctx context.Context) (err error) { ctx, cancel := context.WithCancel(ctx) s.ctx, s.cancel = ctx, cancel - if err := s.initializeState(); err != nil { + defer func() { //nolint: contextcheck // use new context as parent can be cancelled already + if err != nil { + _ = s.Stop(context.Background()) + } + }() + + if err = s.initializeState(); err != nil { return fmt.Errorf("failed to initialize syncer state: %w", err) } @@ -177,16 +181,18 @@ func (s *Syncer) Start(ctx context.Context) error { s.daRetriever = WithTracingDARetriever(s.daRetriever) } - s.fiRetriever = da.NewForcedInclusionRetriever(s.daClient, s.logger, s.config, s.genesis.DAStartHeight, s.genesis.DAEpochForcedInclusion) + s.fiRetriever = da.NewForcedInclusionRetriever(ctx, s.daClient, s.logger, s.config.DA.BlockTime.Duration, s.config.Instrumentation.IsTracingEnabled(), s.genesis.DAStartHeight, s.genesis.DAEpochForcedInclusion) s.p2pHandler = NewP2PHandler(s.headerStore, s.dataStore, s.cache, s.genesis, s.logger) - if currentHeight, err := s.store.Height(ctx); err != nil { - s.logger.Error().Err(err).Msg("failed to set initial processed height for p2p handler") + + currentHeight, initErr := s.store.Height(ctx) + if initErr != nil { + s.logger.Error().Err(initErr).Msg("failed to set initial processed height for p2p handler") } else { s.p2pHandler.SetProcessedHeight(currentHeight) } if s.raftRetriever != nil { - if err := s.raftRetriever.Start(ctx); err != nil { + if err = s.raftRetriever.Start(ctx); err != nil { return fmt.Errorf("start raft retriever: %w", err) } } @@ -198,7 +204,21 @@ func (s *Syncer) Start(ctx context.Context) error { // Start main processing loop s.wg.Go(func() { s.processLoop(ctx) }) - // Start dedicated workers for DA, and pending processing + // Start the DA follower (subscribe + catchup) and other workers + s.daFollower = NewDAFollower(DAFollowerConfig{ + Client: s.daClient, + Retriever: s.daRetriever, + Logger: s.logger, + EventSink: s, + Namespace: s.daClient.GetHeaderNamespace(), + DataNamespace: s.daClient.GetDataNamespace(), + StartDAHeight: s.daRetrieverHeight.Load(), + DABlockTime: s.config.DA.BlockTime.Duration, + }) + if err = s.daFollower.Start(ctx); err != nil { + return fmt.Errorf("failed to start DA follower: %w", err) + } + s.startSyncWorkers(ctx) s.logger.Info().Msg("syncer started") @@ -206,19 +226,25 @@ func (s *Syncer) Start(ctx context.Context) error { } // Stop shuts down the syncing component -func (s *Syncer) Stop() error { +func (s *Syncer) Stop(ctx context.Context) error { if s.cancel == nil { return nil } s.cancel() s.cancelP2PWait(0) + + // Stop the DA follower first (it owns its own goroutines). + if s.daFollower != nil { + s.daFollower.Stop() + } + s.wg.Wait() // Skip draining if we're shutting down due to a critical error (e.g. execution // client unavailable). if !s.hasCriticalError.Load() { - drainCtx, drainCancel := context.WithTimeout(context.Background(), 5*time.Second) + drainCtx, drainCancel := context.WithTimeout(ctx, 5*time.Second) defer drainCancel() drained := 0 @@ -360,111 +386,19 @@ func (s *Syncer) processLoop(ctx context.Context) { } func (s *Syncer) startSyncWorkers(ctx context.Context) { - s.wg.Add(3) - go s.daWorkerLoop(ctx) + // DA follower is already started in Start(). + s.wg.Add(2) go s.pendingWorkerLoop(ctx) go s.p2pWorkerLoop(ctx) } -func (s *Syncer) daWorkerLoop(ctx context.Context) { - defer s.wg.Done() - - s.logger.Info().Msg("starting DA worker") - defer s.logger.Info().Msg("DA worker stopped") - - for { - err := s.fetchDAUntilCaughtUp(ctx) - - var backoff time.Duration - if err == nil { - // No error, means we are caught up. - s.daHeadReached.Store(true) - backoff = s.config.DA.BlockTime.Duration - } else { - // Error, back off for a shorter duration. - backoff = s.config.DA.BlockTime.Duration - if backoff <= 0 { - backoff = 2 * time.Second - } - } - - select { - case <-ctx.Done(): - return - case <-time.After(backoff): - } - } -} - -// HasReachedDAHead returns true once the DA worker has reached the DA head. +// HasReachedDAHead returns true once the DA follower has caught up to the DA head. // Once set, it stays true. func (s *Syncer) HasReachedDAHead() bool { - return s.daHeadReached.Load() -} - -func (s *Syncer) fetchDAUntilCaughtUp(ctx context.Context) error { - for { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - // Check for priority heights from P2P hints first - var daHeight uint64 - if priorityHeight := s.daRetriever.PopPriorityHeight(); priorityHeight > 0 { - // Skip if we've already fetched past this height - currentHeight := s.daRetrieverHeight.Load() - if priorityHeight < currentHeight { - continue - } - daHeight = priorityHeight - s.logger.Debug().Uint64("da_height", daHeight).Msg("fetching priority DA height from P2P hint") - } else { - daHeight = max(s.daRetrieverHeight.Load(), s.cache.DaHeight()) - } - - events, err := s.daRetriever.RetrieveFromDA(ctx, daHeight) - if err != nil { - switch { - case errors.Is(err, datypes.ErrBlobNotFound): - s.daRetrieverHeight.Store(daHeight + 1) - continue // Fetch next height immediately - case errors.Is(err, datypes.ErrHeightFromFuture): - s.logger.Debug().Err(err).Uint64("da_height", daHeight).Msg("DA is ahead of local target; backing off future height requests") - return nil // Caught up - default: - s.logger.Error().Err(err).Uint64("da_height", daHeight).Msg("failed to retrieve from DA; backing off DA requests") - return err // Other errors - } - } - - if len(events) == 0 { - // This can happen if RetrieveFromDA returns no events and no error. - s.logger.Debug().Uint64("da_height", daHeight).Msg("no events returned from DA, but no error either.") - } - - // Process DA events - for _, event := range events { - if err := s.pipeEvent(ctx, event); err != nil { - return err - } - } - - // Update DA retrieval height on successful retrieval - // For priority fetches, only update if the priority height is ahead of current - // For sequential fetches, always increment - newHeight := daHeight + 1 - for { - current := s.daRetrieverHeight.Load() - if newHeight <= current { - break // Already at or past this height - } - if s.daRetrieverHeight.CompareAndSwap(current, newHeight) { - break - } - } + if s.daFollower != nil { + return s.daFollower.HasReachedHead() } + return false } // PendingCount returns the number of unprocessed height events in the pipeline. @@ -558,7 +492,7 @@ func (s *Syncer) waitForGenesis() bool { return true } -func (s *Syncer) pipeEvent(ctx context.Context, event common.DAHeightEvent) error { +func (s *Syncer) PipeEvent(ctx context.Context, event common.DAHeightEvent) error { select { case s.heightInCh <- event: return nil @@ -674,7 +608,7 @@ func (s *Syncer) processHeightEvent(ctx context.Context, event *common.DAHeightE Msg("P2P event with DA height hint, queuing priority DA retrieval") // Queue priority DA retrieval - will be processed in fetchDAUntilCaughtUp - s.daRetriever.QueuePriorityHeight(daHeightHint) + s.daFollower.QueuePriorityHeight(daHeightHint) } } } diff --git a/block/internal/syncing/syncer_backoff_test.go b/block/internal/syncing/syncer_backoff_test.go index c2bed9385d..113cf173d9 100644 --- a/block/internal/syncing/syncer_backoff_test.go +++ b/block/internal/syncing/syncer_backoff_test.go @@ -7,27 +7,20 @@ import ( "testing/synctest" "time" - "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "github.com/evstack/ev-node/block/internal/cache" "github.com/evstack/ev-node/block/internal/common" - "github.com/evstack/ev-node/core/execution" - "github.com/evstack/ev-node/pkg/config" datypes "github.com/evstack/ev-node/pkg/da/types" "github.com/evstack/ev-node/pkg/genesis" - "github.com/evstack/ev-node/pkg/store" - extmocks "github.com/evstack/ev-node/test/mocks/external" "github.com/evstack/ev-node/types" ) -// TestSyncer_BackoffOnDAError verifies that the syncer implements proper backoff -// behavior when encountering different types of DA layer errors. -func TestSyncer_BackoffOnDAError(t *testing.T) { +// TestDAFollower_BackoffOnCatchupError verifies that the DAFollower implements +// proper backoff behavior when encountering different types of DA layer errors. +func TestDAFollower_BackoffOnCatchupError(t *testing.T) { tests := map[string]struct { daBlockTime time.Duration error error @@ -66,27 +59,27 @@ func TestSyncer_BackoffOnDAError(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() - // Setup syncer - syncer := setupTestSyncer(t, tc.daBlockTime) - syncer.ctx = ctx - - // Setup mocks daRetriever := NewMockDARetriever(t) - p2pHandler := newMockp2pHandler(t) - p2pHandler.On("ProcessHeight", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() - syncer.daRetriever = daRetriever - syncer.p2pHandler = p2pHandler - p2pHandler.On("SetProcessedHeight", mock.Anything).Return().Maybe() - // Mock PopPriorityHeight to always return 0 (no priority heights) - daRetriever.On("PopPriorityHeight").Return(uint64(0)).Maybe() + pipeEvent := func(_ context.Context, _ common.DAHeightEvent) error { return nil } + + follower := NewDAFollower(DAFollowerConfig{ + Retriever: daRetriever, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(pipeEvent), + Namespace: []byte("ns"), + StartDAHeight: 100, + DABlockTime: tc.daBlockTime, + }).(*daFollower) - // Create mock stores for P2P - mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) - mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() + // Set up the subscriber for direct testing. + sub := follower.subscriber + sub.SetStartHeight(100) + ctx, subCancel := context.WithCancel(ctx) + defer subCancel() - mockDataStore := extmocks.NewMockStore[*types.Data](t) - mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() + // Set highest to trigger catchup. + sub.UpdateHighestForTest(102) var callTimes []time.Time callCount := 0 @@ -100,32 +93,26 @@ func TestSyncer_BackoffOnDAError(t *testing.T) { Return(nil, tc.error).Once() if tc.expectsBackoff { - // Second call should be delayed due to backoff daRetriever.On("RetrieveFromDA", mock.Anything, uint64(100)). Run(func(args mock.Arguments) { callTimes = append(callTimes, time.Now()) callCount++ - // Cancel to end test - cancel() + subCancel() }). Return(nil, datypes.ErrBlobNotFound).Once() } else { - // For ErrBlobNotFound, DA height should increment daRetriever.On("RetrieveFromDA", mock.Anything, uint64(101)). Run(func(args mock.Arguments) { callTimes = append(callTimes, time.Now()) callCount++ - cancel() + subCancel() }). Return(nil, datypes.ErrBlobNotFound).Once() } - // Run sync loop - syncer.startSyncWorkers(ctx) + go sub.RunCatchupForTest(ctx) <-ctx.Done() - syncer.wg.Wait() - // Verify behavior if tc.expectsBackoff { require.Len(t, callTimes, 2, "should make exactly 2 calls with backoff") @@ -151,35 +138,33 @@ func TestSyncer_BackoffOnDAError(t *testing.T) { } } -// TestSyncer_BackoffResetOnSuccess verifies that backoff is properly reset -// after a successful DA retrieval, allowing the syncer to continue at normal speed. -func TestSyncer_BackoffResetOnSuccess(t *testing.T) { +// TestDAFollower_BackoffResetOnSuccess verifies that backoff is properly reset +// after a successful DA retrieval. +func TestDAFollower_BackoffResetOnSuccess(t *testing.T) { synctest.Test(t, func(t *testing.T) { - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) defer cancel() - syncer := setupTestSyncer(t, 1*time.Second) - syncer.ctx = ctx - addr, pub, signer := buildSyncTestSigner(t) - gen := syncer.genesis + gen := backoffTestGenesis(addr) daRetriever := NewMockDARetriever(t) - p2pHandler := newMockp2pHandler(t) - p2pHandler.On("ProcessHeight", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() - syncer.daRetriever = daRetriever - syncer.p2pHandler = p2pHandler - p2pHandler.On("SetProcessedHeight", mock.Anything).Return().Maybe() - // Mock PopPriorityHeight to always return 0 (no priority heights) - daRetriever.On("PopPriorityHeight").Return(uint64(0)).Maybe() + pipeEvent := func(_ context.Context, _ common.DAHeightEvent) error { return nil } - // Create mock stores for P2P - mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) - mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() + follower := NewDAFollower(DAFollowerConfig{ + Retriever: daRetriever, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(pipeEvent), + Namespace: []byte("ns"), + StartDAHeight: 100, + DABlockTime: 1 * time.Second, + }).(*daFollower) - mockDataStore := extmocks.NewMockStore[*types.Data](t) - mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() + sub := follower.subscriber + ctx, subCancel := context.WithCancel(ctx) + defer subCancel() + sub.UpdateHighestForTest(105) var callTimes []time.Time @@ -190,7 +175,7 @@ func TestSyncer_BackoffResetOnSuccess(t *testing.T) { }). Return(nil, errors.New("temporary failure")).Once() - // Second call - success (should reset backoff and increment DA height) + // Second call - success _, header := makeSignedHeaderBytes(t, gen.ChainID, 1, addr, pub, signer, nil, nil, nil) data := &types.Data{ Metadata: &types.Metadata{ @@ -211,145 +196,178 @@ func TestSyncer_BackoffResetOnSuccess(t *testing.T) { }). Return([]common.DAHeightEvent{event}, nil).Once() - // Third call - should happen immediately after success (DA height incremented to 101) + // Third call - should happen immediately after success (DA height incremented) daRetriever.On("RetrieveFromDA", mock.Anything, uint64(101)). Run(func(args mock.Arguments) { callTimes = append(callTimes, time.Now()) - cancel() + subCancel() }). Return(nil, datypes.ErrBlobNotFound).Once() - // Start process loop to handle events - go syncer.processLoop(ctx) - - // Run workers - syncer.startSyncWorkers(ctx) + go sub.RunCatchupForTest(ctx) <-ctx.Done() - syncer.wg.Wait() require.Len(t, callTimes, 3, "should make exactly 3 calls") - // Verify backoff between first and second call delay1to2 := callTimes[1].Sub(callTimes[0]) assert.GreaterOrEqual(t, delay1to2, 1*time.Second, "should have backed off between error and success (got %v)", delay1to2) - // Verify no backoff between second and third call (backoff reset) delay2to3 := callTimes[2].Sub(callTimes[1]) assert.Less(t, delay2to3, 100*time.Millisecond, "should continue immediately after success (got %v)", delay2to3) }) } -// TestSyncer_BackoffBehaviorIntegration tests the complete backoff flow: -// error -> backoff delay -> recovery -> normal operation. -func TestSyncer_BackoffBehaviorIntegration(t *testing.T) { - // Test simpler backoff behavior: error -> backoff -> success -> continue +// TestDAFollower_CatchupThenReachHead verifies the catchup flow: +// sequential fetch from local → highest → mark head reached. +func TestDAFollower_CatchupThenReachHead(t *testing.T) { synctest.Test(t, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() - syncer := setupTestSyncer(t, 500*time.Millisecond) - syncer.ctx = ctx - daRetriever := NewMockDARetriever(t) - p2pHandler := newMockp2pHandler(t) - p2pHandler.On("ProcessHeight", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() - syncer.daRetriever = daRetriever - syncer.p2pHandler = p2pHandler - // Mock PopPriorityHeight to always return 0 (no priority heights) - daRetriever.On("PopPriorityHeight").Return(uint64(0)).Maybe() + pipeEvent := func(_ context.Context, _ common.DAHeightEvent) error { return nil } - // Create mock stores for P2P - mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) - mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() + follower := NewDAFollower(DAFollowerConfig{ + Retriever: daRetriever, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(pipeEvent), + Namespace: []byte("ns"), + StartDAHeight: 3, + DABlockTime: 500 * time.Millisecond, + }).(*daFollower) - mockDataStore := extmocks.NewMockStore[*types.Data](t) - mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() + sub := follower.subscriber + sub.UpdateHighestForTest(5) - var callTimes []time.Time - p2pHandler.On("SetProcessedHeight", mock.Anything).Return().Maybe() + var fetchedHeights []uint64 - // First call - error (triggers backoff) - daRetriever.On("RetrieveFromDA", mock.Anything, uint64(100)). - Run(func(args mock.Arguments) { - callTimes = append(callTimes, time.Now()) - }). - Return(nil, errors.New("network error")).Once() + for h := uint64(3); h <= 5; h++ { + daRetriever.On("RetrieveFromDA", mock.Anything, h). + Run(func(args mock.Arguments) { + fetchedHeights = append(fetchedHeights, h) + }). + Return(nil, datypes.ErrBlobNotFound).Once() + } - // Second call - should be delayed due to backoff - daRetriever.On("RetrieveFromDA", mock.Anything, uint64(100)). - Run(func(args mock.Arguments) { - callTimes = append(callTimes, time.Now()) - }). - Return(nil, datypes.ErrBlobNotFound).Once() + sub.RunCatchupForTest(ctx) - // Third call - should continue without delay (DA height incremented) - daRetriever.On("RetrieveFromDA", mock.Anything, uint64(101)). - Run(func(args mock.Arguments) { - callTimes = append(callTimes, time.Now()) - cancel() - }). - Return(nil, datypes.ErrBlobNotFound).Once() + assert.True(t, follower.HasReachedHead(), "should have reached DA head") + // Heights 3, 4, 5 processed; local now at 6 which > highest (5) → caught up + assert.Equal(t, []uint64{3, 4, 5}, fetchedHeights, "should have fetched heights sequentially") + }) +} - go syncer.processLoop(ctx) - syncer.startSyncWorkers(ctx) - <-ctx.Done() - syncer.wg.Wait() +// TestDAFollower_InlineProcessing verifies the fast path: when the subscription +// delivers blobs at the current localDAHeight, HandleEvent processes +// them inline via ProcessBlobs (not RetrieveFromDA). +func TestDAFollower_InlineProcessing(t *testing.T) { + t.Run("processes_blobs_inline_when_caught_up", func(t *testing.T) { + daRetriever := NewMockDARetriever(t) - require.Len(t, callTimes, 3, "should make exactly 3 calls") + var pipedEvents []common.DAHeightEvent + pipeEvent := func(_ context.Context, ev common.DAHeightEvent) error { + pipedEvents = append(pipedEvents, ev) + return nil + } - // First to second call should be delayed (backoff) - delay1to2 := callTimes[1].Sub(callTimes[0]) - assert.GreaterOrEqual(t, delay1to2, 500*time.Millisecond, - "should have backoff delay between first and second call (got %v)", delay1to2) + follower := NewDAFollower(DAFollowerConfig{ + Retriever: daRetriever, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(pipeEvent), + Namespace: []byte("ns"), + StartDAHeight: 10, + DABlockTime: 500 * time.Millisecond, + }).(*daFollower) + + blobs := [][]byte{[]byte("header-blob"), []byte("data-blob")} + expectedEvents := []common.DAHeightEvent{ + {DaHeight: 10, Source: common.SourceDA}, + } - // Second to third call should be immediate (no backoff after ErrBlobNotFound) - delay2to3 := callTimes[2].Sub(callTimes[1]) - assert.Less(t, delay2to3, 100*time.Millisecond, - "should continue immediately after ErrBlobNotFound (got %v)", delay2to3) + // ProcessBlobs should be called (not RetrieveFromDA) + daRetriever.On("ProcessBlobs", mock.Anything, blobs, uint64(10)). + Return(expectedEvents).Once() + + // Simulate subscription event at the current localDAHeight + follower.HandleEvent(t.Context(), datypes.SubscriptionEvent{ + Height: 10, + Blobs: blobs, + }) + + // Verify: ProcessBlobs was called, events were piped, height advanced + require.Len(t, pipedEvents, 1, "should pipe 1 event from inline processing") + assert.Equal(t, uint64(10), pipedEvents[0].DaHeight) + assert.Equal(t, uint64(11), follower.subscriber.LocalDAHeight(), "localDAHeight should advance past processed height") + assert.True(t, follower.HasReachedHead(), "should mark head as reached after inline processing") }) -} -func setupTestSyncer(t *testing.T, daBlockTime time.Duration) *Syncer { - t.Helper() + t.Run("falls_through_to_catchup_when_behind", func(t *testing.T) { + daRetriever := NewMockDARetriever(t) - ds := dssync.MutexWrap(datastore.NewMapDatastore()) - st := store.New(ds) + pipeEvent := func(_ context.Context, _ common.DAHeightEvent) error { return nil } + + follower := NewDAFollower(DAFollowerConfig{ + Retriever: daRetriever, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(pipeEvent), + Namespace: []byte("ns"), + StartDAHeight: 10, + DABlockTime: 500 * time.Millisecond, + }).(*daFollower) + + // Subscription reports height 15 but local is at 10 — should NOT process inline + // In production, the subscriber calls updateHighest before HandleEvent. + follower.subscriber.UpdateHighestForTest(15) + follower.HandleEvent(t.Context(), datypes.SubscriptionEvent{ + Height: 15, + Blobs: [][]byte{[]byte("blob")}, + }) - cm, err := cache.NewManager(config.DefaultConfig(), st, zerolog.Nop()) - require.NoError(t, err) + // ProcessBlobs should NOT have been called + daRetriever.AssertNotCalled(t, "ProcessBlobs", mock.Anything, mock.Anything, mock.Anything) + assert.Equal(t, uint64(10), follower.subscriber.LocalDAHeight(), "localDAHeight should not change") + assert.Equal(t, uint64(15), follower.subscriber.HighestSeenDAHeight(), "highestSeen should be updated") + }) - addr, _, _ := buildSyncTestSigner(t) + t.Run("falls_through_when_no_blobs", func(t *testing.T) { + daRetriever := NewMockDARetriever(t) - cfg := config.DefaultConfig() - cfg.DA.BlockTime.Duration = daBlockTime + pipeEvent := func(_ context.Context, _ common.DAHeightEvent) error { return nil } + + follower := NewDAFollower(DAFollowerConfig{ + Retriever: daRetriever, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(pipeEvent), + Namespace: []byte("ns"), + StartDAHeight: 10, + DABlockTime: 500 * time.Millisecond, + }).(*daFollower) + + // Subscription at current height but no blobs — should fall through + // In production, the subscriber calls updateHighest before HandleEvent. + follower.subscriber.UpdateHighestForTest(10) + follower.HandleEvent(t.Context(), datypes.SubscriptionEvent{ + Height: 10, + Blobs: nil, + }) - gen := genesis.Genesis{ + // ProcessBlobs should NOT have been called + daRetriever.AssertNotCalled(t, "ProcessBlobs", mock.Anything, mock.Anything, mock.Anything) + assert.Equal(t, uint64(10), follower.subscriber.LocalDAHeight(), "localDAHeight should not change") + assert.Equal(t, uint64(10), follower.subscriber.HighestSeenDAHeight(), "highestSeen should be updated") + }) +} + +// backoffTestGenesis creates a test genesis for the backoff tests. +func backoffTestGenesis(addr []byte) genesis.Genesis { + return genesis.Genesis{ ChainID: "test-chain", InitialHeight: 1, - StartTime: time.Now().Add(-time.Hour), // Start in past + StartTime: time.Now().Add(-time.Hour), ProposerAddress: addr, DAStartHeight: 100, } - - syncer := NewSyncer( - st, - execution.NewDummyExecutor(), - nil, - cm, - common.NopMetrics(), - cfg, - gen, - extmocks.NewMockStore[*types.P2PSignedHeader](t), - extmocks.NewMockStore[*types.P2PData](t), - zerolog.Nop(), - common.DefaultBlockOptions(), - make(chan error, 1), - nil, - ) - - require.NoError(t, syncer.initializeState()) - return syncer } diff --git a/block/internal/syncing/syncer_benchmark_test.go b/block/internal/syncing/syncer_benchmark_test.go index 6ca482c05b..b0bccc6f74 100644 --- a/block/internal/syncing/syncer_benchmark_test.go +++ b/block/internal/syncing/syncer_benchmark_test.go @@ -43,11 +43,26 @@ func BenchmarkSyncerIO(b *testing.B) { fixt := newBenchFixture(b, spec.heights, spec.shuffledTx, spec.daDelay, spec.execDelay, true) // run both loops - go fixt.s.processLoop(fixt.s.ctx) - fixt.s.startSyncWorkers(fixt.s.ctx) + ctx := b.Context() + go fixt.s.processLoop(ctx) + + // Create a DAFollower to drive DA retrieval. + follower := NewDAFollower(DAFollowerConfig{ + Retriever: fixt.s.daRetriever, + Logger: zerolog.Nop(), + EventSink: fixt.s, + Namespace: []byte("ns"), + StartDAHeight: fixt.s.daRetrieverHeight.Load(), + DABlockTime: 0, + }).(*daFollower) + sub := follower.subscriber + sub.UpdateHighestForTest(spec.heights + daHeightOffset) + go sub.RunCatchupForTest(ctx) + + fixt.s.startSyncWorkers(ctx) require.Eventually(b, func() bool { - processedHeight, _ := fixt.s.store.Height(b.Context()) + processedHeight, _ := fixt.s.store.Height(ctx) return processedHeight == spec.heights }, 5*time.Second, 50*time.Microsecond) fixt.s.cancel() @@ -61,7 +76,7 @@ func BenchmarkSyncerIO(b *testing.B) { require.Len(b, fixt.s.heightInCh, 0) assert.Equal(b, spec.heights+daHeightOffset, fixt.s.daRetrieverHeight) - gotStoreHeight, err := fixt.s.store.Height(b.Context()) + gotStoreHeight, err := fixt.s.store.Height(ctx) require.NoError(b, err) assert.Equal(b, spec.heights, gotStoreHeight) } @@ -137,7 +152,7 @@ func newBenchFixture(b *testing.B, totalHeights uint64, shuffledTx bool, daDelay // Mock DA retriever to emit exactly totalHeights events, then HFF and cancel daR := NewMockDARetriever(b) - daR.On("PopPriorityHeight").Return(uint64(0)).Maybe() + for i := range totalHeights { daHeight := i + daHeightOffset daR.On("RetrieveFromDA", mock.Anything, daHeight). diff --git a/block/internal/syncing/syncer_forced_inclusion_test.go b/block/internal/syncing/syncer_forced_inclusion_test.go index bcb96d5bb1..daba0322dd 100644 --- a/block/internal/syncing/syncer_forced_inclusion_test.go +++ b/block/internal/syncing/syncer_forced_inclusion_test.go @@ -74,9 +74,11 @@ func newForcedInclusionSyncer(t *testing.T, daStart, epochSize uint64) (*Syncer, client.On("GetDataNamespace").Return([]byte(cfg.DA.DataNamespace)).Maybe() client.On("GetForcedInclusionNamespace").Return([]byte(cfg.DA.ForcedInclusionNamespace)).Maybe() client.On("HasForcedInclusionNamespace").Return(true).Maybe() + subCh := make(chan datypes.SubscriptionEvent) + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Maybe() daRetriever := NewDARetriever(client, cm, gen, zerolog.Nop()) - fiRetriever := da.NewForcedInclusionRetriever(client, zerolog.Nop(), cfg, gen.DAStartHeight, gen.DAEpochForcedInclusion) + fiRetriever := da.NewForcedInclusionRetriever(t.Context(), client, zerolog.Nop(), cfg.DA.BlockTime.Duration, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) t.Cleanup(fiRetriever.Stop) s := NewSyncer( @@ -145,8 +147,10 @@ func TestVerifyForcedInclusionTxs_NamespaceNotConfigured(t *testing.T) { client.On("GetDataNamespace").Return([]byte(cfg.DA.DataNamespace)).Maybe() client.On("GetForcedInclusionNamespace").Return([]byte(nil)).Maybe() client.On("HasForcedInclusionNamespace").Return(false).Maybe() + subCh := make(chan datypes.SubscriptionEvent) + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Maybe() - fiRetriever := da.NewForcedInclusionRetriever(client, zerolog.Nop(), cfg, gen.DAStartHeight, gen.DAEpochForcedInclusion) + fiRetriever := da.NewForcedInclusionRetriever(t.Context(), client, zerolog.Nop(), cfg.DA.BlockTime.Duration, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) t.Cleanup(fiRetriever.Stop) s := NewSyncer( diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 693fe5b556..537b3e8ad9 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -375,8 +375,15 @@ func TestSyncLoopPersistState(t *testing.T) { daRtrMock, p2pHndlMock := NewMockDARetriever(t), newMockp2pHandler(t) p2pHndlMock.On("ProcessHeight", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() p2pHndlMock.On("SetProcessedHeight", mock.Anything).Return().Maybe() - daRtrMock.On("PopPriorityHeight").Return(uint64(0)).Maybe() + syncerInst1.daRetriever, syncerInst1.p2pHandler = daRtrMock, p2pHndlMock + syncerInst1.daFollower = NewDAFollower(DAFollowerConfig{ + Retriever: daRtrMock, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(syncerInst1.PipeEvent), + Namespace: []byte("ns"), + StartDAHeight: syncerInst1.daRetrieverHeight.Load(), + }) // with n da blobs fetched var prevHeaderHash, prevAppHash []byte @@ -417,12 +424,25 @@ func TestSyncLoopPersistState(t *testing.T) { Return(nil, datypes.ErrHeightFromFuture) go syncerInst1.processLoop(ctx) + + // Create and start a DAFollower so DA retrieval actually happens. + follower1 := NewDAFollower(DAFollowerConfig{ + Retriever: daRtrMock, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(syncerInst1.PipeEvent), + Namespace: []byte("ns"), + StartDAHeight: syncerInst1.daRetrieverHeight.Load(), + DABlockTime: cfg.DA.BlockTime.Duration, + }).(*daFollower) + sub1 := follower1.subscriber + sub1.UpdateHighestForTest(myFutureDAHeight) + go sub1.RunCatchupForTest(ctx) syncerInst1.startSyncWorkers(ctx) syncerInst1.wg.Wait() requireEmptyChan(t, errorCh) t.Log("sync workers on instance1 completed") - require.Equal(t, myFutureDAHeight, syncerInst1.daRetrieverHeight.Load()) + require.Equal(t, myFutureDAHeight, follower1.subscriber.LocalDAHeight()) // wait for all events consumed require.NoError(t, cm.SaveToStore()) @@ -467,8 +487,15 @@ func TestSyncLoopPersistState(t *testing.T) { daRtrMock, p2pHndlMock = NewMockDARetriever(t), newMockp2pHandler(t) p2pHndlMock.On("ProcessHeight", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() p2pHndlMock.On("SetProcessedHeight", mock.Anything).Return().Maybe() - daRtrMock.On("PopPriorityHeight").Return(uint64(0)).Maybe() + syncerInst2.daRetriever, syncerInst2.p2pHandler = daRtrMock, p2pHndlMock + syncerInst2.daFollower = NewDAFollower(DAFollowerConfig{ + Retriever: daRtrMock, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(syncerInst2.PipeEvent), + Namespace: []byte("ns"), + StartDAHeight: syncerInst2.daRetrieverHeight.Load(), + }) daRtrMock.On("RetrieveFromDA", mock.Anything, mock.Anything). Run(func(arg mock.Arguments) { @@ -480,6 +507,19 @@ func TestSyncLoopPersistState(t *testing.T) { // when it starts, it should fetch from the last height it stopped at t.Log("sync workers on instance2 started") + + // Create a follower for instance 2. + follower2 := NewDAFollower(DAFollowerConfig{ + Retriever: daRtrMock, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(syncerInst2.PipeEvent), + Namespace: []byte("ns"), + StartDAHeight: syncerInst2.daRetrieverHeight.Load(), + DABlockTime: cfg.DA.BlockTime.Duration, + }).(*daFollower) + sub2 := follower2.subscriber + sub2.UpdateHighestForTest(syncerInst2.daRetrieverHeight.Load() + 1) + go sub2.RunCatchupForTest(ctx) syncerInst2.startSyncWorkers(ctx) syncerInst2.wg.Wait() @@ -692,6 +732,13 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { // Create a real daRetriever to test priority queue s.daRetriever = NewDARetriever(nil, cm, gen, zerolog.Nop()) + s.daFollower = NewDAFollower(DAFollowerConfig{ + Retriever: s.daRetriever, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(func(_ context.Context, _ common.DAHeightEvent) error { return nil }), + Namespace: []byte("ns"), + StartDAHeight: 0, + }) // Create event with DA height hint evt := common.DAHeightEvent{ @@ -710,7 +757,7 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { s.processHeightEvent(t.Context(), &evt) // Verify that the priority height was queued in the daRetriever - priorityHeight := s.daRetriever.PopPriorityHeight() + priorityHeight := s.daFollower.(*daFollower).popPriorityHeight() assert.Equal(t, uint64(100), priorityHeight) } @@ -749,6 +796,13 @@ func TestProcessHeightEvent_RejectsUnreasonableDAHint(t *testing.T) { require.NoError(t, s.initializeState()) s.ctx = context.Background() s.daRetriever = NewDARetriever(nil, cm, gen, zerolog.Nop()) + s.daFollower = NewDAFollower(DAFollowerConfig{ + Retriever: s.daRetriever, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(func(_ context.Context, _ common.DAHeightEvent) error { return nil }), + Namespace: []byte("ns"), + StartDAHeight: 0, + }) // Set store height to 1 so event at height 2 is "next" batch, err := st.NewBatch(context.Background()) @@ -767,7 +821,7 @@ func TestProcessHeightEvent_RejectsUnreasonableDAHint(t *testing.T) { s.processHeightEvent(t.Context(), &evt) // Verify that NO priority height was queued — the hint was rejected - priorityHeight := s.daRetriever.PopPriorityHeight() + priorityHeight := s.daFollower.(*daFollower).popPriorityHeight() assert.Equal(t, uint64(0), priorityHeight, "unreasonable DA hint should be rejected") } @@ -806,6 +860,13 @@ func TestProcessHeightEvent_AcceptsValidDAHint(t *testing.T) { require.NoError(t, s.initializeState()) s.ctx = context.Background() s.daRetriever = NewDARetriever(nil, cm, gen, zerolog.Nop()) + s.daFollower = NewDAFollower(DAFollowerConfig{ + Retriever: s.daRetriever, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(func(_ context.Context, _ common.DAHeightEvent) error { return nil }), + Namespace: []byte("ns"), + StartDAHeight: 0, + }) // Set store height to 1 so event at height 2 is "next" batch, err := st.NewBatch(context.Background()) @@ -824,104 +885,11 @@ func TestProcessHeightEvent_AcceptsValidDAHint(t *testing.T) { s.processHeightEvent(t.Context(), &evt) // Verify that the priority height was queued — the hint is valid - priorityHeight := s.daRetriever.PopPriorityHeight() + priorityHeight := s.daFollower.(*daFollower).popPriorityHeight() assert.Equal(t, uint64(50), priorityHeight, "valid DA hint should be queued") } -// TestProcessHeightEvent_SkipsDAHintWhenAlreadyDAIncluded verifies that when the -// DA-inclusion cache already has an entry for the block height carried by a P2P -// event, the DA height hint is NOT queued for priority retrieval. -func TestProcessHeightEvent_SkipsDAHintWhenAlreadyDAIncluded(t *testing.T) { - ds := dssync.MutexWrap(datastore.NewMapDatastore()) - st := store.New(ds) - cm, err := cache.NewManager(config.DefaultConfig(), st, zerolog.Nop()) - require.NoError(t, err) - - addr, _, _ := buildSyncTestSigner(t) - cfg := config.DefaultConfig() - gen := genesis.Genesis{ChainID: "tchain", InitialHeight: 1, StartTime: time.Now().Add(-time.Second), ProposerAddress: addr} - - mockExec := testmocks.NewMockExecutor(t) - mockExec.EXPECT().InitChain(mock.Anything, mock.Anything, uint64(1), "tchain").Return([]byte("app0"), nil).Once() - - mockDAClient := testmocks.NewMockClient(t) - - s := NewSyncer( - st, - mockExec, - mockDAClient, - cm, - common.NopMetrics(), - cfg, - gen, - extmocks.NewMockStore[*types.P2PSignedHeader](t), - extmocks.NewMockStore[*types.P2PData](t), - zerolog.Nop(), - common.DefaultBlockOptions(), - make(chan error, 1), - nil, - ) - require.NoError(t, s.initializeState()) - s.ctx = context.Background() - s.daRetriever = NewDARetriever(nil, cm, gen, zerolog.Nop()) - - // Set the store height to 1 so the event at height 2 is "next". - batch, err := st.NewBatch(context.Background()) - require.NoError(t, err) - require.NoError(t, batch.SetHeight(1)) - require.NoError(t, batch.Commit()) - - // Simulate the DA retriever having already confirmed header and data at height 2. - cm.SetHeaderDAIncluded("somehash-hdr2", 42, 2) - cm.SetDataDAIncluded("somehash-data2", 42, 2) - - // Both hints point to DA height 100, which is above daRetrieverHeight (0), - // so the only reason NOT to queue them is the cache hit. - evt := common.DAHeightEvent{ - Header: &types.SignedHeader{Header: types.Header{BaseHeader: types.BaseHeader{ChainID: gen.ChainID, Height: 2}}}, - Data: &types.Data{Metadata: &types.Metadata{ChainID: gen.ChainID, Height: 2}}, - Source: common.SourceP2P, - DaHeightHints: [2]uint64{100, 100}, - } - - s.processHeightEvent(t.Context(), &evt) - - // Neither hint should be queued — both are already DA-included in the cache. - priorityHeight := s.daRetriever.PopPriorityHeight() - assert.Equal(t, uint64(0), priorityHeight, - "DA hint must not be queued when header and data are already DA-included in cache") - - // ── partial case: only header confirmed ────────────────────────────────── - // Reset with a fresh cache that has only the header entry for height 3. - cm2, err := cache.NewManager(config.DefaultConfig(), st, zerolog.Nop()) - require.NoError(t, err) - s.cache = cm2 - cm2.SetHeaderDAIncluded("somehash-hdr3", 55, 3) - // data at height 3 is NOT in cache - - batch, err = st.NewBatch(context.Background()) - require.NoError(t, err) - require.NoError(t, batch.SetHeight(2)) - require.NoError(t, batch.Commit()) - - evt3 := common.DAHeightEvent{ - Header: &types.SignedHeader{Header: types.Header{BaseHeader: types.BaseHeader{ChainID: gen.ChainID, Height: 3}}}, - Data: &types.Data{Metadata: &types.Metadata{ChainID: gen.ChainID, Height: 3}, Txs: types.Txs{types.Tx("tx2")}}, - Source: common.SourceP2P, - DaHeightHints: [2]uint64{150, 151}, // within daHintMaxDrift(200) of daRetrieverHeight(0) — no DA validation needed - } - - s.processHeightEvent(t.Context(), &evt3) - - // Header hint (150) must be skipped; data hint (151) must be queued. - priorityHeight = s.daRetriever.PopPriorityHeight() - assert.Equal(t, uint64(151), priorityHeight, - "data hint must be queued when only the header is already DA-included") - assert.Equal(t, uint64(0), s.daRetriever.PopPriorityHeight(), - "no further hints should be queued") -} - -func TestProcessHeightEvent_SkipsDAHintWhenBelowRetrieverCursor(t *testing.T) { +func TestProcessHeightEvent_SkipsDAHintWhenAlreadyFetched(t *testing.T) { ds := dssync.MutexWrap(datastore.NewMapDatastore()) st := store.New(ds) cm, err := cache.NewManager(config.DefaultConfig(), st, zerolog.Nop()) @@ -958,9 +926,15 @@ func TestProcessHeightEvent_SkipsDAHintWhenBelowRetrieverCursor(t *testing.T) { // Create a real daRetriever to test priority queue s.daRetriever = NewDARetriever(nil, cm, gen, zerolog.Nop()) + s.daFollower = NewDAFollower(DAFollowerConfig{ + Retriever: s.daRetriever, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(func(_ context.Context, _ common.DAHeightEvent) error { return nil }), + Namespace: []byte("ns"), + StartDAHeight: 0, + }) - // Set DA retriever height to 150 - the sequential DA scan cursor is already past - // the hint (100), so there is no need to queue a priority retrieval. + // Set DA retriever height to 150 - simulating we've already fetched past height 100 s.daRetrieverHeight.Store(150) // Set the store height to 1 so the event can be processed @@ -979,9 +953,8 @@ func TestProcessHeightEvent_SkipsDAHintWhenBelowRetrieverCursor(t *testing.T) { s.processHeightEvent(t.Context(), &evt) - // Verify that no priority height was queued: the hint (100) is below the - // retriever cursor (150), so sequential scanning will cover it naturally. - priorityHeight := s.daRetriever.PopPriorityHeight() + // Verify that no priority height was queued since we've already fetched past it + priorityHeight := s.daFollower.(*daFollower).popPriorityHeight() assert.Equal(t, uint64(0), priorityHeight, "should not queue DA hint that is below current daRetrieverHeight") // Now test with a hint that is ABOVE the current daRetrieverHeight @@ -1001,7 +974,7 @@ func TestProcessHeightEvent_SkipsDAHintWhenBelowRetrieverCursor(t *testing.T) { s.processHeightEvent(t.Context(), &evt2) // Verify that the priority height WAS queued since it's above daRetrieverHeight - priorityHeight = s.daRetriever.PopPriorityHeight() + priorityHeight = s.daFollower.(*daFollower).popPriorityHeight() assert.Equal(t, uint64(200), priorityHeight, "should queue DA hint that is above current daRetrieverHeight") } @@ -1129,7 +1102,7 @@ func TestSyncer_Stop_SkipsDrainOnCriticalError(t *testing.T) { // Stop must complete quickly — no drain, no ExecuteTxs calls done := make(chan struct{}) go func() { - _ = s.Stop() + _ = s.Stop(t.Context()) close(done) }() @@ -1198,7 +1171,7 @@ func TestSyncer_Stop_DrainWorksWithoutCriticalError(t *testing.T) { // hasCriticalError is false (default) — drain should process events including ExecuteTxs s.wg.Go(func() {}) - _ = s.Stop() + _ = s.Stop(t.Context()) // Verify ExecuteTxs was actually called during drain mockExec.AssertExpectations(t) diff --git a/block/public.go b/block/public.go index 39f6c3e308..988760211b 100644 --- a/block/public.go +++ b/block/public.go @@ -84,12 +84,13 @@ type ForcedInclusionRetriever interface { // It internally creates and manages an AsyncBlockRetriever for background prefetching. // Tracing is automatically enabled when configured. func NewForcedInclusionRetriever( + ctx context.Context, client DAClient, cfg config.Config, logger zerolog.Logger, daStartHeight, daEpochSize uint64, ) ForcedInclusionRetriever { - return da.NewForcedInclusionRetriever(client, logger, cfg, daStartHeight, daEpochSize) + return da.NewForcedInclusionRetriever(ctx, client, logger, cfg.DA.BlockTime.Duration, cfg.Instrumentation.IsTracingEnabled(), daStartHeight, daEpochSize) } // Expose Raft types for consensus integration diff --git a/docs/guides/da-layers/celestia.md b/docs/guides/da-layers/celestia.md index 88bbd48e4c..307dde63e9 100644 --- a/docs/guides/da-layers/celestia.md +++ b/docs/guides/da-layers/celestia.md @@ -72,10 +72,10 @@ You can use the same namespace for both headers and data, or separate them for o ### Set DA Address -Default Celestia light node port is 26658: +Default Celestia light node port is 26658. **Note:** Connection to a celestia-node DA requires a websocket connection: ```bash -DA_ADDRESS=http://localhost:26658 +DA_ADDRESS=ws://localhost:26658 ``` ## Running Your Chain diff --git a/node/failover.go b/node/failover.go index f6fd09a3a0..c60e27a5f4 100644 --- a/node/failover.go +++ b/node/failover.go @@ -255,7 +255,7 @@ func (f *failoverState) runCatchupPhase(ctx context.Context) error { if err := f.bc.Syncer.Start(ctx); err != nil { return fmt.Errorf("catchup syncer start: %w", err) } - defer f.bc.Syncer.Stop() // nolint:errcheck // not critical + defer f.bc.Syncer.Stop(context.Background()) // nolint:errcheck,contextcheck // not critical caughtUp, err := f.waitForCatchup(ctx) if err != nil { diff --git a/pkg/cmd/run_node.go b/pkg/cmd/run_node.go index 33b1eba006..4a5c7577c6 100644 --- a/pkg/cmd/run_node.go +++ b/pkg/cmd/run_node.go @@ -107,14 +107,8 @@ func StartNode( }() } - blobClient, err := blobrpc.NewClient(ctx, nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") - if err != nil { - return fmt.Errorf("failed to create blob client: %w", err) - } - defer blobClient.Close() - daClient := block.NewDAClient(blobClient, nodeConfig, logger) - - // create a new remote signer + // Validate and load signer first (before attempting DA connection, which may fail + // eagerly over WebSocket if no DA server is running). var signer signer.Signer if nodeConfig.Signer.SignerType == "file" && (nodeConfig.Node.Aggregator && !nodeConfig.Node.BasedSequencer) { // Get passphrase file path @@ -152,6 +146,13 @@ func StartNode( return fmt.Errorf("unknown signer type: %s", nodeConfig.Signer.SignerType) } + blobClient, err := blobrpc.NewWSClient(ctx, nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") + if err != nil { + return fmt.Errorf("failed to create blob client: %w", err) + } + defer blobClient.Close() + daClient := block.NewDAClient(blobClient, nodeConfig, logger) + // sanity check for based sequencer if nodeConfig.Node.BasedSequencer && genesis.DAStartHeight == 0 { return fmt.Errorf("based sequencing requires DAStartHeight to be set in genesis. This value should be identical for all nodes of the chain") diff --git a/pkg/da/jsonrpc/client.go b/pkg/da/jsonrpc/client.go index f1a31a9738..c856cdd7b9 100644 --- a/pkg/da/jsonrpc/client.go +++ b/pkg/da/jsonrpc/client.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "strings" libshare "github.com/celestiaorg/go-square/v3/share" "github.com/filecoin-project/go-jsonrpc" @@ -23,7 +24,17 @@ func (c *Client) Close() { } } -// NewClient connects to the celestia-node RPC endpoint +// httpToWS converts an HTTP(S) URL to a WebSocket URL. +func httpToWS(addr string) string { + addr = strings.Replace(addr, "https://", "wss://", 1) + addr = strings.Replace(addr, "http://", "ws://", 1) + return addr +} + +// NewClient connects to the DA RPC endpoint using the address as-is. +// Uses HTTP by default (lazy connection — only connects on first RPC call). +// Does NOT support channel-based subscriptions (e.g. Subscribe). +// For subscription support, use NewWSClient instead. func NewClient(ctx context.Context, addr, token string, authHeaderName string) (*Client, error) { var httpHeader http.Header if token != "" { @@ -57,6 +68,15 @@ func NewClient(ctx context.Context, addr, token string, authHeaderName string) ( return &cl, nil } +// NewWSClient connects to the DA RPC endpoint over WebSocket. +// Automatically converts http:// to ws:// (and https:// to wss://). +// Supports channel-based subscriptions (e.g. Subscribe). +// Note: WebSocket connections are eager — they connect at creation time +// and will fail immediately if the server is unavailable. +func NewWSClient(ctx context.Context, addr, token string, authHeaderName string) (*Client, error) { + return NewClient(ctx, httpToWS(addr), token, authHeaderName) +} + // BlobAPI mirrors celestia-node's blob module (nodebuilder/blob/blob.go). // jsonrpc.NewClient wires Internal.* to RPC stubs. type BlobAPI struct { diff --git a/pkg/da/jsonrpc/types.go b/pkg/da/jsonrpc/types.go index b7377e950e..daecb9ec71 100644 --- a/pkg/da/jsonrpc/types.go +++ b/pkg/da/jsonrpc/types.go @@ -8,6 +8,7 @@ type CommitmentProof struct { // SubscriptionResponse mirrors celestia-node's blob.SubscriptionResponse. type SubscriptionResponse struct { - Blobs []*Blob `json:"blobs"` - Height uint64 `json:"height"` + Blobs []*Blob `json:"blobs"` + Height uint64 `json:"height"` + Header *RawHeader `json:"header,omitempty"` // Available in celestia-node v0.29.1+ } diff --git a/pkg/da/types/types.go b/pkg/da/types/types.go index b2f2e7bc30..24bcceea84 100644 --- a/pkg/da/types/types.go +++ b/pkg/da/types/types.go @@ -82,3 +82,16 @@ func SplitID(id []byte) (uint64, []byte, error) { commitment := id[8:] return binary.LittleEndian.Uint64(id[:8]), commitment, nil } + +// SubscriptionEvent is a namespace-agnostic signal that a blob was finalized at +// Height on the DA layer. Produced by Subscribe and consumed by DA followers. +type SubscriptionEvent struct { + // Height is the DA layer height at which the blob was finalized. + Height uint64 + // Timestamp is the DA block timestamp at the given height. + // This is an optional field that is not set by default. Enable via SubscribeExtra method + Timestamp time.Time + // Blobs contains the raw blob data from the subscription response. + // When non-nil, followers can process blobs inline without re-fetching from DA. + Blobs [][]byte +} diff --git a/pkg/sequencers/based/sequencer.go b/pkg/sequencers/based/sequencer.go index f079d8ccab..22bb8947f5 100644 --- a/pkg/sequencers/based/sequencer.go +++ b/pkg/sequencers/based/sequencer.go @@ -97,7 +97,7 @@ func NewBasedSequencer( } } - bs.fiRetriever = block.NewForcedInclusionRetriever(daClient, cfg, logger, genesis.DAStartHeight, genesis.DAEpochForcedInclusion) + bs.fiRetriever = block.NewForcedInclusionRetriever(context.Background(), daClient, cfg, logger, genesis.DAStartHeight, genesis.DAEpochForcedInclusion) return bs, nil } diff --git a/pkg/sequencers/based/sequencer_test.go b/pkg/sequencers/based/sequencer_test.go index c51a7673fe..b4690d527d 100644 --- a/pkg/sequencers/based/sequencer_test.go +++ b/pkg/sequencers/based/sequencer_test.go @@ -71,6 +71,7 @@ func createTestSequencer(t *testing.T, mockRetriever *common.MockForcedInclusion // Mock the forced inclusion namespace call mockDAClient.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockExec := createDefaultMockExecutor(t) @@ -469,6 +470,7 @@ func TestBasedSequencer_CheckpointPersistence(t *testing.T) { } mockDAClient.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // Create first sequencer mockExec1 := createDefaultMockExecutor(t) @@ -496,6 +498,7 @@ func TestBasedSequencer_CheckpointPersistence(t *testing.T) { } mockDAClient2.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient2.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDAClient2.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockExec2 := createDefaultMockExecutor(t) seq2, err := NewBasedSequencer(mockDAClient2, config.DefaultConfig(), db, gen, zerolog.Nop(), mockExec2) require.NoError(t, err) @@ -542,6 +545,7 @@ func TestBasedSequencer_CrashRecoveryMidEpoch(t *testing.T) { mockDAClient.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // Create mock executor that postpones tx2 on first call filterCallCount := 0 @@ -602,6 +606,7 @@ func TestBasedSequencer_CrashRecoveryMidEpoch(t *testing.T) { } mockDAClient2.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient2.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDAClient2.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() seq2, err := NewBasedSequencer(mockDAClient2, config.DefaultConfig(), db, gen, zerolog.Nop(), mockExec) require.NoError(t, err) @@ -927,6 +932,7 @@ func TestBasedSequencer_GetNextBatch_GasFilteringPreservesUnprocessedTxs(t *test } mockDAClient.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() seq, err := NewBasedSequencer(mockDAClient, config.DefaultConfig(), db, gen, zerolog.New(zerolog.NewTestWriter(t)), mockExec) require.NoError(t, err) diff --git a/pkg/sequencers/single/sequencer.go b/pkg/sequencers/single/sequencer.go index 16cdf12b92..44bbb5b311 100644 --- a/pkg/sequencers/single/sequencer.go +++ b/pkg/sequencers/single/sequencer.go @@ -130,7 +130,7 @@ func NewSequencer( // Determine initial DA height for forced inclusion initialDAHeight := s.getInitialDAStartHeight(context.Background()) - s.fiRetriever = block.NewForcedInclusionRetriever(daClient, cfg, logger, initialDAHeight, genesis.DAEpochForcedInclusion) + s.fiRetriever = block.NewForcedInclusionRetriever(context.Background(), daClient, cfg, logger, initialDAHeight, genesis.DAEpochForcedInclusion) return s, nil } @@ -202,7 +202,7 @@ func (c *Sequencer) GetNextBatch(ctx context.Context, req coresequencer.GetNextB if c.fiRetriever != nil { c.fiRetriever.Stop() } - c.fiRetriever = block.NewForcedInclusionRetriever(c.daClient, c.cfg, c.logger, c.getInitialDAStartHeight(ctx), c.genesis.DAEpochForcedInclusion) + c.fiRetriever = block.NewForcedInclusionRetriever(ctx, c.daClient, c.cfg, c.logger, c.getInitialDAStartHeight(ctx), c.genesis.DAEpochForcedInclusion) } // If we have no cached transactions or we've consumed all from the current epoch, diff --git a/pkg/sequencers/single/sequencer_test.go b/pkg/sequencers/single/sequencer_test.go index b0bbe247c5..0a393d4275 100644 --- a/pkg/sequencers/single/sequencer_test.go +++ b/pkg/sequencers/single/sequencer_test.go @@ -380,6 +380,7 @@ func TestSequencer_GetNextBatch_ForcedInclusionAndBatch_MaxBytes(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — same as sequencer start, no catch-up needed mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(100), nil).Maybe() @@ -471,6 +472,7 @@ func TestSequencer_GetNextBatch_ForcedInclusion_ExceedsMaxBytes(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — same as sequencer start, no catch-up needed mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(100), nil).Maybe() @@ -554,6 +556,7 @@ func TestSequencer_GetNextBatch_AlwaysCheckPendingForcedInclusion(t *testing.T) mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — same as sequencer start, no catch-up needed mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(100), nil).Maybe() @@ -895,6 +898,7 @@ func TestSequencer_CheckpointPersistence_CrashRecovery(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 101 — close to sequencer start (100), no catch-up needed. // Use Maybe() since two sequencer instances share this mock. @@ -998,6 +1002,7 @@ func TestSequencer_GetNextBatch_EmptyDABatch_IncreasesDAHeight(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — same as sequencer start, no catch-up needed mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(100), nil).Maybe() @@ -1091,6 +1096,7 @@ func TestSequencer_GetNextBatch_WithGasFiltering(t *testing.T) { mockDA.MockClient.On("GetBlobsAtHeight", mock.Anything, mock.Anything, mock.Anything). Return(forcedTxs, nil).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return([]byte("forced")).Maybe() mockDA.MockClient.On("MaxBlobSize", mock.Anything).Return(uint64(1000000), nil).Maybe() @@ -1190,6 +1196,7 @@ func TestSequencer_GetNextBatch_GasFilterError(t *testing.T) { mockDA := newMockFullDAClient(t) mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return([]byte("forced")).Maybe() mockDA.MockClient.On("MaxBlobSize", mock.Anything).Return(uint64(1000000), nil).Maybe() @@ -1254,6 +1261,7 @@ func TestSequencer_CatchUp_DetectsOldEpoch(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at height 105 — sequencer starts at 100 with epoch size 1, // so it has missed 5 epochs (>1), triggering catch-up. @@ -1324,6 +1332,7 @@ func TestSequencer_CatchUp_SkipsMempoolDuringCatchUp(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — sequencer starts at 100 with epoch size 1, // so it has missed multiple epochs, triggering catch-up. @@ -1416,6 +1425,7 @@ func TestSequencer_CatchUp_UsesDATimestamp(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — multiple epochs ahead, triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(105), nil).Once() @@ -1475,6 +1485,7 @@ func TestSequencer_CatchUp_ExitsCatchUpAtDAHead(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — multiple epochs ahead, triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(105), nil).Once() @@ -1556,6 +1567,7 @@ func TestSequencer_CatchUp_HeightFromFutureExitsCatchUp(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — multiple epochs ahead, triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(105), nil).Once() @@ -1622,6 +1634,7 @@ func TestSequencer_CatchUp_NoCatchUpWhenRecentEpoch(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — sequencer starts at 100 with epoch size 1, // so it is within the same epoch (0 missed). No catch-up. @@ -1688,6 +1701,7 @@ func TestSequencer_CatchUp_MultiEpochReplay(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 106 — sequencer starts at 100 with epoch size 1, // so it has missed 6 epochs (>1), triggering catch-up. @@ -1839,6 +1853,7 @@ func TestSequencer_CatchUp_CheckpointAdvancesDuringCatchUp(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — multiple epochs ahead, triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(105), nil).Once() @@ -1931,6 +1946,7 @@ func TestSequencer_CatchUp_MonotonicTimestamps(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is far ahead — triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(110), nil).Once() @@ -2057,6 +2073,7 @@ func TestSequencer_CatchUp_MonotonicTimestamps_EmptyEpoch(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(110), nil).Once() @@ -2133,6 +2150,7 @@ func TestSequencer_GetNextBatch_GasFilteringPreservesUnprocessedTxs(t *testing.T mockDA := newMockFullDAClient(t) mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return([]byte("forced")).Maybe() mockDA.MockClient.On("MaxBlobSize", mock.Anything).Return(uint64(1000000), nil).Maybe() diff --git a/test/e2e/evm_force_inclusion_e2e_test.go b/test/e2e/evm_force_inclusion_e2e_test.go index f201f7f363..c6c25270c5 100644 --- a/test/e2e/evm_force_inclusion_e2e_test.go +++ b/test/e2e/evm_force_inclusion_e2e_test.go @@ -238,8 +238,10 @@ func TestEvmFullNodeForceInclusionE2E(t *testing.T) { // --- End Sequencer Setup --- // --- Start Full Node Setup --- + // Get sequencer's full P2P address (including peer ID) for the full node to connect to + sequencerP2PAddress := getNodeP2PAddress(t, sut, sequencerHome, env.Endpoints.RollkitRPCPort) // Reuse setupFullNode helper which handles genesis copying and node startup - setupFullNode(t, sut, fullNodeHome, sequencerHome, env.FullNodeJWT, env.GenesisHash, env.Endpoints.GetRollkitP2PAddress(), env.Endpoints) + setupFullNode(t, sut, fullNodeHome, sequencerHome, env.FullNodeJWT, env.GenesisHash, sequencerP2PAddress, env.Endpoints) t.Log("Full node is up") // --- End Full Node Setup --- diff --git a/test/e2e/evm_test_common.go b/test/e2e/evm_test_common.go index 7c089e762b..c5bfb05c23 100644 --- a/test/e2e/evm_test_common.go +++ b/test/e2e/evm_test_common.go @@ -449,15 +449,15 @@ func setupFullNode(t testing.TB, sut *SystemUnderTest, fullNodeHome, sequencerHo "--home", fullNodeHome, "--evm.jwt-secret-file", fullNodeJwtSecretFile, "--evm.genesis-hash", genesisHash, - "--rollkit.p2p.peers", sequencerP2PAddress, + "--evnode.p2p.peers", sequencerP2PAddress, "--evm.engine-url", endpoints.GetFullNodeEngineURL(), "--evm.eth-url", endpoints.GetFullNodeEthURL(), - "--rollkit.da.block_time", DefaultDABlockTime, - "--rollkit.da.address", endpoints.GetDAAddress(), - "--rollkit.da.namespace", DefaultDANamespace, - "--rollkit.da.batching_strategy", "immediate", - "--rollkit.rpc.address", endpoints.GetFullNodeRPCListen(), - "--rollkit.p2p.listen_address", endpoints.GetFullNodeP2PAddress(), + "--evnode.da.block_time", DefaultDABlockTime, + "--evnode.da.address", endpoints.GetDAAddress(), + "--evnode.da.namespace", DefaultDANamespace, + "--evnode.da.batching_strategy", "immediate", + "--evnode.rpc.address", endpoints.GetFullNodeRPCListen(), + "--evnode.p2p.listen_address", endpoints.GetFullNodeP2PAddress(), } sut.ExecCmd(evmSingleBinaryPath, args...) // Use AwaitNodeLive instead of AwaitNodeUp because in lazy mode scenarios, @@ -932,4 +932,3 @@ func PrintTraceReport(t testing.TB, label string, spans []TraceSpan) { t.Logf("%-40s %5.1f%% %s", name, pct, bar) } } - diff --git a/test/mocks/da.go b/test/mocks/da.go index f5293d907a..f3a4e960db 100644 --- a/test/mocks/da.go +++ b/test/mocks/da.go @@ -492,6 +492,80 @@ func (_c *MockClient_Submit_Call) RunAndReturn(run func(ctx context.Context, dat return _c } +// Subscribe provides a mock function for the type MockClient +func (_mock *MockClient) Subscribe(ctx context.Context, namespace []byte, fetchTimestamp bool) (<-chan da.SubscriptionEvent, error) { + ret := _mock.Called(ctx, namespace, fetchTimestamp) + + if len(ret) == 0 { + panic("no return value specified for Subscribe") + } + + var r0 <-chan da.SubscriptionEvent + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, []byte, bool) (<-chan da.SubscriptionEvent, error)); ok { + return returnFunc(ctx, namespace, fetchTimestamp) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, []byte, bool) <-chan da.SubscriptionEvent); ok { + r0 = returnFunc(ctx, namespace, fetchTimestamp) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan da.SubscriptionEvent) + } + } + if returnFunc, ok := ret.Get(1).(func(context.Context, []byte, bool) error); ok { + r1 = returnFunc(ctx, namespace, fetchTimestamp) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockClient_Subscribe_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Subscribe' +type MockClient_Subscribe_Call struct { + *mock.Call +} + +// Subscribe is a helper method to define mock.On call +// - ctx context.Context +// - namespace []byte +// - fetchTimestamp bool +func (_e *MockClient_Expecter) Subscribe(ctx interface{}, namespace interface{}, fetchTimestamp interface{}) *MockClient_Subscribe_Call { + return &MockClient_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, namespace, fetchTimestamp)} +} + +func (_c *MockClient_Subscribe_Call) Run(run func(ctx context.Context, namespace []byte, fetchTimestamp bool)) *MockClient_Subscribe_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 []byte + if args[1] != nil { + arg1 = args[1].([]byte) + } + var arg2 bool + if args[2] != nil { + arg2 = args[2].(bool) + } + run( + arg0, + arg1, + arg2, + ) + }) + return _c +} + +func (_c *MockClient_Subscribe_Call) Return(subscriptionEventCh <-chan da.SubscriptionEvent, err error) *MockClient_Subscribe_Call { + _c.Call.Return(subscriptionEventCh, err) + return _c +} + +func (_c *MockClient_Subscribe_Call) RunAndReturn(run func(ctx context.Context, namespace []byte, fetchTimestamp bool) (<-chan da.SubscriptionEvent, error)) *MockClient_Subscribe_Call { + _c.Call.Return(run) + return _c +} + // NewMockVerifier creates a new instance of MockVerifier. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewMockVerifier(t interface { diff --git a/test/testda/dummy.go b/test/testda/dummy.go index 648021b76a..2f92e47e60 100644 --- a/test/testda/dummy.go +++ b/test/testda/dummy.go @@ -28,6 +28,12 @@ func (h *Header) Time() time.Time { return h.Timestamp } +// subscriber holds a channel and the context for a single Subscribe caller. +type subscriber struct { + ch chan datypes.SubscriptionEvent + ctx context.Context +} + // DummyDA is a test implementation of the DA client interface. // It supports blob storage, height simulation, failure injection, and header retrieval. type DummyDA struct { @@ -38,10 +44,52 @@ type DummyDA struct { headers map[uint64]*Header // height -> header (with timestamp) failSubmit atomic.Bool + // subscribers tracks active Subscribe callers. + subscribers []*subscriber + tickerMu sync.Mutex tickerStop chan struct{} } +// Subscribe returns a channel that emits a SubscriptionEvent for every new DA +// height produced by Submit or StartHeightTicker. The channel is closed when +// ctx is cancelled or Reset is called. +func (d *DummyDA) Subscribe(ctx context.Context, _ []byte, _ bool) (<-chan datypes.SubscriptionEvent, error) { + ch := make(chan datypes.SubscriptionEvent, 64) + sub := &subscriber{ch: ch, ctx: ctx} + + d.mu.Lock() + d.subscribers = append(d.subscribers, sub) + d.mu.Unlock() + + // Remove subscriber and close channel when ctx is cancelled. + go func() { + <-ctx.Done() + d.mu.Lock() + defer d.mu.Unlock() + for i, s := range d.subscribers { + if s == sub { + d.subscribers = append(d.subscribers[:i], d.subscribers[i+1:]...) + close(ch) + break + } + } + }() + + return ch, nil +} + +// notifySubscribers sends an event to all active subscribers. Must be called +// with d.mu held. +func (d *DummyDA) notifySubscribers(ev datypes.SubscriptionEvent) { + for _, sub := range d.subscribers { + select { + case sub.ch <- ev: + case <-sub.ctx.Done(): + } + } +} + // Option configures a DummyDA instance. type Option func(*DummyDA) @@ -111,6 +159,10 @@ func (d *DummyDA) Submit(_ context.Context, data [][]byte, _ float64, namespace Timestamp: now, } } + d.notifySubscribers(datypes.SubscriptionEvent{ + Height: height, + Blobs: data, + }) d.mu.Unlock() return datypes.ResultSubmit{ @@ -253,6 +305,9 @@ func (d *DummyDA) StartHeightTicker(interval time.Duration) func() { Timestamp: now, } } + d.notifySubscribers(datypes.SubscriptionEvent{ + Height: height, + }) d.mu.Unlock() case <-stopCh: return @@ -277,6 +332,11 @@ func (d *DummyDA) Reset() { d.blobs = make(map[uint64]map[string][][]byte) d.headers = make(map[uint64]*Header) d.failSubmit.Store(false) + // Close all subscriber channels. + for _, sub := range d.subscribers { + close(sub.ch) + } + d.subscribers = nil d.mu.Unlock() d.tickerMu.Lock() diff --git a/tools/local-da/local.go b/tools/local-da/local.go index ef519f17ce..fb47f6c3b8 100644 --- a/tools/local-da/local.go +++ b/tools/local-da/local.go @@ -13,6 +13,7 @@ import ( "sync" "time" + libshare "github.com/celestiaorg/go-square/v3/share" "github.com/rs/zerolog" blobrpc "github.com/evstack/ev-node/pkg/da/jsonrpc" @@ -27,6 +28,18 @@ const ( DefaultBlockTime = 1 * time.Second ) +// subscriber holds a registered subscription's channel and namespace filter. +type subscriber struct { + ch chan subscriptionEvent + ns libshare.Namespace +} + +// subscriptionEvent is sent to subscribers when a new DA block is produced. +type subscriptionEvent struct { + height uint64 + blobs []*blobrpc.Blob +} + // LocalDA is a simple implementation of in-memory DA. Not production ready! Intended only for testing! // // Data is stored in a map, where key is a serialized sequence number. This key is returned as ID. @@ -43,6 +56,10 @@ type LocalDA struct { blockTime time.Duration lastTime time.Time // tracks last timestamp to ensure monotonicity + // Subscriber registry (protected by mu) + subscribers map[int]*subscriber + nextSubID int + logger zerolog.Logger } @@ -57,6 +74,7 @@ func NewLocalDA(logger zerolog.Logger, opts ...func(*LocalDA) *LocalDA) *LocalDA data: make(map[uint64][]kvp), timestamps: make(map[uint64]time.Time), blobData: make(map[uint64][]*blobrpc.Blob), + subscribers: make(map[int]*subscriber), maxBlobSize: DefaultMaxBlobSize, blockTime: DefaultBlockTime, lastTime: time.Now(), @@ -204,11 +222,22 @@ func (d *LocalDA) SubmitWithOptions(ctx context.Context, blobs []datypes.Blob, g ids := make([]datypes.ID, len(blobs)) d.height += 1 d.timestamps[d.height] = d.monotonicTime() + + nspace, _ := libshare.NewNamespaceFromBytes(ns) + rpcBlobs := make([]*blobrpc.Blob, len(blobs)) + for i, blob := range blobs { ids[i] = append(d.nextID(), d.getHash(blob)...) d.data[d.height] = append(d.data[d.height], kvp{ids[i], blob}) + + if b, err := blobrpc.NewBlobV0(nspace, blob); err == nil { + rpcBlobs[i] = b + } } + d.blobData[d.height] = rpcBlobs + + d.notifySubscribers(d.height) d.logger.Info().Uint64("newHeight", d.height).Int("count", len(ids)).Msg("SubmitWithOptions successful") return ids, nil } @@ -234,11 +263,22 @@ func (d *LocalDA) Submit(ctx context.Context, blobs []datypes.Blob, gasPrice flo ids := make([]datypes.ID, len(blobs)) d.height += 1 d.timestamps[d.height] = d.monotonicTime() + + nspace, _ := libshare.NewNamespaceFromBytes(ns) + rpcBlobs := make([]*blobrpc.Blob, len(blobs)) + for i, blob := range blobs { ids[i] = append(d.nextID(), d.getHash(blob)...) d.data[d.height] = append(d.data[d.height], kvp{ids[i], blob}) + + if b, err := blobrpc.NewBlobV0(nspace, blob); err == nil { + rpcBlobs[i] = b + } } + d.blobData[d.height] = rpcBlobs + + d.notifySubscribers(d.height) d.logger.Info().Uint64("newHeight", d.height).Int("count", len(ids)).Msg("Submit successful") return ids, nil } @@ -335,5 +375,68 @@ func (d *LocalDA) produceEmptyBlock() { defer d.mu.Unlock() d.height++ d.timestamps[d.height] = d.monotonicTime() + d.notifySubscribers(d.height) d.logger.Debug().Uint64("height", d.height).Msg("produced empty block") } + +// subscribe registers a new subscriber for blobs matching the given namespace. +// Returns a read-only channel and a subscription ID for later unsubscription. +// Must NOT be called with d.mu held. +func (d *LocalDA) subscribe(ns libshare.Namespace) (<-chan subscriptionEvent, int) { + d.mu.Lock() + defer d.mu.Unlock() + + id := d.nextSubID + d.nextSubID++ + ch := make(chan subscriptionEvent, 64) + d.subscribers[id] = &subscriber{ch: ch, ns: ns} + d.logger.Info().Int("subID", id).Str("namespace", hex.EncodeToString(ns.Bytes())).Msg("subscriber registered") + return ch, id +} + +// unsubscribe removes a subscriber and closes its channel. +// Must NOT be called with d.mu held. +func (d *LocalDA) unsubscribe(id int) { + d.mu.Lock() + defer d.mu.Unlock() + + if sub, ok := d.subscribers[id]; ok { + close(sub.ch) + delete(d.subscribers, id) + d.logger.Info().Int("subID", id).Msg("subscriber unregistered") + } +} + +// notifySubscribers sends a subscriptionEvent to all registered subscribers. +// For each subscriber, only blobs matching the subscriber's namespace are included. +// Slow consumers (full channel) are dropped to avoid blocking block production. +// MUST be called with d.mu held. +func (d *LocalDA) notifySubscribers(height uint64) { + if len(d.subscribers) == 0 { + return + } + + allBlobs := d.blobData[height] // may be nil for empty blocks + + for id, sub := range d.subscribers { + // Filter blobs matching subscriber namespace + var matched []*blobrpc.Blob + for _, b := range allBlobs { + if b != nil && b.Namespace().Equals(sub.ns) { + matched = append(matched, b) + } + } + + evt := subscriptionEvent{ + height: height, + blobs: matched, + } + + select { + case sub.ch <- evt: + default: + // Slow consumer — drop to avoid blocking block production + d.logger.Warn().Int("subID", id).Uint64("height", height).Msg("dropping event for slow subscriber") + } + } +} diff --git a/tools/local-da/rpc.go b/tools/local-da/rpc.go index c8a3c97bee..024910c6bd 100644 --- a/tools/local-da/rpc.go +++ b/tools/local-da/rpc.go @@ -31,6 +31,7 @@ func (s *blobServer) Submit(_ context.Context, blobs []*jsonrpc.Blob, _ *jsonrpc if len(blobs) == 0 { s.da.timestamps[height] = time.Now() + s.da.notifySubscribers(height) return height, nil } @@ -48,6 +49,7 @@ func (s *blobServer) Submit(_ context.Context, blobs []*jsonrpc.Blob, _ *jsonrpc s.da.blobData[height] = append(s.da.blobData[height], b) } s.da.timestamps[height] = time.Now() + s.da.notifySubscribers(height) return height, nil } @@ -127,11 +129,39 @@ func (s *blobServer) GetCommitmentProof(_ context.Context, _ uint64, _ libshare. return &jsonrpc.CommitmentProof{}, nil } -// Subscribe returns a closed channel; LocalDA does not push live updates. -func (s *blobServer) Subscribe(_ context.Context, _ libshare.Namespace) (<-chan *jsonrpc.SubscriptionResponse, error) { - ch := make(chan *jsonrpc.SubscriptionResponse) - close(ch) - return ch, nil +// Subscribe streams blobs as they are included for the given namespace. +// The returned channel emits a SubscriptionResponse for every new DA block. +// The channel is closed when ctx is cancelled. +func (s *blobServer) Subscribe(ctx context.Context, namespace libshare.Namespace) (<-chan *jsonrpc.SubscriptionResponse, error) { + eventCh, subID := s.da.subscribe(namespace) + + out := make(chan *jsonrpc.SubscriptionResponse, 64) + go func() { + defer close(out) + defer s.da.unsubscribe(subID) + + for { + select { + case <-ctx.Done(): + return + case evt, ok := <-eventCh: + if !ok { + return + } + resp := &jsonrpc.SubscriptionResponse{ + Height: evt.height, + Blobs: evt.blobs, + } + select { + case out <- resp: + case <-ctx.Done(): + return + } + } + } + }() + + return out, nil } // startBlobServer starts an HTTP JSON-RPC server on addr serving the blob namespace.