From 24139bc2f627114df5df44037566e17a98960bf4 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Tue, 3 Mar 2026 10:15:31 +0100 Subject: [PATCH 01/17] feat: Replace the Syncer's polling DA worker with an event-driven DAFollower and introduce DA client subscription. --- apps/evm/server/force_inclusion_test.go | 7 + block/internal/da/client.go | 42 +++ block/internal/da/interface.go | 5 + block/internal/da/tracing.go | 3 + block/internal/da/tracing_test.go | 8 + block/internal/syncing/da_follower.go | 317 ++++++++++++++++++ block/internal/syncing/syncer.go | 134 ++------ block/internal/syncing/syncer_backoff_test.go | 233 ++++--------- .../internal/syncing/syncer_benchmark_test.go | 15 +- block/internal/syncing/syncer_test.go | 29 +- pkg/da/types/types.go | 7 + test/mocks/da.go | 68 ++++ test/testda/dummy.go | 5 + 13 files changed, 608 insertions(+), 265 deletions(-) create mode 100644 block/internal/syncing/da_follower.go diff --git a/apps/evm/server/force_inclusion_test.go b/apps/evm/server/force_inclusion_test.go index d04a356622..6be6e6ca76 100644 --- a/apps/evm/server/force_inclusion_test.go +++ b/apps/evm/server/force_inclusion_test.go @@ -50,6 +50,13 @@ func (m *mockDA) Get(ctx context.Context, ids []da.ID, namespace []byte) ([]da.B return nil, nil } +func (m *mockDA) Subscribe(_ context.Context, _ []byte) (<-chan da.SubscriptionEvent, error) { + // Not needed in these tests; return a closed channel. + ch := make(chan da.SubscriptionEvent) + close(ch) + return ch, nil +} + func (m *mockDA) Validate(ctx context.Context, ids []da.ID, proofs []da.Proof, namespace []byte) ([]bool, error) { return nil, nil } diff --git a/block/internal/da/client.go b/block/internal/da/client.go index a92a9eef28..35f604acab 100644 --- a/block/internal/da/client.go +++ b/block/internal/da/client.go @@ -350,6 +350,48 @@ func (c *client) HasForcedInclusionNamespace() bool { return c.hasForcedNamespace } +// Subscribe subscribes to blobs in the given namespace via the celestia-node +// Subscribe API. It returns a channel that emits a SubscriptionEvent for every +// DA block containing a matching blob. The channel is closed when ctx is +// cancelled. The caller must drain the channel after cancellation to avoid +// goroutine leaks. +func (c *client) Subscribe(ctx context.Context, namespace []byte) (<-chan datypes.SubscriptionEvent, error) { + ns, err := share.NewNamespaceFromBytes(namespace) + if err != nil { + return nil, fmt.Errorf("invalid namespace: %w", err) + } + + rawCh, err := c.blobAPI.Subscribe(ctx, ns) + if err != nil { + return nil, fmt.Errorf("blob subscribe: %w", err) + } + + out := make(chan datypes.SubscriptionEvent, 16) + go func() { + defer close(out) + for { + select { + case <-ctx.Done(): + return + case resp, ok := <-rawCh: + if !ok { + return + } + if resp == nil { + continue + } + select { + case out <- datypes.SubscriptionEvent{Height: resp.Height}: + case <-ctx.Done(): + return + } + } + } + }() + + return out, nil +} + // Get fetches blobs by their IDs. Used for visualization and fetching specific blobs. func (c *client) Get(ctx context.Context, ids []datypes.ID, namespace []byte) ([]datypes.Blob, error) { if len(ids) == 0 { diff --git a/block/internal/da/interface.go b/block/internal/da/interface.go index dd7a15d8f9..3cc0677580 100644 --- a/block/internal/da/interface.go +++ b/block/internal/da/interface.go @@ -17,6 +17,11 @@ type Client interface { // Get retrieves blobs by their IDs. Used for visualization and fetching specific blobs. Get(ctx context.Context, ids []datypes.ID, namespace []byte) ([]datypes.Blob, error) + // Subscribe returns a channel that emits one SubscriptionEvent per DA block + // that contains a blob in the given namespace. The channel is closed when ctx + // is cancelled. Callers MUST drain the channel after cancellation. + Subscribe(ctx context.Context, namespace []byte) (<-chan datypes.SubscriptionEvent, error) + // GetLatestDAHeight returns the latest height available on the DA layer. GetLatestDAHeight(ctx context.Context) (uint64, error) diff --git a/block/internal/da/tracing.go b/block/internal/da/tracing.go index 4d946a8b74..161293c2c3 100644 --- a/block/internal/da/tracing.go +++ b/block/internal/da/tracing.go @@ -145,6 +145,9 @@ func (t *tracedClient) GetForcedInclusionNamespace() []byte { func (t *tracedClient) HasForcedInclusionNamespace() bool { return t.inner.HasForcedInclusionNamespace() } +func (t *tracedClient) Subscribe(ctx context.Context, namespace []byte) (<-chan datypes.SubscriptionEvent, error) { + return t.inner.Subscribe(ctx, namespace) +} type submitError struct{ msg string } diff --git a/block/internal/da/tracing_test.go b/block/internal/da/tracing_test.go index de32532a31..e20bf3b84b 100644 --- a/block/internal/da/tracing_test.go +++ b/block/internal/da/tracing_test.go @@ -22,6 +22,14 @@ type mockFullClient struct { getFn func(ctx context.Context, ids []datypes.ID, namespace []byte) ([]datypes.Blob, error) getProofsFn func(ctx context.Context, ids []datypes.ID, namespace []byte) ([]datypes.Proof, error) validateFn func(ctx context.Context, ids []datypes.ID, proofs []datypes.Proof, namespace []byte) ([]bool, error) + subscribeFn func(ctx context.Context, namespace []byte) (<-chan datypes.SubscriptionEvent, error) +} + +func (m *mockFullClient) Subscribe(ctx context.Context, namespace []byte) (<-chan datypes.SubscriptionEvent, error) { + if m.subscribeFn == nil { + panic("not expected to be called") + } + return m.subscribeFn(ctx, namespace) } func (m *mockFullClient) Submit(ctx context.Context, data [][]byte, gasPrice float64, namespace []byte, options []byte) datypes.ResultSubmit { diff --git a/block/internal/syncing/da_follower.go b/block/internal/syncing/da_follower.go new file mode 100644 index 0000000000..27ec33b6fc --- /dev/null +++ b/block/internal/syncing/da_follower.go @@ -0,0 +1,317 @@ +package syncing + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "time" + + "github.com/rs/zerolog" + + "github.com/evstack/ev-node/block/internal/common" + "github.com/evstack/ev-node/block/internal/da" + datypes "github.com/evstack/ev-node/pkg/da/types" +) + +// DAFollower subscribes to DA blob events and drives sequential catchup. +// +// Architecture: +// - followLoop listens on the subscription channel and atomically updates +// highestSeenDAHeight. +// - catchupLoop sequentially retrieves from localDAHeight → highestSeenDAHeight, +// piping events to the Syncer's heightInCh. +// +// The two goroutines share only atomic state; no mutexes needed. +type DAFollower interface { + // Start begins the follow and catchup loops. + Start(ctx context.Context) error + // Stop cancels the context and waits for goroutines. + Stop() + // HasReachedHead returns true once the catchup loop has processed the + // DA head at least once. Once true, it stays true. + HasReachedHead() bool +} + +// daFollower is the concrete implementation of DAFollower. +type daFollower struct { + client da.Client + retriever DARetriever + logger zerolog.Logger + + // pipeEvent sends a DA height event to the syncer's processLoop. + pipeEvent func(ctx context.Context, event common.DAHeightEvent) error + + // Namespace to subscribe on (header namespace). + namespace []byte + + // localDAHeight is only written by catchupLoop and read by followLoop + // to determine whether a catchup is needed. + localDAHeight atomic.Uint64 + + // highestSeenDAHeight is written by followLoop and read by catchupLoop. + highestSeenDAHeight atomic.Uint64 + + // headReached tracks whether the follower has caught up to DA head. + headReached atomic.Bool + + // catchupSignal is sent by followLoop to wake catchupLoop when a new + // height is seen that is above localDAHeight. + catchupSignal chan struct{} + + // daBlockTime is used as a backoff before retrying after errors. + daBlockTime time.Duration + + // lifecycle + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup +} + +// DAFollowerConfig holds configuration for creating a DAFollower. +type DAFollowerConfig struct { + Client da.Client + Retriever DARetriever + Logger zerolog.Logger + PipeEvent func(ctx context.Context, event common.DAHeightEvent) error + Namespace []byte + StartDAHeight uint64 + DABlockTime time.Duration +} + +// NewDAFollower creates a new daFollower. +func NewDAFollower(cfg DAFollowerConfig) DAFollower { + f := &daFollower{ + client: cfg.Client, + retriever: cfg.Retriever, + logger: cfg.Logger.With().Str("component", "da_follower").Logger(), + pipeEvent: cfg.PipeEvent, + namespace: cfg.Namespace, + catchupSignal: make(chan struct{}, 1), + daBlockTime: cfg.DABlockTime, + } + f.localDAHeight.Store(cfg.StartDAHeight) + return f +} + +// Start begins the follow and catchup goroutines. +func (f *daFollower) Start(ctx context.Context) error { + f.ctx, f.cancel = context.WithCancel(ctx) + + f.wg.Add(2) + go f.followLoop() + go f.catchupLoop() + + f.logger.Info(). + Uint64("start_da_height", f.localDAHeight.Load()). + Msg("DA follower started") + return nil +} + +// Stop cancels and waits. +func (f *daFollower) Stop() { + if f.cancel != nil { + f.cancel() + } + f.wg.Wait() +} + +// HasReachedHead returns whether the DA head has been reached at least once. +func (f *daFollower) HasReachedHead() bool { + return f.headReached.Load() +} + +// signalCatchup sends a non-blocking signal to wake catchupLoop. +func (f *daFollower) signalCatchup() { + select { + case f.catchupSignal <- struct{}{}: + default: + // Already signaled, catchupLoop will pick up the new highestSeen. + } +} + +// followLoop subscribes to DA blob events and keeps highestSeenDAHeight up to date. +// When a new height appears above localDAHeight, it wakes the catchup loop. +func (f *daFollower) followLoop() { + defer f.wg.Done() + + f.logger.Info().Msg("starting follow loop") + defer f.logger.Info().Msg("follow loop stopped") + + for { + if err := f.runSubscription(); err != nil { + if f.ctx.Err() != nil { + return + } + f.logger.Warn().Err(err).Msg("DA subscription failed, reconnecting") + select { + case <-f.ctx.Done(): + return + case <-time.After(f.backoff()): + } + } + } +} + +// runSubscription opens a single subscription and processes events until the +// channel is closed or an error occurs. +func (f *daFollower) runSubscription() error { + ch, err := f.client.Subscribe(f.ctx, f.namespace) + if err != nil { + return err + } + + for { + select { + case <-f.ctx.Done(): + return f.ctx.Err() + case ev, ok := <-ch: + if !ok { + return errors.New("subscription channel closed") + } + f.updateHighest(ev.Height) + } + } +} + +// updateHighest atomically bumps highestSeenDAHeight and signals catchup if needed. +func (f *daFollower) updateHighest(height uint64) { + for { + cur := f.highestSeenDAHeight.Load() + if height <= cur { + return + } + if f.highestSeenDAHeight.CompareAndSwap(cur, height) { + f.logger.Debug(). + Uint64("da_height", height). + Msg("new highest DA height seen from subscription") + f.signalCatchup() + return + } + } +} + +// catchupLoop waits for signals and sequentially retrieves DA heights +// from localDAHeight up to highestSeenDAHeight. +func (f *daFollower) catchupLoop() { + defer f.wg.Done() + + f.logger.Info().Msg("starting catchup loop") + defer f.logger.Info().Msg("catchup loop stopped") + + for { + select { + case <-f.ctx.Done(): + return + case <-f.catchupSignal: + f.runCatchup() + } + } +} + +// runCatchup sequentially retrieves from localDAHeight to highestSeenDAHeight. +// It handles priority heights first, then sequential heights. +func (f *daFollower) runCatchup() { + for { + if f.ctx.Err() != nil { + return + } + + // Check for priority heights from P2P hints first. + if priorityHeight := f.retriever.PopPriorityHeight(); priorityHeight > 0 { + currentHeight := f.localDAHeight.Load() + if priorityHeight < currentHeight { + continue + } + f.logger.Debug(). + Uint64("da_height", priorityHeight). + Msg("fetching priority DA height from P2P hint") + if err := f.fetchAndPipeHeight(priorityHeight); err != nil { + if !f.waitOnCatchupError(err, priorityHeight) { + return + } + } + continue + } + + // Sequential catchup. + local := f.localDAHeight.Load() + highest := f.highestSeenDAHeight.Load() + + if local > highest { + // Caught up. + f.headReached.Store(true) + return + } + + if err := f.fetchAndPipeHeight(local); err != nil { + if !f.waitOnCatchupError(err, local) { + return + } + // Retry the same height after backoff. + continue + } + + // Advance local height. + f.localDAHeight.Store(local + 1) + } +} + +// fetchAndPipeHeight retrieves events at a single DA height and pipes them +// to the syncer. +func (f *daFollower) fetchAndPipeHeight(daHeight uint64) error { + events, err := f.retriever.RetrieveFromDA(f.ctx, daHeight) + if err != nil { + switch { + case errors.Is(err, datypes.ErrBlobNotFound): + // No blobs at this height — not an error, just skip. + return nil + case errors.Is(err, datypes.ErrHeightFromFuture): + // DA hasn't produced this height yet — mark head reached + // but return the error to trigger a backoff retry. + f.headReached.Store(true) + return err + default: + return err + } + } + + for _, event := range events { + if err := f.pipeEvent(f.ctx, event); err != nil { + return err + } + } + + return nil +} + +// errCaughtUp is a sentinel used to signal that the catchup loop has reached DA head. +var errCaughtUp = errors.New("caught up with DA head") + +// waitOnCatchupError logs the error and backs off before retrying. +// It returns true if the caller should continue (retry), or false if the +// catchup loop should exit (context cancelled or caught-up sentinel). +func (f *daFollower) waitOnCatchupError(err error, daHeight uint64) bool { + if errors.Is(err, errCaughtUp) { + f.logger.Debug().Uint64("da_height", daHeight).Msg("DA catchup reached head, waiting for subscription signal") + return false + } + if f.ctx.Err() != nil { + return false + } + f.logger.Warn().Err(err).Uint64("da_height", daHeight).Msg("catchup error, backing off") + select { + case <-f.ctx.Done(): + return false + case <-time.After(f.backoff()): + return true + } +} + +// backoff returns the configured DA block time or a sane default. +func (f *daFollower) backoff() time.Duration { + if f.daBlockTime > 0 { + return f.daBlockTime + } + return 2 * time.Second +} diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index ce96c23ef2..154cd927ec 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -21,7 +21,6 @@ import ( "github.com/evstack/ev-node/block/internal/da" coreexecutor "github.com/evstack/ev-node/core/execution" "github.com/evstack/ev-node/pkg/config" - datypes "github.com/evstack/ev-node/pkg/da/types" "github.com/evstack/ev-node/pkg/genesis" "github.com/evstack/ev-node/pkg/raft" "github.com/evstack/ev-node/pkg/store" @@ -80,6 +79,9 @@ type Syncer struct { p2pHandler p2pHandler raftRetriever *raftRetriever + // DA follower (replaces the old polling daWorkerLoop) + daFollower DAFollower + // Forced inclusion tracking forcedInclusionMu sync.RWMutex seenBlockTxs map[string]struct{} // SHA-256 hex of every tx seen in a DA-sourced block @@ -96,9 +98,6 @@ type Syncer struct { // P2P wait coordination p2pWaitState atomic.Value // stores p2pWaitState - // DA head-reached signal for recovery mode (stays true once DA head is seen) - daHeadReached atomic.Bool - // blockSyncer is the interface used for block sync operations. // defaults to self, but can be wrapped with tracing. blockSyncer BlockSyncer @@ -197,7 +196,20 @@ func (s *Syncer) Start(ctx context.Context) error { // Start main processing loop s.wg.Go(s.processLoop) - // Start dedicated workers for DA, and pending processing + // Start the DA follower (subscribe + catchup) and other workers + s.daFollower = NewDAFollower(DAFollowerConfig{ + Client: s.daClient, + Retriever: s.daRetriever, + Logger: s.logger, + PipeEvent: s.pipeEvent, + Namespace: s.daClient.GetHeaderNamespace(), + StartDAHeight: s.daRetrieverHeight.Load(), + DABlockTime: s.config.DA.BlockTime.Duration, + }) + if err := s.daFollower.Start(s.ctx); err != nil { + return fmt.Errorf("failed to start DA follower: %w", err) + } + s.startSyncWorkers() s.logger.Info().Msg("syncer started") @@ -212,6 +224,12 @@ func (s *Syncer) Stop() error { s.cancel() s.cancelP2PWait(0) + + // Stop the DA follower first (it owns its own goroutines). + if s.daFollower != nil { + s.daFollower.Stop() + } + s.wg.Wait() // Skip draining if we're shutting down due to a critical error (e.g. execution @@ -359,113 +377,23 @@ func (s *Syncer) processLoop() { } func (s *Syncer) startSyncWorkers() { - s.wg.Add(3) - go s.daWorkerLoop() + // DA follower is already started in Start(). + s.wg.Add(2) go s.pendingWorkerLoop() go s.p2pWorkerLoop() } -func (s *Syncer) daWorkerLoop() { - defer s.wg.Done() - - s.logger.Info().Msg("starting DA worker") - defer s.logger.Info().Msg("DA worker stopped") - - for { - err := s.fetchDAUntilCaughtUp() - - var backoff time.Duration - if err == nil { - // No error, means we are caught up. - s.daHeadReached.Store(true) - backoff = s.config.DA.BlockTime.Duration - } else { - // Error, back off for a shorter duration. - backoff = s.config.DA.BlockTime.Duration - if backoff <= 0 { - backoff = 2 * time.Second - } - } - - select { - case <-s.ctx.Done(): - return - case <-time.After(backoff): - } - } -} - -// HasReachedDAHead returns true once the DA worker has reached the DA head. +// HasReachedDAHead returns true once the DA follower has caught up to the DA head. // Once set, it stays true. func (s *Syncer) HasReachedDAHead() bool { - return s.daHeadReached.Load() -} - -func (s *Syncer) fetchDAUntilCaughtUp() error { - for { - select { - case <-s.ctx.Done(): - return s.ctx.Err() - default: - } - - // Check for priority heights from P2P hints first - var daHeight uint64 - if priorityHeight := s.daRetriever.PopPriorityHeight(); priorityHeight > 0 { - // Skip if we've already fetched past this height - currentHeight := s.daRetrieverHeight.Load() - if priorityHeight < currentHeight { - continue - } - daHeight = priorityHeight - s.logger.Debug().Uint64("da_height", daHeight).Msg("fetching priority DA height from P2P hint") - } else { - daHeight = max(s.daRetrieverHeight.Load(), s.cache.DaHeight()) - } - - events, err := s.daRetriever.RetrieveFromDA(s.ctx, daHeight) - if err != nil { - switch { - case errors.Is(err, datypes.ErrBlobNotFound): - s.daRetrieverHeight.Store(daHeight + 1) - continue // Fetch next height immediately - case errors.Is(err, datypes.ErrHeightFromFuture): - s.logger.Debug().Err(err).Uint64("da_height", daHeight).Msg("DA is ahead of local target; backing off future height requests") - return nil // Caught up - default: - s.logger.Error().Err(err).Uint64("da_height", daHeight).Msg("failed to retrieve from DA; backing off DA requests") - return err // Other errors - } - } - - if len(events) == 0 { - // This can happen if RetrieveFromDA returns no events and no error. - s.logger.Debug().Uint64("da_height", daHeight).Msg("no events returned from DA, but no error either.") - } - - // Process DA events - for _, event := range events { - if err := s.pipeEvent(s.ctx, event); err != nil { - return err - } - } - - // Update DA retrieval height on successful retrieval - // For priority fetches, only update if the priority height is ahead of current - // For sequential fetches, always increment - newHeight := daHeight + 1 - for { - current := s.daRetrieverHeight.Load() - if newHeight <= current { - break // Already at or past this height - } - if s.daRetrieverHeight.CompareAndSwap(current, newHeight) { - break - } - } + if s.daFollower != nil { + return s.daFollower.HasReachedHead() } + return false } +// fetchDAUntilCaughtUp was removed — the DAFollower handles this concern. + func (s *Syncer) pendingWorkerLoop() { defer s.wg.Done() diff --git a/block/internal/syncing/syncer_backoff_test.go b/block/internal/syncing/syncer_backoff_test.go index d6c6689f29..10564b4b28 100644 --- a/block/internal/syncing/syncer_backoff_test.go +++ b/block/internal/syncing/syncer_backoff_test.go @@ -7,27 +7,20 @@ import ( "testing/synctest" "time" - "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "github.com/evstack/ev-node/block/internal/cache" "github.com/evstack/ev-node/block/internal/common" - "github.com/evstack/ev-node/core/execution" - "github.com/evstack/ev-node/pkg/config" datypes "github.com/evstack/ev-node/pkg/da/types" "github.com/evstack/ev-node/pkg/genesis" - "github.com/evstack/ev-node/pkg/store" - extmocks "github.com/evstack/ev-node/test/mocks/external" "github.com/evstack/ev-node/types" ) -// TestSyncer_BackoffOnDAError verifies that the syncer implements proper backoff -// behavior when encountering different types of DA layer errors. -func TestSyncer_BackoffOnDAError(t *testing.T) { +// TestDAFollower_BackoffOnCatchupError verifies that the DAFollower implements +// proper backoff behavior when encountering different types of DA layer errors. +func TestDAFollower_BackoffOnCatchupError(t *testing.T) { tests := map[string]struct { daBlockTime time.Duration error error @@ -63,30 +56,25 @@ func TestSyncer_BackoffOnDAError(t *testing.T) { for name, tc := range tests { t.Run(name, func(t *testing.T) { synctest.Test(t, func(t *testing.T) { - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) defer cancel() - // Setup syncer - syncer := setupTestSyncer(t, tc.daBlockTime) - syncer.ctx = ctx - - // Setup mocks daRetriever := NewMockDARetriever(t) - p2pHandler := newMockp2pHandler(t) - p2pHandler.On("ProcessHeight", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() - syncer.daRetriever = daRetriever - syncer.p2pHandler = p2pHandler - p2pHandler.On("SetProcessedHeight", mock.Anything).Return().Maybe() - - // Mock PopPriorityHeight to always return 0 (no priority heights) daRetriever.On("PopPriorityHeight").Return(uint64(0)).Maybe() - // Create mock stores for P2P - mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) - mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() + pipeEvent := func(_ context.Context, _ common.DAHeightEvent) error { return nil } + + follower := NewDAFollower(DAFollowerConfig{ + Retriever: daRetriever, + Logger: zerolog.Nop(), + PipeEvent: pipeEvent, + Namespace: []byte("ns"), + StartDAHeight: 100, + DABlockTime: tc.daBlockTime, + }).(*daFollower) - mockDataStore := extmocks.NewMockStore[*types.Data](t) - mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() + follower.ctx, follower.cancel = context.WithCancel(ctx) + follower.highestSeenDAHeight.Store(102) var callTimes []time.Time callCount := 0 @@ -100,17 +88,14 @@ func TestSyncer_BackoffOnDAError(t *testing.T) { Return(nil, tc.error).Once() if tc.expectsBackoff { - // Second call should be delayed due to backoff daRetriever.On("RetrieveFromDA", mock.Anything, uint64(100)). Run(func(args mock.Arguments) { callTimes = append(callTimes, time.Now()) callCount++ - // Cancel to end test cancel() }). Return(nil, datypes.ErrBlobNotFound).Once() } else { - // For ErrBlobNotFound, DA height should increment daRetriever.On("RetrieveFromDA", mock.Anything, uint64(101)). Run(func(args mock.Arguments) { callTimes = append(callTimes, time.Now()) @@ -120,12 +105,9 @@ func TestSyncer_BackoffOnDAError(t *testing.T) { Return(nil, datypes.ErrBlobNotFound).Once() } - // Run sync loop - syncer.startSyncWorkers() + go follower.runCatchup() <-ctx.Done() - syncer.wg.Wait() - // Verify behavior if tc.expectsBackoff { require.Len(t, callTimes, 2, "should make exactly 2 calls with backoff") @@ -151,35 +133,32 @@ func TestSyncer_BackoffOnDAError(t *testing.T) { } } -// TestSyncer_BackoffResetOnSuccess verifies that backoff is properly reset -// after a successful DA retrieval, allowing the syncer to continue at normal speed. -func TestSyncer_BackoffResetOnSuccess(t *testing.T) { +// TestDAFollower_BackoffResetOnSuccess verifies that backoff is properly reset +// after a successful DA retrieval. +func TestDAFollower_BackoffResetOnSuccess(t *testing.T) { synctest.Test(t, func(t *testing.T) { - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) defer cancel() - syncer := setupTestSyncer(t, 1*time.Second) - syncer.ctx = ctx - addr, pub, signer := buildSyncTestSigner(t) - gen := syncer.genesis + gen := backoffTestGenesis(addr) daRetriever := NewMockDARetriever(t) - p2pHandler := newMockp2pHandler(t) - p2pHandler.On("ProcessHeight", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() - syncer.daRetriever = daRetriever - syncer.p2pHandler = p2pHandler - p2pHandler.On("SetProcessedHeight", mock.Anything).Return().Maybe() - - // Mock PopPriorityHeight to always return 0 (no priority heights) daRetriever.On("PopPriorityHeight").Return(uint64(0)).Maybe() - // Create mock stores for P2P - mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) - mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() + pipeEvent := func(_ context.Context, _ common.DAHeightEvent) error { return nil } + + follower := NewDAFollower(DAFollowerConfig{ + Retriever: daRetriever, + Logger: zerolog.Nop(), + PipeEvent: pipeEvent, + Namespace: []byte("ns"), + StartDAHeight: 100, + DABlockTime: 1 * time.Second, + }).(*daFollower) - mockDataStore := extmocks.NewMockStore[*types.Data](t) - mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() + follower.ctx, follower.cancel = context.WithCancel(ctx) + follower.highestSeenDAHeight.Store(105) var callTimes []time.Time @@ -190,7 +169,7 @@ func TestSyncer_BackoffResetOnSuccess(t *testing.T) { }). Return(nil, errors.New("temporary failure")).Once() - // Second call - success (should reset backoff and increment DA height) + // Second call - success _, header := makeSignedHeaderBytes(t, gen.ChainID, 1, addr, pub, signer, nil, nil, nil) data := &types.Data{ Metadata: &types.Metadata{ @@ -211,7 +190,7 @@ func TestSyncer_BackoffResetOnSuccess(t *testing.T) { }). Return([]common.DAHeightEvent{event}, nil).Once() - // Third call - should happen immediately after success (DA height incremented to 101) + // Third call - should happen immediately after success (DA height incremented) daRetriever.On("RetrieveFromDA", mock.Anything, uint64(101)). Run(func(args mock.Arguments) { callTimes = append(callTimes, time.Now()) @@ -219,137 +198,71 @@ func TestSyncer_BackoffResetOnSuccess(t *testing.T) { }). Return(nil, datypes.ErrBlobNotFound).Once() - // Start process loop to handle events - go syncer.processLoop() - - // Run workers - syncer.startSyncWorkers() + go follower.runCatchup() <-ctx.Done() - syncer.wg.Wait() require.Len(t, callTimes, 3, "should make exactly 3 calls") - // Verify backoff between first and second call delay1to2 := callTimes[1].Sub(callTimes[0]) assert.GreaterOrEqual(t, delay1to2, 1*time.Second, "should have backed off between error and success (got %v)", delay1to2) - // Verify no backoff between second and third call (backoff reset) delay2to3 := callTimes[2].Sub(callTimes[1]) assert.Less(t, delay2to3, 100*time.Millisecond, "should continue immediately after success (got %v)", delay2to3) }) } -// TestSyncer_BackoffBehaviorIntegration tests the complete backoff flow: -// error -> backoff delay -> recovery -> normal operation. -func TestSyncer_BackoffBehaviorIntegration(t *testing.T) { - // Test simpler backoff behavior: error -> backoff -> success -> continue +// TestDAFollower_CatchupThenReachHead verifies the catchup flow: +// sequential fetch from local → highest → mark head reached. +func TestDAFollower_CatchupThenReachHead(t *testing.T) { synctest.Test(t, func(t *testing.T) { - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) defer cancel() - syncer := setupTestSyncer(t, 500*time.Millisecond) - syncer.ctx = ctx - daRetriever := NewMockDARetriever(t) - p2pHandler := newMockp2pHandler(t) - p2pHandler.On("ProcessHeight", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() - syncer.daRetriever = daRetriever - syncer.p2pHandler = p2pHandler - - // Mock PopPriorityHeight to always return 0 (no priority heights) daRetriever.On("PopPriorityHeight").Return(uint64(0)).Maybe() - // Create mock stores for P2P - mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) - mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() - - mockDataStore := extmocks.NewMockStore[*types.Data](t) - mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() - - var callTimes []time.Time - p2pHandler.On("SetProcessedHeight", mock.Anything).Return().Maybe() - - // First call - error (triggers backoff) - daRetriever.On("RetrieveFromDA", mock.Anything, uint64(100)). - Run(func(args mock.Arguments) { - callTimes = append(callTimes, time.Now()) - }). - Return(nil, errors.New("network error")).Once() - - // Second call - should be delayed due to backoff - daRetriever.On("RetrieveFromDA", mock.Anything, uint64(100)). - Run(func(args mock.Arguments) { - callTimes = append(callTimes, time.Now()) - }). - Return(nil, datypes.ErrBlobNotFound).Once() - - // Third call - should continue without delay (DA height incremented) - daRetriever.On("RetrieveFromDA", mock.Anything, uint64(101)). - Run(func(args mock.Arguments) { - callTimes = append(callTimes, time.Now()) - cancel() - }). - Return(nil, datypes.ErrBlobNotFound).Once() - - go syncer.processLoop() - syncer.startSyncWorkers() - <-ctx.Done() - syncer.wg.Wait() - - require.Len(t, callTimes, 3, "should make exactly 3 calls") + pipeEvent := func(_ context.Context, _ common.DAHeightEvent) error { return nil } + + follower := NewDAFollower(DAFollowerConfig{ + Retriever: daRetriever, + Logger: zerolog.Nop(), + PipeEvent: pipeEvent, + Namespace: []byte("ns"), + StartDAHeight: 3, + DABlockTime: 500 * time.Millisecond, + }).(*daFollower) + + follower.ctx, follower.cancel = context.WithCancel(ctx) + follower.highestSeenDAHeight.Store(5) + + var fetchedHeights []uint64 + + for h := uint64(3); h <= 5; h++ { + h := h + daRetriever.On("RetrieveFromDA", mock.Anything, h). + Run(func(args mock.Arguments) { + fetchedHeights = append(fetchedHeights, h) + }). + Return(nil, datypes.ErrBlobNotFound).Once() + } - // First to second call should be delayed (backoff) - delay1to2 := callTimes[1].Sub(callTimes[0]) - assert.GreaterOrEqual(t, delay1to2, 500*time.Millisecond, - "should have backoff delay between first and second call (got %v)", delay1to2) + follower.runCatchup() - // Second to third call should be immediate (no backoff after ErrBlobNotFound) - delay2to3 := callTimes[2].Sub(callTimes[1]) - assert.Less(t, delay2to3, 100*time.Millisecond, - "should continue immediately after ErrBlobNotFound (got %v)", delay2to3) + assert.True(t, follower.HasReachedHead(), "should have reached DA head") + // Heights 3, 4, 5 processed; local now at 6 which > highest (5) → caught up + assert.Equal(t, []uint64{3, 4, 5}, fetchedHeights, "should have fetched heights sequentially") }) } -func setupTestSyncer(t *testing.T, daBlockTime time.Duration) *Syncer { - t.Helper() - - ds := dssync.MutexWrap(datastore.NewMapDatastore()) - st := store.New(ds) - - cm, err := cache.NewManager(config.DefaultConfig(), st, zerolog.Nop()) - require.NoError(t, err) - - addr, _, _ := buildSyncTestSigner(t) - - cfg := config.DefaultConfig() - cfg.DA.BlockTime.Duration = daBlockTime - - gen := genesis.Genesis{ +// backoffTestGenesis creates a test genesis for the backoff tests. +func backoffTestGenesis(addr []byte) genesis.Genesis { + return genesis.Genesis{ ChainID: "test-chain", InitialHeight: 1, - StartTime: time.Now().Add(-time.Hour), // Start in past + StartTime: time.Now().Add(-time.Hour), ProposerAddress: addr, DAStartHeight: 100, } - - syncer := NewSyncer( - st, - execution.NewDummyExecutor(), - nil, - cm, - common.NopMetrics(), - cfg, - gen, - extmocks.NewMockStore[*types.P2PSignedHeader](t), - extmocks.NewMockStore[*types.P2PData](t), - zerolog.Nop(), - common.DefaultBlockOptions(), - make(chan error, 1), - nil, - ) - - require.NoError(t, syncer.initializeState()) - return syncer } diff --git a/block/internal/syncing/syncer_benchmark_test.go b/block/internal/syncing/syncer_benchmark_test.go index 330afc2e53..063afd3c64 100644 --- a/block/internal/syncing/syncer_benchmark_test.go +++ b/block/internal/syncing/syncer_benchmark_test.go @@ -44,6 +44,20 @@ func BenchmarkSyncerIO(b *testing.B) { // run both loops go fixt.s.processLoop() + + // Create a DAFollower to drive DA retrieval. + follower := NewDAFollower(DAFollowerConfig{ + Retriever: fixt.s.daRetriever, + Logger: zerolog.Nop(), + PipeEvent: fixt.s.pipeEvent, + Namespace: []byte("ns"), + StartDAHeight: fixt.s.daRetrieverHeight.Load(), + DABlockTime: 0, + }).(*daFollower) + follower.ctx, follower.cancel = context.WithCancel(fixt.s.ctx) + follower.highestSeenDAHeight.Store(spec.heights + daHeightOffset) + go follower.runCatchup() + fixt.s.startSyncWorkers() require.Eventually(b, func() bool { @@ -60,7 +74,6 @@ func BenchmarkSyncerIO(b *testing.B) { } require.Len(b, fixt.s.heightInCh, 0) - assert.Equal(b, spec.heights+daHeightOffset, fixt.s.daRetrieverHeight) gotStoreHeight, err := fixt.s.store.Height(b.Context()) require.NoError(b, err) assert.Equal(b, spec.heights, gotStoreHeight) diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 5ffd607d5b..892dd1050a 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -416,12 +416,26 @@ func TestSyncLoopPersistState(t *testing.T) { Return(nil, datypes.ErrHeightFromFuture) go syncerInst1.processLoop() + + // Create and start a DAFollower so DA retrieval actually happens. + follower1 := NewDAFollower(DAFollowerConfig{ + Retriever: daRtrMock, + Logger: zerolog.Nop(), + PipeEvent: syncerInst1.pipeEvent, + Namespace: []byte("ns"), + StartDAHeight: syncerInst1.daRetrieverHeight.Load(), + DABlockTime: cfg.DA.BlockTime.Duration, + }).(*daFollower) + follower1.ctx, follower1.cancel = context.WithCancel(ctx) + // Set highest so catchup runs through all mocked heights. + follower1.highestSeenDAHeight.Store(myFutureDAHeight) + go follower1.runCatchup() syncerInst1.startSyncWorkers() syncerInst1.wg.Wait() requireEmptyChan(t, errorCh) t.Log("sync workers on instance1 completed") - require.Equal(t, myFutureDAHeight, syncerInst1.daRetrieverHeight.Load()) + require.Equal(t, myFutureDAHeight, follower1.localDAHeight.Load()) // wait for all events consumed require.NoError(t, cm.SaveToStore()) @@ -479,6 +493,19 @@ func TestSyncLoopPersistState(t *testing.T) { // when it starts, it should fetch from the last height it stopped at t.Log("sync workers on instance2 started") + + // Create a follower for instance 2. + follower2 := NewDAFollower(DAFollowerConfig{ + Retriever: daRtrMock, + Logger: zerolog.Nop(), + PipeEvent: syncerInst2.pipeEvent, + Namespace: []byte("ns"), + StartDAHeight: syncerInst2.daRetrieverHeight.Load(), + DABlockTime: cfg.DA.BlockTime.Duration, + }).(*daFollower) + follower2.ctx, follower2.cancel = context.WithCancel(ctx) + follower2.highestSeenDAHeight.Store(syncerInst2.daRetrieverHeight.Load() + 1) + go follower2.runCatchup() syncerInst2.startSyncWorkers() syncerInst2.wg.Wait() diff --git a/pkg/da/types/types.go b/pkg/da/types/types.go index b2f2e7bc30..7f48fc43ef 100644 --- a/pkg/da/types/types.go +++ b/pkg/da/types/types.go @@ -82,3 +82,10 @@ func SplitID(id []byte) (uint64, []byte, error) { commitment := id[8:] return binary.LittleEndian.Uint64(id[:8]), commitment, nil } + +// SubscriptionEvent is a namespace-agnostic signal that a blob was finalized at +// Height on the DA layer. Produced by Subscribe and consumed by DA followers. +type SubscriptionEvent struct { + // Height is the DA layer height at which the blob was finalized. + Height uint64 +} diff --git a/test/mocks/da.go b/test/mocks/da.go index f5293d907a..27ce9badf3 100644 --- a/test/mocks/da.go +++ b/test/mocks/da.go @@ -492,6 +492,74 @@ func (_c *MockClient_Submit_Call) RunAndReturn(run func(ctx context.Context, dat return _c } +// Subscribe provides a mock function for the type MockClient +func (_mock *MockClient) Subscribe(ctx context.Context, namespace []byte) (<-chan da.SubscriptionEvent, error) { + ret := _mock.Called(ctx, namespace) + + if len(ret) == 0 { + panic("no return value specified for Subscribe") + } + + var r0 <-chan da.SubscriptionEvent + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, []byte) (<-chan da.SubscriptionEvent, error)); ok { + return returnFunc(ctx, namespace) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, []byte) <-chan da.SubscriptionEvent); ok { + r0 = returnFunc(ctx, namespace) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan da.SubscriptionEvent) + } + } + if returnFunc, ok := ret.Get(1).(func(context.Context, []byte) error); ok { + r1 = returnFunc(ctx, namespace) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockClient_Subscribe_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Subscribe' +type MockClient_Subscribe_Call struct { + *mock.Call +} + +// Subscribe is a helper method to define mock.On call +// - ctx context.Context +// - namespace []byte +func (_e *MockClient_Expecter) Subscribe(ctx interface{}, namespace interface{}) *MockClient_Subscribe_Call { + return &MockClient_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, namespace)} +} + +func (_c *MockClient_Subscribe_Call) Run(run func(ctx context.Context, namespace []byte)) *MockClient_Subscribe_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 []byte + if args[1] != nil { + arg1 = args[1].([]byte) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockClient_Subscribe_Call) Return(subscriptionEventCh <-chan da.SubscriptionEvent, err error) *MockClient_Subscribe_Call { + _c.Call.Return(subscriptionEventCh, err) + return _c +} + +func (_c *MockClient_Subscribe_Call) RunAndReturn(run func(ctx context.Context, namespace []byte) (<-chan da.SubscriptionEvent, error)) *MockClient_Subscribe_Call { + _c.Call.Return(run) + return _c +} + // NewMockVerifier creates a new instance of MockVerifier. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewMockVerifier(t interface { diff --git a/test/testda/dummy.go b/test/testda/dummy.go index 648021b76a..360bbf686f 100644 --- a/test/testda/dummy.go +++ b/test/testda/dummy.go @@ -42,6 +42,11 @@ type DummyDA struct { tickerStop chan struct{} } +func (d *DummyDA) Subscribe(ctx context.Context, namespace []byte) (<-chan datypes.SubscriptionEvent, error) { + //TODO implement me + panic("implement me") +} + // Option configures a DummyDA instance. type Option func(*DummyDA) From 2ae32fbfc6320cb7d86f1b96f507c729bc1aced5 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Tue, 3 Mar 2026 13:29:49 +0100 Subject: [PATCH 02/17] feat: add inline blob processing to DAFollower for zero-latency follow mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the DA subscription delivers blobs at the current local DA height, the followLoop now processes them inline via ProcessBlobs — avoiding a round-trip re-fetch from the DA layer. Architecture: - followLoop: processes subscription blobs inline when caught up (fast path), falls through to catchupLoop when behind (slow path). - catchupLoop: unchanged — sequential RetrieveFromDA() for bulk sync. Changes: - Add Blobs field to SubscriptionEvent for carrying raw blob data - Add extractBlobData() to DA client Subscribe adapter - Export ProcessBlobs on DARetriever interface - Add handleSubscriptionEvent() to DAFollower with inline fast path - Add TestDAFollower_InlineProcessing with 3 sub-tests --- block/internal/da/client.go | 25 ++++- block/internal/syncing/da_follower.go | 37 +++++- block/internal/syncing/da_retriever.go | 9 ++ block/internal/syncing/da_retriever_mock.go | 65 +++++++++++ .../internal/syncing/da_retriever_tracing.go | 4 + .../syncing/da_retriever_tracing_test.go | 4 + block/internal/syncing/syncer_backoff_test.go | 106 ++++++++++++++++++ pkg/da/types/types.go | 3 + 8 files changed, 249 insertions(+), 4 deletions(-) diff --git a/block/internal/da/client.go b/block/internal/da/client.go index 35f604acab..aa2a5be6ae 100644 --- a/block/internal/da/client.go +++ b/block/internal/da/client.go @@ -381,7 +381,10 @@ func (c *client) Subscribe(ctx context.Context, namespace []byte) (<-chan datype continue } select { - case out <- datypes.SubscriptionEvent{Height: resp.Height}: + case out <- datypes.SubscriptionEvent{ + Height: resp.Height, + Blobs: extractBlobData(resp), + }: case <-ctx.Done(): return } @@ -392,6 +395,26 @@ func (c *client) Subscribe(ctx context.Context, namespace []byte) (<-chan datype return out, nil } +// extractBlobData extracts raw byte slices from a subscription response, +// filtering out nil blobs and empty data. +func extractBlobData(resp *blobrpc.SubscriptionResponse) [][]byte { + if resp == nil || len(resp.Blobs) == 0 { + return nil + } + blobs := make([][]byte, 0, len(resp.Blobs)) + for _, blob := range resp.Blobs { + if blob == nil { + continue + } + data := blob.Data() + if len(data) == 0 { + continue + } + blobs = append(blobs, data) + } + return blobs +} + // Get fetches blobs by their IDs. Used for visualization and fetching specific blobs. func (c *client) Get(ctx context.Context, ids []datypes.ID, namespace []byte) ([]datypes.Blob, error) { if len(ids) == 0 { diff --git a/block/internal/syncing/da_follower.go b/block/internal/syncing/da_follower.go index 27ec33b6fc..e9f95a1666 100644 --- a/block/internal/syncing/da_follower.go +++ b/block/internal/syncing/da_follower.go @@ -17,8 +17,9 @@ import ( // DAFollower subscribes to DA blob events and drives sequential catchup. // // Architecture: -// - followLoop listens on the subscription channel and atomically updates -// highestSeenDAHeight. +// - followLoop listens on the subscription channel. When caught up, it processes +// subscription blobs inline (fast path, no DA re-fetch). Otherwise, it updates +// highestSeenDAHeight and signals the catchup loop. // - catchupLoop sequentially retrieves from localDAHeight → highestSeenDAHeight, // piping events to the Syncer's heightInCh. // @@ -169,11 +170,41 @@ func (f *daFollower) runSubscription() error { if !ok { return errors.New("subscription channel closed") } - f.updateHighest(ev.Height) + f.handleSubscriptionEvent(ev) } } } +// handleSubscriptionEvent processes a subscription event. When the follower is +// caught up (ev.Height == localDAHeight) and blobs are available, it processes +// them inline — avoiding a DA re-fetch round trip. Otherwise, it just updates +// highestSeenDAHeight and lets catchupLoop handle retrieval. +func (f *daFollower) handleSubscriptionEvent(ev datypes.SubscriptionEvent) { + // Always record the highest height we've seen from the subscription. + f.updateHighest(ev.Height) + + // Fast path: process blobs inline when caught up. + // Only fire when ev.Height == localDAHeight to avoid out-of-order processing. + if len(ev.Blobs) > 0 && ev.Height == f.localDAHeight.Load() { + events := f.retriever.ProcessBlobs(f.ctx, ev.Blobs, ev.Height) + for _, event := range events { + if err := f.pipeEvent(f.ctx, event); err != nil { + f.logger.Warn().Err(err).Uint64("da_height", ev.Height). + Msg("failed to pipe inline event, catchup will retry") + return // catchupLoop already signaled via updateHighest + } + } + // Advance local height — we processed this height inline. + f.localDAHeight.Store(ev.Height + 1) + f.headReached.Store(true) + f.logger.Debug().Uint64("da_height", ev.Height).Int("events", len(events)). + Msg("processed subscription blobs inline (fast path)") + return + } + + // Slow path: behind or no blobs — catchupLoop will handle via signal from updateHighest. +} + // updateHighest atomically bumps highestSeenDAHeight and signals catchup if needed. func (f *daFollower) updateHighest(height uint64) { for { diff --git a/block/internal/syncing/da_retriever.go b/block/internal/syncing/da_retriever.go index a6c9d43c7c..177ff98628 100644 --- a/block/internal/syncing/da_retriever.go +++ b/block/internal/syncing/da_retriever.go @@ -24,6 +24,9 @@ import ( type DARetriever interface { // RetrieveFromDA retrieves blocks from the specified DA height and returns height events RetrieveFromDA(ctx context.Context, daHeight uint64) ([]common.DAHeightEvent, error) + // ProcessBlobs parses raw blob bytes at a given DA height into height events. + // Used by the DAFollower to process subscription blobs inline without re-fetching. + ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent // QueuePriorityHeight queues a DA height for priority retrieval (from P2P hints). // These heights take precedence over sequential fetching. QueuePriorityHeight(daHeight uint64) @@ -191,6 +194,12 @@ func (r *daRetriever) validateBlobResponse(res datypes.ResultRetrieve, daHeight } } +// ProcessBlobs processes raw blob bytes to extract headers and data and returns height events. +// This is the public interface used by the DAFollower for inline subscription processing. +func (r *daRetriever) ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { + return r.processBlobs(ctx, blobs, daHeight) +} + // processBlobs processes retrieved blobs to extract headers and data and returns height events func (r *daRetriever) processBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { // Decode all blobs diff --git a/block/internal/syncing/da_retriever_mock.go b/block/internal/syncing/da_retriever_mock.go index 2e191c8851..dc6926485e 100644 --- a/block/internal/syncing/da_retriever_mock.go +++ b/block/internal/syncing/da_retriever_mock.go @@ -189,3 +189,68 @@ func (_c *MockDARetriever_RetrieveFromDA_Call) RunAndReturn(run func(ctx context _c.Call.Return(run) return _c } + +// ProcessBlobs provides a mock function for the type MockDARetriever +func (_mock *MockDARetriever) ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { + ret := _mock.Called(ctx, blobs, daHeight) + + if len(ret) == 0 { + panic("no return value specified for ProcessBlobs") + } + + var r0 []common.DAHeightEvent + if returnFunc, ok := ret.Get(0).(func(context.Context, [][]byte, uint64) []common.DAHeightEvent); ok { + r0 = returnFunc(ctx, blobs, daHeight) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]common.DAHeightEvent) + } + } + return r0 +} + +// MockDARetriever_ProcessBlobs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ProcessBlobs' +type MockDARetriever_ProcessBlobs_Call struct { + *mock.Call +} + +// ProcessBlobs is a helper method to define mock.On call +// - ctx context.Context +// - blobs [][]byte +// - daHeight uint64 +func (_e *MockDARetriever_Expecter) ProcessBlobs(ctx interface{}, blobs interface{}, daHeight interface{}) *MockDARetriever_ProcessBlobs_Call { + return &MockDARetriever_ProcessBlobs_Call{Call: _e.mock.On("ProcessBlobs", ctx, blobs, daHeight)} +} + +func (_c *MockDARetriever_ProcessBlobs_Call) Run(run func(ctx context.Context, blobs [][]byte, daHeight uint64)) *MockDARetriever_ProcessBlobs_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 [][]byte + if args[1] != nil { + arg1 = args[1].([][]byte) + } + var arg2 uint64 + if args[2] != nil { + arg2 = args[2].(uint64) + } + run( + arg0, + arg1, + arg2, + ) + }) + return _c +} + +func (_c *MockDARetriever_ProcessBlobs_Call) Return(dAHeightEvents []common.DAHeightEvent) *MockDARetriever_ProcessBlobs_Call { + _c.Call.Return(dAHeightEvents) + return _c +} + +func (_c *MockDARetriever_ProcessBlobs_Call) RunAndReturn(run func(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent) *MockDARetriever_ProcessBlobs_Call { + _c.Call.Return(run) + return _c +} diff --git a/block/internal/syncing/da_retriever_tracing.go b/block/internal/syncing/da_retriever_tracing.go index 2bc7a4094d..1e1a9ea7c0 100644 --- a/block/internal/syncing/da_retriever_tracing.go +++ b/block/internal/syncing/da_retriever_tracing.go @@ -63,3 +63,7 @@ func (t *tracedDARetriever) QueuePriorityHeight(daHeight uint64) { func (t *tracedDARetriever) PopPriorityHeight() uint64 { return t.inner.PopPriorityHeight() } + +func (t *tracedDARetriever) ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { + return t.inner.ProcessBlobs(ctx, blobs, daHeight) +} diff --git a/block/internal/syncing/da_retriever_tracing_test.go b/block/internal/syncing/da_retriever_tracing_test.go index 99ce1eb639..930fc43617 100644 --- a/block/internal/syncing/da_retriever_tracing_test.go +++ b/block/internal/syncing/da_retriever_tracing_test.go @@ -31,6 +31,10 @@ func (m *mockDARetriever) QueuePriorityHeight(daHeight uint64) {} func (m *mockDARetriever) PopPriorityHeight() uint64 { return 0 } +func (m *mockDARetriever) ProcessBlobs(_ context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { + return nil +} + func setupDARetrieverTrace(t *testing.T, inner DARetriever) (DARetriever, *tracetest.SpanRecorder) { t.Helper() sr := tracetest.NewSpanRecorder() diff --git a/block/internal/syncing/syncer_backoff_test.go b/block/internal/syncing/syncer_backoff_test.go index 10564b4b28..58327c0819 100644 --- a/block/internal/syncing/syncer_backoff_test.go +++ b/block/internal/syncing/syncer_backoff_test.go @@ -256,6 +256,112 @@ func TestDAFollower_CatchupThenReachHead(t *testing.T) { }) } +// TestDAFollower_InlineProcessing verifies the fast path: when the subscription +// delivers blobs at the current localDAHeight, handleSubscriptionEvent processes +// them inline via ProcessBlobs (not RetrieveFromDA). +func TestDAFollower_InlineProcessing(t *testing.T) { + t.Run("processes_blobs_inline_when_caught_up", func(t *testing.T) { + daRetriever := NewMockDARetriever(t) + + var pipedEvents []common.DAHeightEvent + pipeEvent := func(_ context.Context, ev common.DAHeightEvent) error { + pipedEvents = append(pipedEvents, ev) + return nil + } + + follower := NewDAFollower(DAFollowerConfig{ + Retriever: daRetriever, + Logger: zerolog.Nop(), + PipeEvent: pipeEvent, + Namespace: []byte("ns"), + StartDAHeight: 10, + DABlockTime: 500 * time.Millisecond, + }).(*daFollower) + + follower.ctx, follower.cancel = context.WithCancel(t.Context()) + defer follower.cancel() + + blobs := [][]byte{[]byte("header-blob"), []byte("data-blob")} + expectedEvents := []common.DAHeightEvent{ + {DaHeight: 10, Source: common.SourceDA}, + } + + // ProcessBlobs should be called (not RetrieveFromDA) + daRetriever.On("ProcessBlobs", mock.Anything, blobs, uint64(10)). + Return(expectedEvents).Once() + + // Simulate subscription event at the current localDAHeight + follower.handleSubscriptionEvent(datypes.SubscriptionEvent{ + Height: 10, + Blobs: blobs, + }) + + // Verify: ProcessBlobs was called, events were piped, height advanced + require.Len(t, pipedEvents, 1, "should pipe 1 event from inline processing") + assert.Equal(t, uint64(10), pipedEvents[0].DaHeight) + assert.Equal(t, uint64(11), follower.localDAHeight.Load(), "localDAHeight should advance past processed height") + assert.True(t, follower.HasReachedHead(), "should mark head as reached after inline processing") + }) + + t.Run("falls_through_to_catchup_when_behind", func(t *testing.T) { + daRetriever := NewMockDARetriever(t) + + pipeEvent := func(_ context.Context, _ common.DAHeightEvent) error { return nil } + + follower := NewDAFollower(DAFollowerConfig{ + Retriever: daRetriever, + Logger: zerolog.Nop(), + PipeEvent: pipeEvent, + Namespace: []byte("ns"), + StartDAHeight: 10, + DABlockTime: 500 * time.Millisecond, + }).(*daFollower) + + follower.ctx, follower.cancel = context.WithCancel(t.Context()) + defer follower.cancel() + + // Subscription reports height 15 but local is at 10 — should NOT process inline + follower.handleSubscriptionEvent(datypes.SubscriptionEvent{ + Height: 15, + Blobs: [][]byte{[]byte("blob")}, + }) + + // ProcessBlobs should NOT have been called + daRetriever.AssertNotCalled(t, "ProcessBlobs", mock.Anything, mock.Anything, mock.Anything) + assert.Equal(t, uint64(10), follower.localDAHeight.Load(), "localDAHeight should not change") + assert.Equal(t, uint64(15), follower.highestSeenDAHeight.Load(), "highestSeen should be updated") + }) + + t.Run("falls_through_when_no_blobs", func(t *testing.T) { + daRetriever := NewMockDARetriever(t) + + pipeEvent := func(_ context.Context, _ common.DAHeightEvent) error { return nil } + + follower := NewDAFollower(DAFollowerConfig{ + Retriever: daRetriever, + Logger: zerolog.Nop(), + PipeEvent: pipeEvent, + Namespace: []byte("ns"), + StartDAHeight: 10, + DABlockTime: 500 * time.Millisecond, + }).(*daFollower) + + follower.ctx, follower.cancel = context.WithCancel(t.Context()) + defer follower.cancel() + + // Subscription at current height but no blobs — should fall through + follower.handleSubscriptionEvent(datypes.SubscriptionEvent{ + Height: 10, + Blobs: nil, + }) + + // ProcessBlobs should NOT have been called + daRetriever.AssertNotCalled(t, "ProcessBlobs", mock.Anything, mock.Anything, mock.Anything) + assert.Equal(t, uint64(10), follower.localDAHeight.Load(), "localDAHeight should not change") + assert.Equal(t, uint64(10), follower.highestSeenDAHeight.Load(), "highestSeen should be updated") + }) +} + // backoffTestGenesis creates a test genesis for the backoff tests. func backoffTestGenesis(addr []byte) genesis.Genesis { return genesis.Genesis{ diff --git a/pkg/da/types/types.go b/pkg/da/types/types.go index 7f48fc43ef..8c11ce5cb5 100644 --- a/pkg/da/types/types.go +++ b/pkg/da/types/types.go @@ -88,4 +88,7 @@ func SplitID(id []byte) (uint64, []byte, error) { type SubscriptionEvent struct { // Height is the DA layer height at which the blob was finalized. Height uint64 + // Blobs contains the raw blob data from the subscription response. + // When non-nil, followers can process blobs inline without re-fetching from DA. + Blobs [][]byte } From 72abe51c9b8746c67f4f4ec3494ddd2de6563bc3 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Tue, 3 Mar 2026 13:55:14 +0100 Subject: [PATCH 03/17] feat: subscribe to both header and data namespaces for inline processing When header and data use different DA namespaces, the DAFollower now subscribes to both and merges events via a fan-in goroutine. This ensures inline blob processing works correctly for split-namespace configurations. Changes: - Add DataNamespace to DAFollowerConfig and daFollower - Subscribe to both namespaces in runSubscription with mergeSubscriptions fan-in - Guard handleSubscriptionEvent to only advance localDAHeight when ProcessBlobs returns at least one complete event (header+data matched) - Pass DataNamespace from syncer.go - Implement Subscribe on DummyDA test helper with subscriber notification --- block/internal/syncing/da_follower.go | 78 +++++++++++++++++++++++---- block/internal/syncing/syncer.go | 1 + test/testda/dummy.go | 61 +++++++++++++++++++-- 3 files changed, 128 insertions(+), 12 deletions(-) diff --git a/block/internal/syncing/da_follower.go b/block/internal/syncing/da_follower.go index e9f95a1666..cfadf36cb8 100644 --- a/block/internal/syncing/da_follower.go +++ b/block/internal/syncing/da_follower.go @@ -1,8 +1,10 @@ package syncing import ( + "bytes" "context" "errors" + "fmt" "sync" "sync/atomic" "time" @@ -45,6 +47,9 @@ type daFollower struct { // Namespace to subscribe on (header namespace). namespace []byte + // dataNamespace is the data namespace (may equal namespace when header+data + // share the same namespace). When different, we subscribe to both and merge. + dataNamespace []byte // localDAHeight is only written by catchupLoop and read by followLoop // to determine whether a catchup is needed. @@ -76,18 +81,24 @@ type DAFollowerConfig struct { Logger zerolog.Logger PipeEvent func(ctx context.Context, event common.DAHeightEvent) error Namespace []byte + DataNamespace []byte // may be nil or equal to Namespace StartDAHeight uint64 DABlockTime time.Duration } // NewDAFollower creates a new daFollower. func NewDAFollower(cfg DAFollowerConfig) DAFollower { + dataNs := cfg.DataNamespace + if len(dataNs) == 0 { + dataNs = cfg.Namespace + } f := &daFollower{ client: cfg.Client, retriever: cfg.Retriever, logger: cfg.Logger.With().Str("component", "da_follower").Logger(), pipeEvent: cfg.PipeEvent, namespace: cfg.Namespace, + dataNamespace: dataNs, catchupSignal: make(chan struct{}, 1), daBlockTime: cfg.DABlockTime, } @@ -154,12 +165,22 @@ func (f *daFollower) followLoop() { } } -// runSubscription opens a single subscription and processes events until the -// channel is closed or an error occurs. +// runSubscription opens subscriptions on both header and data namespaces (if +// different) and processes events until a channel is closed or an error occurs. func (f *daFollower) runSubscription() error { - ch, err := f.client.Subscribe(f.ctx, f.namespace) + headerCh, err := f.client.Subscribe(f.ctx, f.namespace) if err != nil { - return err + return fmt.Errorf("subscribe header namespace: %w", err) + } + + // If namespaces differ, subscribe to the data namespace too and fan-in. + ch := headerCh + if !bytes.Equal(f.namespace, f.dataNamespace) { + dataCh, err := f.client.Subscribe(f.ctx, f.dataNamespace) + if err != nil { + return fmt.Errorf("subscribe data namespace: %w", err) + } + ch = f.mergeSubscriptions(headerCh, dataCh) } for { @@ -175,6 +196,41 @@ func (f *daFollower) runSubscription() error { } } +// mergeSubscriptions fans two subscription channels into one, concatenating +// blobs from both namespaces when events arrive at the same DA height. +func (f *daFollower) mergeSubscriptions( + headerCh, dataCh <-chan datypes.SubscriptionEvent, +) <-chan datypes.SubscriptionEvent { + out := make(chan datypes.SubscriptionEvent, 16) + go func() { + defer close(out) + for headerCh != nil || dataCh != nil { + var ev datypes.SubscriptionEvent + var ok bool + select { + case <-f.ctx.Done(): + return + case ev, ok = <-headerCh: + if !ok { + headerCh = nil + continue + } + case ev, ok = <-dataCh: + if !ok { + dataCh = nil + continue + } + } + select { + case out <- ev: + case <-f.ctx.Done(): + return + } + } + }() + return out +} + // handleSubscriptionEvent processes a subscription event. When the follower is // caught up (ev.Height == localDAHeight) and blobs are available, it processes // them inline — avoiding a DA re-fetch round trip. Otherwise, it just updates @@ -194,11 +250,15 @@ func (f *daFollower) handleSubscriptionEvent(ev datypes.SubscriptionEvent) { return // catchupLoop already signaled via updateHighest } } - // Advance local height — we processed this height inline. - f.localDAHeight.Store(ev.Height + 1) - f.headReached.Store(true) - f.logger.Debug().Uint64("da_height", ev.Height).Int("events", len(events)). - Msg("processed subscription blobs inline (fast path)") + // Only advance if we produced at least one complete event. + // With split namespaces, the first namespace's blobs may not produce + // events until the second namespace's blobs arrive at the same height. + if len(events) != 0 { + f.localDAHeight.Store(ev.Height + 1) + f.headReached.Store(true) + f.logger.Debug().Uint64("da_height", ev.Height).Int("events", len(events)). + Msg("processed subscription blobs inline (fast path)") + } return } diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index 154cd927ec..b5f429090a 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -203,6 +203,7 @@ func (s *Syncer) Start(ctx context.Context) error { Logger: s.logger, PipeEvent: s.pipeEvent, Namespace: s.daClient.GetHeaderNamespace(), + DataNamespace: s.daClient.GetDataNamespace(), StartDAHeight: s.daRetrieverHeight.Load(), DABlockTime: s.config.DA.BlockTime.Duration, }) diff --git a/test/testda/dummy.go b/test/testda/dummy.go index 360bbf686f..7f29140b28 100644 --- a/test/testda/dummy.go +++ b/test/testda/dummy.go @@ -28,6 +28,12 @@ func (h *Header) Time() time.Time { return h.Timestamp } +// subscriber holds a channel and the context for a single Subscribe caller. +type subscriber struct { + ch chan datypes.SubscriptionEvent + ctx context.Context +} + // DummyDA is a test implementation of the DA client interface. // It supports blob storage, height simulation, failure injection, and header retrieval. type DummyDA struct { @@ -38,13 +44,50 @@ type DummyDA struct { headers map[uint64]*Header // height -> header (with timestamp) failSubmit atomic.Bool + // subscribers tracks active Subscribe callers. + subscribers []*subscriber + tickerMu sync.Mutex tickerStop chan struct{} } -func (d *DummyDA) Subscribe(ctx context.Context, namespace []byte) (<-chan datypes.SubscriptionEvent, error) { - //TODO implement me - panic("implement me") +// Subscribe returns a channel that emits a SubscriptionEvent for every new DA +// height produced by Submit or StartHeightTicker. The channel is closed when +// ctx is cancelled or Reset is called. +func (d *DummyDA) Subscribe(ctx context.Context, _ []byte) (<-chan datypes.SubscriptionEvent, error) { + ch := make(chan datypes.SubscriptionEvent, 64) + sub := &subscriber{ch: ch, ctx: ctx} + + d.mu.Lock() + d.subscribers = append(d.subscribers, sub) + d.mu.Unlock() + + // Remove subscriber and close channel when ctx is cancelled. + go func() { + <-ctx.Done() + d.mu.Lock() + defer d.mu.Unlock() + for i, s := range d.subscribers { + if s == sub { + d.subscribers = append(d.subscribers[:i], d.subscribers[i+1:]...) + break + } + } + close(ch) + }() + + return ch, nil +} + +// notifySubscribers sends an event to all active subscribers. Must be called +// with d.mu held. +func (d *DummyDA) notifySubscribers(ev datypes.SubscriptionEvent) { + for _, sub := range d.subscribers { + select { + case sub.ch <- ev: + case <-sub.ctx.Done(): + } + } } // Option configures a DummyDA instance. @@ -116,6 +159,10 @@ func (d *DummyDA) Submit(_ context.Context, data [][]byte, _ float64, namespace Timestamp: now, } } + d.notifySubscribers(datypes.SubscriptionEvent{ + Height: height, + Blobs: data, + }) d.mu.Unlock() return datypes.ResultSubmit{ @@ -258,6 +305,9 @@ func (d *DummyDA) StartHeightTicker(interval time.Duration) func() { Timestamp: now, } } + d.notifySubscribers(datypes.SubscriptionEvent{ + Height: height, + }) d.mu.Unlock() case <-stopCh: return @@ -282,6 +332,11 @@ func (d *DummyDA) Reset() { d.blobs = make(map[uint64]map[string][][]byte) d.headers = make(map[uint64]*Header) d.failSubmit.Store(false) + // Close all subscriber channels. + for _, sub := range d.subscribers { + close(sub.ch) + } + d.subscribers = nil d.mu.Unlock() d.tickerMu.Lock() From 24fe982c540fb6e9471e295876efe3c0f20d307e Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Tue, 3 Mar 2026 14:45:25 +0100 Subject: [PATCH 04/17] feat: add subscription watchdog to detect stalled DA subscriptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If no subscription events arrive within 3× the DA block time (default 30s), the watchdog triggers and returns an error. The followLoop then reconnects the subscription with the standard backoff. This prevents the node from silently stopping sync when the DA subscription stalls (e.g., network partition, DA node freeze). --- block/internal/syncing/da_follower.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/block/internal/syncing/da_follower.go b/block/internal/syncing/da_follower.go index cfadf36cb8..5b97acfb5b 100644 --- a/block/internal/syncing/da_follower.go +++ b/block/internal/syncing/da_follower.go @@ -167,6 +167,8 @@ func (f *daFollower) followLoop() { // runSubscription opens subscriptions on both header and data namespaces (if // different) and processes events until a channel is closed or an error occurs. +// A watchdog timer triggers if no events arrive within watchdogTimeout(), +// causing a reconnect. func (f *daFollower) runSubscription() error { headerCh, err := f.client.Subscribe(f.ctx, f.namespace) if err != nil { @@ -183,6 +185,10 @@ func (f *daFollower) runSubscription() error { ch = f.mergeSubscriptions(headerCh, dataCh) } + watchdogTimeout := f.watchdogTimeout() + watchdog := time.NewTimer(watchdogTimeout) + defer watchdog.Stop() + for { select { case <-f.ctx.Done(): @@ -192,6 +198,9 @@ func (f *daFollower) runSubscription() error { return errors.New("subscription channel closed") } f.handleSubscriptionEvent(ev) + watchdog.Reset(watchdogTimeout) + case <-watchdog.C: + return errors.New("subscription watchdog: no events received, reconnecting") } } } @@ -406,3 +415,14 @@ func (f *daFollower) backoff() time.Duration { } return 2 * time.Second } + +// watchdogTimeout returns how long to wait for a subscription event before +// assuming the subscription is stalled. Defaults to 3× the DA block time. +const watchdogMultiplier = 3 + +func (f *daFollower) watchdogTimeout() time.Duration { + if f.daBlockTime > 0 { + return f.daBlockTime * watchdogMultiplier + } + return 30 * time.Second +} From f4b9f2feda861f6ebaa5974114232e733159ac34 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Tue, 3 Mar 2026 15:24:32 +0100 Subject: [PATCH 05/17] fix: security hardening for DA subscription path --- block/internal/da/client.go | 4 +- block/internal/syncing/da_follower.go | 60 +++++++++++++++----------- block/internal/syncing/da_retriever.go | 3 ++ 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/block/internal/da/client.go b/block/internal/da/client.go index aa2a5be6ae..bb0f118436 100644 --- a/block/internal/da/client.go +++ b/block/internal/da/client.go @@ -396,7 +396,7 @@ func (c *client) Subscribe(ctx context.Context, namespace []byte) (<-chan datype } // extractBlobData extracts raw byte slices from a subscription response, -// filtering out nil blobs and empty data. +// filtering out nil blobs, empty data, and blobs exceeding DefaultMaxBlobSize. func extractBlobData(resp *blobrpc.SubscriptionResponse) [][]byte { if resp == nil || len(resp.Blobs) == 0 { return nil @@ -407,7 +407,7 @@ func extractBlobData(resp *blobrpc.SubscriptionResponse) [][]byte { continue } data := blob.Data() - if len(data) == 0 { + if len(data) == 0 || len(data) > common.DefaultMaxBlobSize { continue } blobs = append(blobs, data) diff --git a/block/internal/syncing/da_follower.go b/block/internal/syncing/da_follower.go index 5b97acfb5b..785bf859cc 100644 --- a/block/internal/syncing/da_follower.go +++ b/block/internal/syncing/da_follower.go @@ -170,7 +170,11 @@ func (f *daFollower) followLoop() { // A watchdog timer triggers if no events arrive within watchdogTimeout(), // causing a reconnect. func (f *daFollower) runSubscription() error { - headerCh, err := f.client.Subscribe(f.ctx, f.namespace) + // Sub-context ensures the merge goroutine is cancelled when this function returns. + subCtx, subCancel := context.WithCancel(f.ctx) + defer subCancel() + + headerCh, err := f.client.Subscribe(subCtx, f.namespace) if err != nil { return fmt.Errorf("subscribe header namespace: %w", err) } @@ -178,11 +182,11 @@ func (f *daFollower) runSubscription() error { // If namespaces differ, subscribe to the data namespace too and fan-in. ch := headerCh if !bytes.Equal(f.namespace, f.dataNamespace) { - dataCh, err := f.client.Subscribe(f.ctx, f.dataNamespace) + dataCh, err := f.client.Subscribe(subCtx, f.dataNamespace) if err != nil { return fmt.Errorf("subscribe data namespace: %w", err) } - ch = f.mergeSubscriptions(headerCh, dataCh) + ch = f.mergeSubscriptions(subCtx, headerCh, dataCh) } watchdogTimeout := f.watchdogTimeout() @@ -191,8 +195,8 @@ func (f *daFollower) runSubscription() error { for { select { - case <-f.ctx.Done(): - return f.ctx.Err() + case <-subCtx.Done(): + return subCtx.Err() case ev, ok := <-ch: if !ok { return errors.New("subscription channel closed") @@ -205,9 +209,9 @@ func (f *daFollower) runSubscription() error { } } -// mergeSubscriptions fans two subscription channels into one, concatenating -// blobs from both namespaces when events arrive at the same DA height. +// mergeSubscriptions fans two subscription channels into one. func (f *daFollower) mergeSubscriptions( + ctx context.Context, headerCh, dataCh <-chan datypes.SubscriptionEvent, ) <-chan datypes.SubscriptionEvent { out := make(chan datypes.SubscriptionEvent, 16) @@ -217,7 +221,7 @@ func (f *daFollower) mergeSubscriptions( var ev datypes.SubscriptionEvent var ok bool select { - case <-f.ctx.Done(): + case <-ctx.Done(): return case ev, ok = <-headerCh: if !ok { @@ -232,7 +236,7 @@ func (f *daFollower) mergeSubscriptions( } select { case out <- ev: - case <-f.ctx.Done(): + case <-ctx.Done(): return } } @@ -244,34 +248,39 @@ func (f *daFollower) mergeSubscriptions( // caught up (ev.Height == localDAHeight) and blobs are available, it processes // them inline — avoiding a DA re-fetch round trip. Otherwise, it just updates // highestSeenDAHeight and lets catchupLoop handle retrieval. +// +// Uses CAS on localDAHeight to claim exclusive access to processBlobs, +// preventing concurrent map access with catchupLoop. func (f *daFollower) handleSubscriptionEvent(ev datypes.SubscriptionEvent) { // Always record the highest height we've seen from the subscription. f.updateHighest(ev.Height) - // Fast path: process blobs inline when caught up. - // Only fire when ev.Height == localDAHeight to avoid out-of-order processing. - if len(ev.Blobs) > 0 && ev.Height == f.localDAHeight.Load() { + // Fast path: try to claim this height for inline processing. + // CAS(N, N+1) ensures only one goroutine (followLoop or catchupLoop) + // can enter processBlobs for height N. + if len(ev.Blobs) > 0 && f.localDAHeight.CompareAndSwap(ev.Height, ev.Height+1) { events := f.retriever.ProcessBlobs(f.ctx, ev.Blobs, ev.Height) for _, event := range events { if err := f.pipeEvent(f.ctx, event); err != nil { + // Roll back so catchupLoop can retry this height. + f.localDAHeight.Store(ev.Height) f.logger.Warn().Err(err).Uint64("da_height", ev.Height). Msg("failed to pipe inline event, catchup will retry") - return // catchupLoop already signaled via updateHighest + return } } - // Only advance if we produced at least one complete event. - // With split namespaces, the first namespace's blobs may not produce - // events until the second namespace's blobs arrive at the same height. if len(events) != 0 { - f.localDAHeight.Store(ev.Height + 1) f.headReached.Store(true) f.logger.Debug().Uint64("da_height", ev.Height).Int("events", len(events)). Msg("processed subscription blobs inline (fast path)") + } else { + // No complete events (split namespace, waiting for other half). + f.localDAHeight.Store(ev.Height) } return } - // Slow path: behind or no blobs — catchupLoop will handle via signal from updateHighest. + // Slow path: behind, no blobs, or catchupLoop claimed this height. } // updateHighest atomically bumps highestSeenDAHeight and signals catchup if needed. @@ -282,9 +291,6 @@ func (f *daFollower) updateHighest(height uint64) { return } if f.highestSeenDAHeight.CompareAndSwap(cur, height) { - f.logger.Debug(). - Uint64("da_height", height). - Msg("new highest DA height seen from subscription") f.signalCatchup() return } @@ -344,16 +350,20 @@ func (f *daFollower) runCatchup() { return } + // CAS claims this height — prevents followLoop from inline-processing + if !f.localDAHeight.CompareAndSwap(local, local+1) { + // followLoop already advanced past this height via inline processing. + continue + } + if err := f.fetchAndPipeHeight(local); err != nil { + // Roll back so we can retry after backoff. + f.localDAHeight.Store(local) if !f.waitOnCatchupError(err, local) { return } - // Retry the same height after backoff. continue } - - // Advance local height. - f.localDAHeight.Store(local + 1) } } diff --git a/block/internal/syncing/da_retriever.go b/block/internal/syncing/da_retriever.go index 177ff98628..782fa9c318 100644 --- a/block/internal/syncing/da_retriever.go +++ b/block/internal/syncing/da_retriever.go @@ -196,6 +196,9 @@ func (r *daRetriever) validateBlobResponse(res datypes.ResultRetrieve, daHeight // ProcessBlobs processes raw blob bytes to extract headers and data and returns height events. // This is the public interface used by the DAFollower for inline subscription processing. +// +// NOT thread-safe: the caller (DAFollower) must ensure exclusive access via CAS +// on localDAHeight before calling this method. func (r *daRetriever) ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { return r.processBlobs(ctx, blobs, daHeight) } From c21c211e27e8927850aa053c2012cfca371ad902 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Wed, 4 Mar 2026 12:29:57 +0100 Subject: [PATCH 06/17] feat: Implement blob subscription for local DA and update JSON-RPC client to use WebSockets, along with E2E test updates for new `evnode` flags and P2P address retrieval. --- pkg/da/jsonrpc/client.go | 17 ++++- test/e2e/evm_force_inclusion_e2e_test.go | 4 +- test/e2e/evm_test_common.go | 15 ++--- tools/local-da/local.go | 83 ++++++++++++++++++++++++ tools/local-da/rpc.go | 40 ++++++++++-- 5 files changed, 143 insertions(+), 16 deletions(-) diff --git a/pkg/da/jsonrpc/client.go b/pkg/da/jsonrpc/client.go index f1a31a9738..3f7c0dd381 100644 --- a/pkg/da/jsonrpc/client.go +++ b/pkg/da/jsonrpc/client.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "strings" libshare "github.com/celestiaorg/go-square/v3/share" "github.com/filecoin-project/go-jsonrpc" @@ -23,6 +24,15 @@ func (c *Client) Close() { } } +// httpToWS converts an HTTP(S) URL to a WebSocket URL. +// go-jsonrpc requires WebSocket for channel-based subscriptions (e.g. Subscribe). +// WebSocket connections also support regular RPC calls, so this is backward-compatible. +func httpToWS(addr string) string { + addr = strings.Replace(addr, "https://", "wss://", 1) + addr = strings.Replace(addr, "http://", "ws://", 1) + return addr +} + // NewClient connects to the celestia-node RPC endpoint func NewClient(ctx context.Context, addr, token string, authHeaderName string) (*Client, error) { var httpHeader http.Header @@ -33,16 +43,19 @@ func NewClient(ctx context.Context, addr, token string, authHeaderName string) ( httpHeader = http.Header{authHeaderName: []string{fmt.Sprintf("Bearer %s", token)}} } + // Use WebSocket so that channel-based subscriptions (blob.Subscribe) work. + wsAddr := httpToWS(addr) + var cl Client // Connect to the blob namespace - blobCloser, err := jsonrpc.NewClient(ctx, addr, "blob", &cl.Blob.Internal, httpHeader) + blobCloser, err := jsonrpc.NewClient(ctx, wsAddr, "blob", &cl.Blob.Internal, httpHeader) if err != nil { return nil, fmt.Errorf("failed to connect to blob namespace: %w", err) } // Connect to the header namespace - headerCloser, err := jsonrpc.NewClient(ctx, addr, "header", &cl.Header.Internal, httpHeader) + headerCloser, err := jsonrpc.NewClient(ctx, wsAddr, "header", &cl.Header.Internal, httpHeader) if err != nil { blobCloser() return nil, fmt.Errorf("failed to connect to header namespace: %w", err) diff --git a/test/e2e/evm_force_inclusion_e2e_test.go b/test/e2e/evm_force_inclusion_e2e_test.go index f201f7f363..c6c25270c5 100644 --- a/test/e2e/evm_force_inclusion_e2e_test.go +++ b/test/e2e/evm_force_inclusion_e2e_test.go @@ -238,8 +238,10 @@ func TestEvmFullNodeForceInclusionE2E(t *testing.T) { // --- End Sequencer Setup --- // --- Start Full Node Setup --- + // Get sequencer's full P2P address (including peer ID) for the full node to connect to + sequencerP2PAddress := getNodeP2PAddress(t, sut, sequencerHome, env.Endpoints.RollkitRPCPort) // Reuse setupFullNode helper which handles genesis copying and node startup - setupFullNode(t, sut, fullNodeHome, sequencerHome, env.FullNodeJWT, env.GenesisHash, env.Endpoints.GetRollkitP2PAddress(), env.Endpoints) + setupFullNode(t, sut, fullNodeHome, sequencerHome, env.FullNodeJWT, env.GenesisHash, sequencerP2PAddress, env.Endpoints) t.Log("Full node is up") // --- End Full Node Setup --- diff --git a/test/e2e/evm_test_common.go b/test/e2e/evm_test_common.go index 7c089e762b..c5bfb05c23 100644 --- a/test/e2e/evm_test_common.go +++ b/test/e2e/evm_test_common.go @@ -449,15 +449,15 @@ func setupFullNode(t testing.TB, sut *SystemUnderTest, fullNodeHome, sequencerHo "--home", fullNodeHome, "--evm.jwt-secret-file", fullNodeJwtSecretFile, "--evm.genesis-hash", genesisHash, - "--rollkit.p2p.peers", sequencerP2PAddress, + "--evnode.p2p.peers", sequencerP2PAddress, "--evm.engine-url", endpoints.GetFullNodeEngineURL(), "--evm.eth-url", endpoints.GetFullNodeEthURL(), - "--rollkit.da.block_time", DefaultDABlockTime, - "--rollkit.da.address", endpoints.GetDAAddress(), - "--rollkit.da.namespace", DefaultDANamespace, - "--rollkit.da.batching_strategy", "immediate", - "--rollkit.rpc.address", endpoints.GetFullNodeRPCListen(), - "--rollkit.p2p.listen_address", endpoints.GetFullNodeP2PAddress(), + "--evnode.da.block_time", DefaultDABlockTime, + "--evnode.da.address", endpoints.GetDAAddress(), + "--evnode.da.namespace", DefaultDANamespace, + "--evnode.da.batching_strategy", "immediate", + "--evnode.rpc.address", endpoints.GetFullNodeRPCListen(), + "--evnode.p2p.listen_address", endpoints.GetFullNodeP2PAddress(), } sut.ExecCmd(evmSingleBinaryPath, args...) // Use AwaitNodeLive instead of AwaitNodeUp because in lazy mode scenarios, @@ -932,4 +932,3 @@ func PrintTraceReport(t testing.TB, label string, spans []TraceSpan) { t.Logf("%-40s %5.1f%% %s", name, pct, bar) } } - diff --git a/tools/local-da/local.go b/tools/local-da/local.go index ef519f17ce..07d6876f9d 100644 --- a/tools/local-da/local.go +++ b/tools/local-da/local.go @@ -13,6 +13,7 @@ import ( "sync" "time" + libshare "github.com/celestiaorg/go-square/v3/share" "github.com/rs/zerolog" blobrpc "github.com/evstack/ev-node/pkg/da/jsonrpc" @@ -27,6 +28,18 @@ const ( DefaultBlockTime = 1 * time.Second ) +// subscriber holds a registered subscription's channel and namespace filter. +type subscriber struct { + ch chan subscriptionEvent + ns libshare.Namespace +} + +// subscriptionEvent is sent to subscribers when a new DA block is produced. +type subscriptionEvent struct { + height uint64 + blobs []*blobrpc.Blob +} + // LocalDA is a simple implementation of in-memory DA. Not production ready! Intended only for testing! // // Data is stored in a map, where key is a serialized sequence number. This key is returned as ID. @@ -43,6 +56,10 @@ type LocalDA struct { blockTime time.Duration lastTime time.Time // tracks last timestamp to ensure monotonicity + // Subscriber registry (protected by mu) + subscribers map[int]*subscriber + nextSubID int + logger zerolog.Logger } @@ -57,6 +74,7 @@ func NewLocalDA(logger zerolog.Logger, opts ...func(*LocalDA) *LocalDA) *LocalDA data: make(map[uint64][]kvp), timestamps: make(map[uint64]time.Time), blobData: make(map[uint64][]*blobrpc.Blob), + subscribers: make(map[int]*subscriber), maxBlobSize: DefaultMaxBlobSize, blockTime: DefaultBlockTime, lastTime: time.Now(), @@ -209,6 +227,7 @@ func (d *LocalDA) SubmitWithOptions(ctx context.Context, blobs []datypes.Blob, g d.data[d.height] = append(d.data[d.height], kvp{ids[i], blob}) } + d.notifySubscribers(d.height) d.logger.Info().Uint64("newHeight", d.height).Int("count", len(ids)).Msg("SubmitWithOptions successful") return ids, nil } @@ -239,6 +258,7 @@ func (d *LocalDA) Submit(ctx context.Context, blobs []datypes.Blob, gasPrice flo d.data[d.height] = append(d.data[d.height], kvp{ids[i], blob}) } + d.notifySubscribers(d.height) d.logger.Info().Uint64("newHeight", d.height).Int("count", len(ids)).Msg("Submit successful") return ids, nil } @@ -335,5 +355,68 @@ func (d *LocalDA) produceEmptyBlock() { defer d.mu.Unlock() d.height++ d.timestamps[d.height] = d.monotonicTime() + d.notifySubscribers(d.height) d.logger.Debug().Uint64("height", d.height).Msg("produced empty block") } + +// subscribe registers a new subscriber for blobs matching the given namespace. +// Returns a read-only channel and a subscription ID for later unsubscription. +// Must NOT be called with d.mu held. +func (d *LocalDA) subscribe(ns libshare.Namespace) (<-chan subscriptionEvent, int) { + d.mu.Lock() + defer d.mu.Unlock() + + id := d.nextSubID + d.nextSubID++ + ch := make(chan subscriptionEvent, 64) + d.subscribers[id] = &subscriber{ch: ch, ns: ns} + d.logger.Info().Int("subID", id).Str("namespace", hex.EncodeToString(ns.Bytes())).Msg("subscriber registered") + return ch, id +} + +// unsubscribe removes a subscriber and closes its channel. +// Must NOT be called with d.mu held. +func (d *LocalDA) unsubscribe(id int) { + d.mu.Lock() + defer d.mu.Unlock() + + if sub, ok := d.subscribers[id]; ok { + close(sub.ch) + delete(d.subscribers, id) + d.logger.Info().Int("subID", id).Msg("subscriber unregistered") + } +} + +// notifySubscribers sends a subscriptionEvent to all registered subscribers. +// For each subscriber, only blobs matching the subscriber's namespace are included. +// Slow consumers (full channel) are dropped to avoid blocking block production. +// MUST be called with d.mu held. +func (d *LocalDA) notifySubscribers(height uint64) { + if len(d.subscribers) == 0 { + return + } + + allBlobs := d.blobData[height] // may be nil for empty blocks + + for id, sub := range d.subscribers { + // Filter blobs matching subscriber namespace + var matched []*blobrpc.Blob + for _, b := range allBlobs { + if b != nil && b.Namespace().Equals(sub.ns) { + matched = append(matched, b) + } + } + + evt := subscriptionEvent{ + height: height, + blobs: matched, + } + + select { + case sub.ch <- evt: + default: + // Slow consumer — drop to avoid blocking block production + d.logger.Warn().Int("subID", id).Uint64("height", height).Msg("dropping event for slow subscriber") + } + } +} diff --git a/tools/local-da/rpc.go b/tools/local-da/rpc.go index 6f681d6538..de8e0da070 100644 --- a/tools/local-da/rpc.go +++ b/tools/local-da/rpc.go @@ -31,6 +31,7 @@ func (s *blobServer) Submit(_ context.Context, blobs []*jsonrpc.Blob, _ *jsonrpc if len(blobs) == 0 { s.da.timestamps[height] = time.Now() + s.da.notifySubscribers(height) return height, nil } @@ -48,6 +49,7 @@ func (s *blobServer) Submit(_ context.Context, blobs []*jsonrpc.Blob, _ *jsonrpc s.da.blobData[height] = append(s.da.blobData[height], b) } s.da.timestamps[height] = time.Now() + s.da.notifySubscribers(height) return height, nil } @@ -127,11 +129,39 @@ func (s *blobServer) GetCommitmentProof(_ context.Context, _ uint64, _ libshare. return &jsonrpc.CommitmentProof{}, nil } -// Subscribe returns a closed channel; LocalDA does not push live updates. -func (s *blobServer) Subscribe(_ context.Context, _ libshare.Namespace) (<-chan *jsonrpc.SubscriptionResponse, error) { - ch := make(chan *jsonrpc.SubscriptionResponse) - close(ch) - return ch, nil +// Subscribe streams blobs as they are included for the given namespace. +// The returned channel emits a SubscriptionResponse for every new DA block. +// The channel is closed when ctx is cancelled. +func (s *blobServer) Subscribe(ctx context.Context, namespace libshare.Namespace) (<-chan *jsonrpc.SubscriptionResponse, error) { + eventCh, subID := s.da.subscribe(namespace) + + out := make(chan *jsonrpc.SubscriptionResponse, 64) + go func() { + defer close(out) + defer s.da.unsubscribe(subID) + + for { + select { + case <-ctx.Done(): + return + case evt, ok := <-eventCh: + if !ok { + return + } + resp := &jsonrpc.SubscriptionResponse{ + Height: evt.height, + Blobs: evt.blobs, + } + select { + case out <- resp: + case <-ctx.Done(): + return + } + } + } + }() + + return out, nil } // startBlobServer starts an HTTP JSON-RPC server on addr serving the blob namespace. From b1f7db1c6bf661db545f5b3f6bc4f3ebf4ce7e6e Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Wed, 4 Mar 2026 14:13:02 +0100 Subject: [PATCH 07/17] WS client constructor --- apps/evm/cmd/run.go | 2 +- apps/grpc/cmd/run.go | 2 +- apps/testapp/cmd/run.go | 2 +- pkg/cmd/run_node.go | 17 +++++++++-------- pkg/da/jsonrpc/client.go | 23 +++++++++++++++-------- 5 files changed, 27 insertions(+), 19 deletions(-) diff --git a/apps/evm/cmd/run.go b/apps/evm/cmd/run.go index 60730eac52..0d3fc25f14 100644 --- a/apps/evm/cmd/run.go +++ b/apps/evm/cmd/run.go @@ -60,7 +60,7 @@ var RunCmd = &cobra.Command{ return err } - blobClient, err := blobrpc.NewClient(context.Background(), nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") + blobClient, err := blobrpc.NewWSClient(context.Background(), nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") if err != nil { return fmt.Errorf("failed to create blob client: %w", err) } diff --git a/apps/grpc/cmd/run.go b/apps/grpc/cmd/run.go index 41278999cb..6db5a409da 100644 --- a/apps/grpc/cmd/run.go +++ b/apps/grpc/cmd/run.go @@ -108,7 +108,7 @@ func createSequencer( genesis genesis.Genesis, executor execution.Executor, ) (coresequencer.Sequencer, error) { - blobClient, err := blobrpc.NewClient(ctx, nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") + blobClient, err := blobrpc.NewWSClient(ctx, nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") if err != nil { return nil, fmt.Errorf("failed to create blob client: %w", err) } diff --git a/apps/testapp/cmd/run.go b/apps/testapp/cmd/run.go index fe026cdfcc..fff3a9e064 100644 --- a/apps/testapp/cmd/run.go +++ b/apps/testapp/cmd/run.go @@ -111,7 +111,7 @@ func createSequencer( genesis genesis.Genesis, executor execution.Executor, ) (coresequencer.Sequencer, error) { - blobClient, err := blobrpc.NewClient(ctx, nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") + blobClient, err := blobrpc.NewWSClient(ctx, nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") if err != nil { return nil, fmt.Errorf("failed to create blob client: %w", err) } diff --git a/pkg/cmd/run_node.go b/pkg/cmd/run_node.go index 33b1eba006..4a5c7577c6 100644 --- a/pkg/cmd/run_node.go +++ b/pkg/cmd/run_node.go @@ -107,14 +107,8 @@ func StartNode( }() } - blobClient, err := blobrpc.NewClient(ctx, nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") - if err != nil { - return fmt.Errorf("failed to create blob client: %w", err) - } - defer blobClient.Close() - daClient := block.NewDAClient(blobClient, nodeConfig, logger) - - // create a new remote signer + // Validate and load signer first (before attempting DA connection, which may fail + // eagerly over WebSocket if no DA server is running). var signer signer.Signer if nodeConfig.Signer.SignerType == "file" && (nodeConfig.Node.Aggregator && !nodeConfig.Node.BasedSequencer) { // Get passphrase file path @@ -152,6 +146,13 @@ func StartNode( return fmt.Errorf("unknown signer type: %s", nodeConfig.Signer.SignerType) } + blobClient, err := blobrpc.NewWSClient(ctx, nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") + if err != nil { + return fmt.Errorf("failed to create blob client: %w", err) + } + defer blobClient.Close() + daClient := block.NewDAClient(blobClient, nodeConfig, logger) + // sanity check for based sequencer if nodeConfig.Node.BasedSequencer && genesis.DAStartHeight == 0 { return fmt.Errorf("based sequencing requires DAStartHeight to be set in genesis. This value should be identical for all nodes of the chain") diff --git a/pkg/da/jsonrpc/client.go b/pkg/da/jsonrpc/client.go index 3f7c0dd381..c856cdd7b9 100644 --- a/pkg/da/jsonrpc/client.go +++ b/pkg/da/jsonrpc/client.go @@ -25,15 +25,16 @@ func (c *Client) Close() { } // httpToWS converts an HTTP(S) URL to a WebSocket URL. -// go-jsonrpc requires WebSocket for channel-based subscriptions (e.g. Subscribe). -// WebSocket connections also support regular RPC calls, so this is backward-compatible. func httpToWS(addr string) string { addr = strings.Replace(addr, "https://", "wss://", 1) addr = strings.Replace(addr, "http://", "ws://", 1) return addr } -// NewClient connects to the celestia-node RPC endpoint +// NewClient connects to the DA RPC endpoint using the address as-is. +// Uses HTTP by default (lazy connection — only connects on first RPC call). +// Does NOT support channel-based subscriptions (e.g. Subscribe). +// For subscription support, use NewWSClient instead. func NewClient(ctx context.Context, addr, token string, authHeaderName string) (*Client, error) { var httpHeader http.Header if token != "" { @@ -43,19 +44,16 @@ func NewClient(ctx context.Context, addr, token string, authHeaderName string) ( httpHeader = http.Header{authHeaderName: []string{fmt.Sprintf("Bearer %s", token)}} } - // Use WebSocket so that channel-based subscriptions (blob.Subscribe) work. - wsAddr := httpToWS(addr) - var cl Client // Connect to the blob namespace - blobCloser, err := jsonrpc.NewClient(ctx, wsAddr, "blob", &cl.Blob.Internal, httpHeader) + blobCloser, err := jsonrpc.NewClient(ctx, addr, "blob", &cl.Blob.Internal, httpHeader) if err != nil { return nil, fmt.Errorf("failed to connect to blob namespace: %w", err) } // Connect to the header namespace - headerCloser, err := jsonrpc.NewClient(ctx, wsAddr, "header", &cl.Header.Internal, httpHeader) + headerCloser, err := jsonrpc.NewClient(ctx, addr, "header", &cl.Header.Internal, httpHeader) if err != nil { blobCloser() return nil, fmt.Errorf("failed to connect to header namespace: %w", err) @@ -70,6 +68,15 @@ func NewClient(ctx context.Context, addr, token string, authHeaderName string) ( return &cl, nil } +// NewWSClient connects to the DA RPC endpoint over WebSocket. +// Automatically converts http:// to ws:// (and https:// to wss://). +// Supports channel-based subscriptions (e.g. Subscribe). +// Note: WebSocket connections are eager — they connect at creation time +// and will fail immediately if the server is unavailable. +func NewWSClient(ctx context.Context, addr, token string, authHeaderName string) (*Client, error) { + return NewClient(ctx, httpToWS(addr), token, authHeaderName) +} + // BlobAPI mirrors celestia-node's blob module (nodebuilder/blob/blob.go). // jsonrpc.NewClient wires Internal.* to RPC stubs. type BlobAPI struct { From 04aefda14e351ab491b4e9d1c837844f3b8b664e Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Wed, 4 Mar 2026 15:00:13 +0100 Subject: [PATCH 08/17] Merge --- block/internal/syncing/syncer_benchmark_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/block/internal/syncing/syncer_benchmark_test.go b/block/internal/syncing/syncer_benchmark_test.go index 5e7a76ce14..395f292702 100644 --- a/block/internal/syncing/syncer_benchmark_test.go +++ b/block/internal/syncing/syncer_benchmark_test.go @@ -43,7 +43,7 @@ func BenchmarkSyncerIO(b *testing.B) { fixt := newBenchFixture(b, spec.heights, spec.shuffledTx, spec.daDelay, spec.execDelay, true) // run both loops - go fixt.s.processLoop() + go fixt.s.processLoop(b.Context()) // Create a DAFollower to drive DA retrieval. follower := NewDAFollower(DAFollowerConfig{ @@ -58,7 +58,7 @@ func BenchmarkSyncerIO(b *testing.B) { follower.highestSeenDAHeight.Store(spec.heights + daHeightOffset) go follower.runCatchup() - fixt.s.startSyncWorkers() + fixt.s.startSyncWorkers(b.Context()) require.Eventually(b, func() bool { processedHeight, _ := fixt.s.store.Height(b.Context()) From dd7e0cdd75fa6f737aa6a5ee766c921877f755ad Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Wed, 4 Mar 2026 17:08:52 +0100 Subject: [PATCH 09/17] Linter --- block/internal/syncing/da_follower.go | 59 +++++++++---------- block/internal/syncing/syncer.go | 2 +- block/internal/syncing/syncer_backoff_test.go | 27 ++++----- .../internal/syncing/syncer_benchmark_test.go | 12 ++-- block/internal/syncing/syncer_test.go | 8 +-- 5 files changed, 50 insertions(+), 58 deletions(-) diff --git a/block/internal/syncing/da_follower.go b/block/internal/syncing/da_follower.go index 785bf859cc..e92d5be1ab 100644 --- a/block/internal/syncing/da_follower.go +++ b/block/internal/syncing/da_follower.go @@ -69,7 +69,6 @@ type daFollower struct { daBlockTime time.Duration // lifecycle - ctx context.Context cancel context.CancelFunc wg sync.WaitGroup } @@ -108,11 +107,11 @@ func NewDAFollower(cfg DAFollowerConfig) DAFollower { // Start begins the follow and catchup goroutines. func (f *daFollower) Start(ctx context.Context) error { - f.ctx, f.cancel = context.WithCancel(ctx) + ctx, f.cancel = context.WithCancel(ctx) f.wg.Add(2) - go f.followLoop() - go f.catchupLoop() + go f.followLoop(ctx) + go f.catchupLoop(ctx) f.logger.Info(). Uint64("start_da_height", f.localDAHeight.Load()). @@ -144,20 +143,20 @@ func (f *daFollower) signalCatchup() { // followLoop subscribes to DA blob events and keeps highestSeenDAHeight up to date. // When a new height appears above localDAHeight, it wakes the catchup loop. -func (f *daFollower) followLoop() { +func (f *daFollower) followLoop(ctx context.Context) { defer f.wg.Done() f.logger.Info().Msg("starting follow loop") defer f.logger.Info().Msg("follow loop stopped") for { - if err := f.runSubscription(); err != nil { - if f.ctx.Err() != nil { + if err := f.runSubscription(ctx); err != nil { + if ctx.Err() != nil { return } f.logger.Warn().Err(err).Msg("DA subscription failed, reconnecting") select { - case <-f.ctx.Done(): + case <-ctx.Done(): return case <-time.After(f.backoff()): } @@ -169,9 +168,9 @@ func (f *daFollower) followLoop() { // different) and processes events until a channel is closed or an error occurs. // A watchdog timer triggers if no events arrive within watchdogTimeout(), // causing a reconnect. -func (f *daFollower) runSubscription() error { +func (f *daFollower) runSubscription(ctx context.Context) error { // Sub-context ensures the merge goroutine is cancelled when this function returns. - subCtx, subCancel := context.WithCancel(f.ctx) + subCtx, subCancel := context.WithCancel(ctx) defer subCancel() headerCh, err := f.client.Subscribe(subCtx, f.namespace) @@ -201,7 +200,7 @@ func (f *daFollower) runSubscription() error { if !ok { return errors.New("subscription channel closed") } - f.handleSubscriptionEvent(ev) + f.handleSubscriptionEvent(ctx, ev) watchdog.Reset(watchdogTimeout) case <-watchdog.C: return errors.New("subscription watchdog: no events received, reconnecting") @@ -251,7 +250,7 @@ func (f *daFollower) mergeSubscriptions( // // Uses CAS on localDAHeight to claim exclusive access to processBlobs, // preventing concurrent map access with catchupLoop. -func (f *daFollower) handleSubscriptionEvent(ev datypes.SubscriptionEvent) { +func (f *daFollower) handleSubscriptionEvent(ctx context.Context, ev datypes.SubscriptionEvent) { // Always record the highest height we've seen from the subscription. f.updateHighest(ev.Height) @@ -259,9 +258,9 @@ func (f *daFollower) handleSubscriptionEvent(ev datypes.SubscriptionEvent) { // CAS(N, N+1) ensures only one goroutine (followLoop or catchupLoop) // can enter processBlobs for height N. if len(ev.Blobs) > 0 && f.localDAHeight.CompareAndSwap(ev.Height, ev.Height+1) { - events := f.retriever.ProcessBlobs(f.ctx, ev.Blobs, ev.Height) + events := f.retriever.ProcessBlobs(ctx, ev.Blobs, ev.Height) for _, event := range events { - if err := f.pipeEvent(f.ctx, event); err != nil { + if err := f.pipeEvent(ctx, event); err != nil { // Roll back so catchupLoop can retry this height. f.localDAHeight.Store(ev.Height) f.logger.Warn().Err(err).Uint64("da_height", ev.Height). @@ -299,7 +298,7 @@ func (f *daFollower) updateHighest(height uint64) { // catchupLoop waits for signals and sequentially retrieves DA heights // from localDAHeight up to highestSeenDAHeight. -func (f *daFollower) catchupLoop() { +func (f *daFollower) catchupLoop(ctx context.Context) { defer f.wg.Done() f.logger.Info().Msg("starting catchup loop") @@ -307,19 +306,19 @@ func (f *daFollower) catchupLoop() { for { select { - case <-f.ctx.Done(): + case <-ctx.Done(): return case <-f.catchupSignal: - f.runCatchup() + f.runCatchup(ctx) } } } // runCatchup sequentially retrieves from localDAHeight to highestSeenDAHeight. // It handles priority heights first, then sequential heights. -func (f *daFollower) runCatchup() { +func (f *daFollower) runCatchup(ctx context.Context) { for { - if f.ctx.Err() != nil { + if ctx.Err() != nil { return } @@ -332,8 +331,8 @@ func (f *daFollower) runCatchup() { f.logger.Debug(). Uint64("da_height", priorityHeight). Msg("fetching priority DA height from P2P hint") - if err := f.fetchAndPipeHeight(priorityHeight); err != nil { - if !f.waitOnCatchupError(err, priorityHeight) { + if err := f.fetchAndPipeHeight(ctx, priorityHeight); err != nil { + if !f.waitOnCatchupError(ctx, err, priorityHeight) { return } } @@ -350,16 +349,16 @@ func (f *daFollower) runCatchup() { return } - // CAS claims this height — prevents followLoop from inline-processing + // CAS claims this height prevents followLoop from inline-processing if !f.localDAHeight.CompareAndSwap(local, local+1) { // followLoop already advanced past this height via inline processing. continue } - if err := f.fetchAndPipeHeight(local); err != nil { + if err := f.fetchAndPipeHeight(ctx, local); err != nil { // Roll back so we can retry after backoff. f.localDAHeight.Store(local) - if !f.waitOnCatchupError(err, local) { + if !f.waitOnCatchupError(ctx, err, local) { return } continue @@ -369,8 +368,8 @@ func (f *daFollower) runCatchup() { // fetchAndPipeHeight retrieves events at a single DA height and pipes them // to the syncer. -func (f *daFollower) fetchAndPipeHeight(daHeight uint64) error { - events, err := f.retriever.RetrieveFromDA(f.ctx, daHeight) +func (f *daFollower) fetchAndPipeHeight(ctx context.Context, daHeight uint64) error { + events, err := f.retriever.RetrieveFromDA(ctx, daHeight) if err != nil { switch { case errors.Is(err, datypes.ErrBlobNotFound): @@ -387,7 +386,7 @@ func (f *daFollower) fetchAndPipeHeight(daHeight uint64) error { } for _, event := range events { - if err := f.pipeEvent(f.ctx, event); err != nil { + if err := f.pipeEvent(ctx, event); err != nil { return err } } @@ -401,17 +400,17 @@ var errCaughtUp = errors.New("caught up with DA head") // waitOnCatchupError logs the error and backs off before retrying. // It returns true if the caller should continue (retry), or false if the // catchup loop should exit (context cancelled or caught-up sentinel). -func (f *daFollower) waitOnCatchupError(err error, daHeight uint64) bool { +func (f *daFollower) waitOnCatchupError(ctx context.Context, err error, daHeight uint64) bool { if errors.Is(err, errCaughtUp) { f.logger.Debug().Uint64("da_height", daHeight).Msg("DA catchup reached head, waiting for subscription signal") return false } - if f.ctx.Err() != nil { + if ctx.Err() != nil { return false } f.logger.Warn().Err(err).Uint64("da_height", daHeight).Msg("catchup error, backing off") select { - case <-f.ctx.Done(): + case <-ctx.Done(): return false case <-time.After(f.backoff()): return true diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index b20d800196..b355bcd87e 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -208,7 +208,7 @@ func (s *Syncer) Start(ctx context.Context) error { StartDAHeight: s.daRetrieverHeight.Load(), DABlockTime: s.config.DA.BlockTime.Duration, }) - if err := s.daFollower.Start(s.ctx); err != nil { + if err := s.daFollower.Start(ctx); err != nil { return fmt.Errorf("failed to start DA follower: %w", err) } diff --git a/block/internal/syncing/syncer_backoff_test.go b/block/internal/syncing/syncer_backoff_test.go index 57981e8878..bd2d6000fe 100644 --- a/block/internal/syncing/syncer_backoff_test.go +++ b/block/internal/syncing/syncer_backoff_test.go @@ -73,7 +73,7 @@ func TestDAFollower_BackoffOnCatchupError(t *testing.T) { DABlockTime: tc.daBlockTime, }).(*daFollower) - follower.ctx, follower.cancel = context.WithCancel(ctx) + ctx, follower.cancel = context.WithCancel(ctx) follower.highestSeenDAHeight.Store(102) var callTimes []time.Time @@ -105,7 +105,7 @@ func TestDAFollower_BackoffOnCatchupError(t *testing.T) { Return(nil, datypes.ErrBlobNotFound).Once() } - go follower.runCatchup() + go follower.runCatchup(ctx) <-ctx.Done() if tc.expectsBackoff { @@ -157,7 +157,7 @@ func TestDAFollower_BackoffResetOnSuccess(t *testing.T) { DABlockTime: 1 * time.Second, }).(*daFollower) - follower.ctx, follower.cancel = context.WithCancel(ctx) + ctx, follower.cancel = context.WithCancel(ctx) follower.highestSeenDAHeight.Store(105) var callTimes []time.Time @@ -198,7 +198,7 @@ func TestDAFollower_BackoffResetOnSuccess(t *testing.T) { }). Return(nil, datypes.ErrBlobNotFound).Once() - go follower.runCatchup() + go follower.runCatchup(ctx) <-ctx.Done() require.Len(t, callTimes, 3, "should make exactly 3 calls") @@ -234,13 +234,11 @@ func TestDAFollower_CatchupThenReachHead(t *testing.T) { DABlockTime: 500 * time.Millisecond, }).(*daFollower) - follower.ctx, follower.cancel = context.WithCancel(ctx) follower.highestSeenDAHeight.Store(5) var fetchedHeights []uint64 for h := uint64(3); h <= 5; h++ { - h := h daRetriever.On("RetrieveFromDA", mock.Anything, h). Run(func(args mock.Arguments) { fetchedHeights = append(fetchedHeights, h) @@ -248,7 +246,7 @@ func TestDAFollower_CatchupThenReachHead(t *testing.T) { Return(nil, datypes.ErrBlobNotFound).Once() } - follower.runCatchup() + follower.runCatchup(ctx) assert.True(t, follower.HasReachedHead(), "should have reached DA head") // Heights 3, 4, 5 processed; local now at 6 which > highest (5) → caught up @@ -278,9 +276,6 @@ func TestDAFollower_InlineProcessing(t *testing.T) { DABlockTime: 500 * time.Millisecond, }).(*daFollower) - follower.ctx, follower.cancel = context.WithCancel(t.Context()) - defer follower.cancel() - blobs := [][]byte{[]byte("header-blob"), []byte("data-blob")} expectedEvents := []common.DAHeightEvent{ {DaHeight: 10, Source: common.SourceDA}, @@ -291,7 +286,7 @@ func TestDAFollower_InlineProcessing(t *testing.T) { Return(expectedEvents).Once() // Simulate subscription event at the current localDAHeight - follower.handleSubscriptionEvent(datypes.SubscriptionEvent{ + follower.handleSubscriptionEvent(t.Context(), datypes.SubscriptionEvent{ Height: 10, Blobs: blobs, }) @@ -317,11 +312,12 @@ func TestDAFollower_InlineProcessing(t *testing.T) { DABlockTime: 500 * time.Millisecond, }).(*daFollower) - follower.ctx, follower.cancel = context.WithCancel(t.Context()) + ctx := t.Context() + ctx, follower.cancel = context.WithCancel(ctx) defer follower.cancel() // Subscription reports height 15 but local is at 10 — should NOT process inline - follower.handleSubscriptionEvent(datypes.SubscriptionEvent{ + follower.handleSubscriptionEvent(ctx, datypes.SubscriptionEvent{ Height: 15, Blobs: [][]byte{[]byte("blob")}, }) @@ -346,11 +342,8 @@ func TestDAFollower_InlineProcessing(t *testing.T) { DABlockTime: 500 * time.Millisecond, }).(*daFollower) - follower.ctx, follower.cancel = context.WithCancel(t.Context()) - defer follower.cancel() - // Subscription at current height but no blobs — should fall through - follower.handleSubscriptionEvent(datypes.SubscriptionEvent{ + follower.handleSubscriptionEvent(t.Context(), datypes.SubscriptionEvent{ Height: 10, Blobs: nil, }) diff --git a/block/internal/syncing/syncer_benchmark_test.go b/block/internal/syncing/syncer_benchmark_test.go index 395f292702..a69da71507 100644 --- a/block/internal/syncing/syncer_benchmark_test.go +++ b/block/internal/syncing/syncer_benchmark_test.go @@ -43,7 +43,8 @@ func BenchmarkSyncerIO(b *testing.B) { fixt := newBenchFixture(b, spec.heights, spec.shuffledTx, spec.daDelay, spec.execDelay, true) // run both loops - go fixt.s.processLoop(b.Context()) + ctx := b.Context() + go fixt.s.processLoop(ctx) // Create a DAFollower to drive DA retrieval. follower := NewDAFollower(DAFollowerConfig{ @@ -54,14 +55,13 @@ func BenchmarkSyncerIO(b *testing.B) { StartDAHeight: fixt.s.daRetrieverHeight.Load(), DABlockTime: 0, }).(*daFollower) - follower.ctx, follower.cancel = context.WithCancel(fixt.s.ctx) follower.highestSeenDAHeight.Store(spec.heights + daHeightOffset) - go follower.runCatchup() + go follower.runCatchup(ctx) - fixt.s.startSyncWorkers(b.Context()) + fixt.s.startSyncWorkers(ctx) require.Eventually(b, func() bool { - processedHeight, _ := fixt.s.store.Height(b.Context()) + processedHeight, _ := fixt.s.store.Height(ctx) return processedHeight == spec.heights }, 5*time.Second, 50*time.Microsecond) fixt.s.cancel() @@ -75,7 +75,7 @@ func BenchmarkSyncerIO(b *testing.B) { require.Len(b, fixt.s.heightInCh, 0) assert.Equal(b, spec.heights+daHeightOffset, fixt.s.daRetrieverHeight) - gotStoreHeight, err := fixt.s.store.Height(b.Context()) + gotStoreHeight, err := fixt.s.store.Height(ctx) require.NoError(b, err) assert.Equal(b, spec.heights, gotStoreHeight) } diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 205e45c104..64ccea439d 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -427,10 +427,10 @@ func TestSyncLoopPersistState(t *testing.T) { StartDAHeight: syncerInst1.daRetrieverHeight.Load(), DABlockTime: cfg.DA.BlockTime.Duration, }).(*daFollower) - follower1.ctx, follower1.cancel = context.WithCancel(ctx) + ctx, follower1.cancel = context.WithCancel(ctx) // Set highest so catchup runs through all mocked heights. follower1.highestSeenDAHeight.Store(myFutureDAHeight) - go follower1.runCatchup() + go follower1.runCatchup(ctx) syncerInst1.startSyncWorkers(ctx) syncerInst1.wg.Wait() requireEmptyChan(t, errorCh) @@ -504,9 +504,9 @@ func TestSyncLoopPersistState(t *testing.T) { StartDAHeight: syncerInst2.daRetrieverHeight.Load(), DABlockTime: cfg.DA.BlockTime.Duration, }).(*daFollower) - follower2.ctx, follower2.cancel = context.WithCancel(ctx) + ctx, follower2.cancel = context.WithCancel(ctx) follower2.highestSeenDAHeight.Store(syncerInst2.daRetrieverHeight.Load() + 1) - go follower2.runCatchup() + go follower2.runCatchup(ctx) syncerInst2.startSyncWorkers(ctx) syncerInst2.wg.Wait() From 6c1e630c6ee6d15d30940c0deb83f03cc3a4fce9 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 5 Mar 2026 13:28:43 +0100 Subject: [PATCH 10/17] Review feedback --- apps/evm/cmd/run.go | 2 +- block/internal/syncing/da_follower.go | 42 +++++++------ block/internal/syncing/da_retriever.go | 61 ++++++++++++------- block/internal/syncing/syncer.go | 5 +- block/internal/syncing/syncer_backoff_test.go | 10 +-- .../internal/syncing/syncer_benchmark_test.go | 12 ++-- block/internal/syncing/syncer_test.go | 2 +- tools/local-da/local.go | 20 ++++++ 8 files changed, 95 insertions(+), 59 deletions(-) diff --git a/apps/evm/cmd/run.go b/apps/evm/cmd/run.go index 0d3fc25f14..73b809de46 100644 --- a/apps/evm/cmd/run.go +++ b/apps/evm/cmd/run.go @@ -60,7 +60,7 @@ var RunCmd = &cobra.Command{ return err } - blobClient, err := blobrpc.NewWSClient(context.Background(), nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") + blobClient, err := blobrpc.NewWSClient(cmd.Context(), nodeConfig.DA.Address, nodeConfig.DA.AuthToken, "") if err != nil { return fmt.Errorf("failed to create blob client: %w", err) } diff --git a/block/internal/syncing/da_follower.go b/block/internal/syncing/da_follower.go index e92d5be1ab..71b15f570e 100644 --- a/block/internal/syncing/da_follower.go +++ b/block/internal/syncing/da_follower.go @@ -22,7 +22,7 @@ import ( // - followLoop listens on the subscription channel. When caught up, it processes // subscription blobs inline (fast path, no DA re-fetch). Otherwise, it updates // highestSeenDAHeight and signals the catchup loop. -// - catchupLoop sequentially retrieves from localDAHeight → highestSeenDAHeight, +// - catchupLoop sequentially retrieves from localNextDAHeight → highestSeenDAHeight, // piping events to the Syncer's heightInCh. // // The two goroutines share only atomic state; no mutexes needed. @@ -51,9 +51,9 @@ type daFollower struct { // share the same namespace). When different, we subscribe to both and merge. dataNamespace []byte - // localDAHeight is only written by catchupLoop and read by followLoop + // localNextDAHeight is only written by catchupLoop and read by followLoop // to determine whether a catchup is needed. - localDAHeight atomic.Uint64 + localNextDAHeight atomic.Uint64 // highestSeenDAHeight is written by followLoop and read by catchupLoop. highestSeenDAHeight atomic.Uint64 @@ -62,7 +62,7 @@ type daFollower struct { headReached atomic.Bool // catchupSignal is sent by followLoop to wake catchupLoop when a new - // height is seen that is above localDAHeight. + // height is seen that is above localNextDAHeight. catchupSignal chan struct{} // daBlockTime is used as a backoff before retrying after errors. @@ -101,7 +101,7 @@ func NewDAFollower(cfg DAFollowerConfig) DAFollower { catchupSignal: make(chan struct{}, 1), daBlockTime: cfg.DABlockTime, } - f.localDAHeight.Store(cfg.StartDAHeight) + f.localNextDAHeight.Store(cfg.StartDAHeight) return f } @@ -114,7 +114,7 @@ func (f *daFollower) Start(ctx context.Context) error { go f.catchupLoop(ctx) f.logger.Info(). - Uint64("start_da_height", f.localDAHeight.Load()). + Uint64("start_da_height", f.localNextDAHeight.Load()). Msg("DA follower started") return nil } @@ -142,7 +142,7 @@ func (f *daFollower) signalCatchup() { } // followLoop subscribes to DA blob events and keeps highestSeenDAHeight up to date. -// When a new height appears above localDAHeight, it wakes the catchup loop. +// When a new height appears above localNextDAHeight, it wakes the catchup loop. func (f *daFollower) followLoop(ctx context.Context) { defer f.wg.Done() @@ -244,11 +244,11 @@ func (f *daFollower) mergeSubscriptions( } // handleSubscriptionEvent processes a subscription event. When the follower is -// caught up (ev.Height == localDAHeight) and blobs are available, it processes +// caught up (ev.Height == localNextDAHeight) and blobs are available, it processes // them inline — avoiding a DA re-fetch round trip. Otherwise, it just updates // highestSeenDAHeight and lets catchupLoop handle retrieval. // -// Uses CAS on localDAHeight to claim exclusive access to processBlobs, +// Uses CAS on localNextDAHeight to claim exclusive access to processBlobs, // preventing concurrent map access with catchupLoop. func (f *daFollower) handleSubscriptionEvent(ctx context.Context, ev datypes.SubscriptionEvent) { // Always record the highest height we've seen from the subscription. @@ -257,24 +257,26 @@ func (f *daFollower) handleSubscriptionEvent(ctx context.Context, ev datypes.Sub // Fast path: try to claim this height for inline processing. // CAS(N, N+1) ensures only one goroutine (followLoop or catchupLoop) // can enter processBlobs for height N. - if len(ev.Blobs) > 0 && f.localDAHeight.CompareAndSwap(ev.Height, ev.Height+1) { + if len(ev.Blobs) > 0 && f.localNextDAHeight.CompareAndSwap(ev.Height, ev.Height+1) { events := f.retriever.ProcessBlobs(ctx, ev.Blobs, ev.Height) for _, event := range events { if err := f.pipeEvent(ctx, event); err != nil { // Roll back so catchupLoop can retry this height. - f.localDAHeight.Store(ev.Height) + f.localNextDAHeight.Store(ev.Height) f.logger.Warn().Err(err).Uint64("da_height", ev.Height). Msg("failed to pipe inline event, catchup will retry") return } } if len(events) != 0 { - f.headReached.Store(true) + if !f.headReached.Load() && f.localNextDAHeight.Load() > f.highestSeenDAHeight.Load() { + f.headReached.Store(true) + } f.logger.Debug().Uint64("da_height", ev.Height).Int("events", len(events)). Msg("processed subscription blobs inline (fast path)") } else { // No complete events (split namespace, waiting for other half). - f.localDAHeight.Store(ev.Height) + f.localNextDAHeight.Store(ev.Height) } return } @@ -297,7 +299,7 @@ func (f *daFollower) updateHighest(height uint64) { } // catchupLoop waits for signals and sequentially retrieves DA heights -// from localDAHeight up to highestSeenDAHeight. +// from localNextDAHeight up to highestSeenDAHeight. func (f *daFollower) catchupLoop(ctx context.Context) { defer f.wg.Done() @@ -314,7 +316,7 @@ func (f *daFollower) catchupLoop(ctx context.Context) { } } -// runCatchup sequentially retrieves from localDAHeight to highestSeenDAHeight. +// runCatchup sequentially retrieves from localNextDAHeight to highestSeenDAHeight. // It handles priority heights first, then sequential heights. func (f *daFollower) runCatchup(ctx context.Context) { for { @@ -324,8 +326,8 @@ func (f *daFollower) runCatchup(ctx context.Context) { // Check for priority heights from P2P hints first. if priorityHeight := f.retriever.PopPriorityHeight(); priorityHeight > 0 { - currentHeight := f.localDAHeight.Load() - if priorityHeight < currentHeight { + nextHeight := f.localNextDAHeight.Load() + if priorityHeight < nextHeight { continue } f.logger.Debug(). @@ -340,7 +342,7 @@ func (f *daFollower) runCatchup(ctx context.Context) { } // Sequential catchup. - local := f.localDAHeight.Load() + local := f.localNextDAHeight.Load() highest := f.highestSeenDAHeight.Load() if local > highest { @@ -350,14 +352,14 @@ func (f *daFollower) runCatchup(ctx context.Context) { } // CAS claims this height prevents followLoop from inline-processing - if !f.localDAHeight.CompareAndSwap(local, local+1) { + if !f.localNextDAHeight.CompareAndSwap(local, local+1) { // followLoop already advanced past this height via inline processing. continue } if err := f.fetchAndPipeHeight(ctx, local); err != nil { // Roll back so we can retry after backoff. - f.localDAHeight.Store(local) + f.localNextDAHeight.Store(local) if !f.waitOnCatchupError(ctx, err, local) { return } diff --git a/block/internal/syncing/da_retriever.go b/block/internal/syncing/da_retriever.go index 782fa9c318..366d3ed30c 100644 --- a/block/internal/syncing/da_retriever.go +++ b/block/internal/syncing/da_retriever.go @@ -7,6 +7,7 @@ import ( "fmt" "slices" "sync" + "sync/atomic" "github.com/rs/zerolog" "google.golang.org/protobuf/proto" @@ -43,12 +44,13 @@ type daRetriever struct { // transient cache, only full event need to be passed to the syncer // on restart, will be refetch as da height is updated by syncer + pendingMu sync.Mutex pendingHeaders map[uint64]*types.SignedHeader pendingData map[uint64]*types.Data // strictMode indicates if the node has seen a valid DAHeaderEnvelope // and should now reject all legacy/unsigned headers. - strictMode bool + strictMode atomic.Bool // priorityMu protects priorityHeights from concurrent access priorityMu sync.Mutex @@ -71,7 +73,6 @@ func NewDARetriever( logger: logger.With().Str("component", "da_retriever").Logger(), pendingHeaders: make(map[uint64]*types.SignedHeader), pendingData: make(map[uint64]*types.Data), - strictMode: false, priorityHeights: make([]uint64, 0), } } @@ -198,41 +199,55 @@ func (r *daRetriever) validateBlobResponse(res datypes.ResultRetrieve, daHeight // This is the public interface used by the DAFollower for inline subscription processing. // // NOT thread-safe: the caller (DAFollower) must ensure exclusive access via CAS -// on localDAHeight before calling this method. +// on localNextDAHeight before calling this method. func (r *daRetriever) ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { return r.processBlobs(ctx, blobs, daHeight) } // processBlobs processes retrieved blobs to extract headers and data and returns height events func (r *daRetriever) processBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { - // Decode all blobs + var decodedHeaders []*types.SignedHeader + var decodedData []*types.Data + + // Decode all blobs first without holding the lock for _, bz := range blobs { if len(bz) == 0 { continue } if header := r.tryDecodeHeader(bz, daHeight); header != nil { - if _, ok := r.pendingHeaders[header.Height()]; ok { - // a (malicious) node may have re-published valid header to another da height (should never happen) - // we can already discard it, only the first one is valid - r.logger.Debug().Uint64("height", header.Height()).Uint64("da_height", daHeight).Msg("header blob already exists for height, discarding") - continue - } - - r.pendingHeaders[header.Height()] = header + decodedHeaders = append(decodedHeaders, header) continue } if data := r.tryDecodeData(bz, daHeight); data != nil { - if _, ok := r.pendingData[data.Height()]; ok { - // a (malicious) node may have re-published valid data to another da height (should never happen) - // we can already discard it, only the first one is valid - r.logger.Debug().Uint64("height", data.Height()).Uint64("da_height", daHeight).Msg("data blob already exists for height, discarding") - continue - } + decodedData = append(decodedData, data) + } + } - r.pendingData[data.Height()] = data + r.pendingMu.Lock() + defer r.pendingMu.Unlock() + + for _, header := range decodedHeaders { + if _, ok := r.pendingHeaders[header.Height()]; ok { + // a (malicious) node may have re-published valid header to another da height (should never happen) + // we can already discard it, only the first one is valid + r.logger.Debug().Uint64("height", header.Height()).Uint64("da_height", daHeight).Msg("header blob already exists for height, discarding") + continue + } + + r.pendingHeaders[header.Height()] = header + } + + for _, data := range decodedData { + if _, ok := r.pendingData[data.Height()]; ok { + // a (malicious) node may have re-published valid data to another da height (should never happen) + // we can already discard it, only the first one is valid + r.logger.Debug().Uint64("height", data.Height()).Uint64("da_height", daHeight).Msg("data blob already exists for height, discarding") + continue } + + r.pendingData[data.Height()] = data } var events []common.DAHeightEvent @@ -294,7 +309,7 @@ func (r *daRetriever) tryDecodeHeader(bz []byte, daHeight uint64) *types.SignedH // Attempt to unmarshal as DAHeaderEnvelope and get the envelope signature if envelopeSignature, err := header.UnmarshalDAEnvelope(bz); err != nil { // If in strict mode, we REQUIRE an envelope. - if r.strictMode { + if r.strictMode.Load() { r.logger.Warn().Err(err).Msg("strict mode is enabled, rejecting non-envelope blob") return nil } @@ -328,14 +343,14 @@ func (r *daRetriever) tryDecodeHeader(bz []byte, daHeight uint64) *types.SignedH isValidEnvelope = true } } - if r.strictMode && !isValidEnvelope { + if r.strictMode.Load() && !isValidEnvelope { r.logger.Warn().Msg("strict mode: rejecting block that is not a fully valid envelope") return nil } // Mode Switch Logic - if isValidEnvelope && !r.strictMode { + if isValidEnvelope && !r.strictMode.Load() { r.logger.Info().Uint64("height", header.Height()).Msg("valid DA envelope detected, switching to STRICT MODE") - r.strictMode = true + r.strictMode.Store(true) } // Legacy blob support implies: strictMode == false AND (!isValidEnvelope). diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index 333a2b556b..bf99181cbb 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -79,7 +79,6 @@ type Syncer struct { p2pHandler p2pHandler raftRetriever *raftRetriever - // DA follower (replaces the old polling daWorkerLoop) daFollower DAFollower // Forced inclusion tracking @@ -209,6 +208,8 @@ func (s *Syncer) Start(ctx context.Context) error { DABlockTime: s.config.DA.BlockTime.Duration, }) if err := s.daFollower.Start(ctx); err != nil { + s.cancel() + s.wg.Wait() return fmt.Errorf("failed to start DA follower: %w", err) } @@ -394,8 +395,6 @@ func (s *Syncer) HasReachedDAHead() bool { return false } -// fetchDAUntilCaughtUp was removed — the DAFollower handles this concern. - // PendingCount returns the number of unprocessed height events in the pipeline. func (s *Syncer) PendingCount() int { return len(s.heightInCh) diff --git a/block/internal/syncing/syncer_backoff_test.go b/block/internal/syncing/syncer_backoff_test.go index bd2d6000fe..b8ebf269e9 100644 --- a/block/internal/syncing/syncer_backoff_test.go +++ b/block/internal/syncing/syncer_backoff_test.go @@ -255,7 +255,7 @@ func TestDAFollower_CatchupThenReachHead(t *testing.T) { } // TestDAFollower_InlineProcessing verifies the fast path: when the subscription -// delivers blobs at the current localDAHeight, handleSubscriptionEvent processes +// delivers blobs at the current localNextDAHeight, handleSubscriptionEvent processes // them inline via ProcessBlobs (not RetrieveFromDA). func TestDAFollower_InlineProcessing(t *testing.T) { t.Run("processes_blobs_inline_when_caught_up", func(t *testing.T) { @@ -285,7 +285,7 @@ func TestDAFollower_InlineProcessing(t *testing.T) { daRetriever.On("ProcessBlobs", mock.Anything, blobs, uint64(10)). Return(expectedEvents).Once() - // Simulate subscription event at the current localDAHeight + // Simulate subscription event at the current localNextDAHeight follower.handleSubscriptionEvent(t.Context(), datypes.SubscriptionEvent{ Height: 10, Blobs: blobs, @@ -294,7 +294,7 @@ func TestDAFollower_InlineProcessing(t *testing.T) { // Verify: ProcessBlobs was called, events were piped, height advanced require.Len(t, pipedEvents, 1, "should pipe 1 event from inline processing") assert.Equal(t, uint64(10), pipedEvents[0].DaHeight) - assert.Equal(t, uint64(11), follower.localDAHeight.Load(), "localDAHeight should advance past processed height") + assert.Equal(t, uint64(11), follower.localNextDAHeight.Load(), "localNextDAHeight should advance past processed height") assert.True(t, follower.HasReachedHead(), "should mark head as reached after inline processing") }) @@ -324,7 +324,7 @@ func TestDAFollower_InlineProcessing(t *testing.T) { // ProcessBlobs should NOT have been called daRetriever.AssertNotCalled(t, "ProcessBlobs", mock.Anything, mock.Anything, mock.Anything) - assert.Equal(t, uint64(10), follower.localDAHeight.Load(), "localDAHeight should not change") + assert.Equal(t, uint64(10), follower.localNextDAHeight.Load(), "localNextDAHeight should not change") assert.Equal(t, uint64(15), follower.highestSeenDAHeight.Load(), "highestSeen should be updated") }) @@ -350,7 +350,7 @@ func TestDAFollower_InlineProcessing(t *testing.T) { // ProcessBlobs should NOT have been called daRetriever.AssertNotCalled(t, "ProcessBlobs", mock.Anything, mock.Anything, mock.Anything) - assert.Equal(t, uint64(10), follower.localDAHeight.Load(), "localDAHeight should not change") + assert.Equal(t, uint64(10), follower.localNextDAHeight.Load(), "localNextDAHeight should not change") assert.Equal(t, uint64(10), follower.highestSeenDAHeight.Load(), "highestSeen should be updated") }) } diff --git a/block/internal/syncing/syncer_benchmark_test.go b/block/internal/syncing/syncer_benchmark_test.go index a69da71507..9536808657 100644 --- a/block/internal/syncing/syncer_benchmark_test.go +++ b/block/internal/syncing/syncer_benchmark_test.go @@ -43,8 +43,8 @@ func BenchmarkSyncerIO(b *testing.B) { fixt := newBenchFixture(b, spec.heights, spec.shuffledTx, spec.daDelay, spec.execDelay, true) // run both loops - ctx := b.Context() - go fixt.s.processLoop(ctx) + runCtx := fixt.s.ctx + go fixt.s.processLoop(runCtx) // Create a DAFollower to drive DA retrieval. follower := NewDAFollower(DAFollowerConfig{ @@ -56,12 +56,12 @@ func BenchmarkSyncerIO(b *testing.B) { DABlockTime: 0, }).(*daFollower) follower.highestSeenDAHeight.Store(spec.heights + daHeightOffset) - go follower.runCatchup(ctx) + go follower.runCatchup(runCtx) - fixt.s.startSyncWorkers(ctx) + fixt.s.startSyncWorkers(runCtx) require.Eventually(b, func() bool { - processedHeight, _ := fixt.s.store.Height(ctx) + processedHeight, _ := fixt.s.store.Height(b.Context()) return processedHeight == spec.heights }, 5*time.Second, 50*time.Microsecond) fixt.s.cancel() @@ -75,7 +75,7 @@ func BenchmarkSyncerIO(b *testing.B) { require.Len(b, fixt.s.heightInCh, 0) assert.Equal(b, spec.heights+daHeightOffset, fixt.s.daRetrieverHeight) - gotStoreHeight, err := fixt.s.store.Height(ctx) + gotStoreHeight, err := fixt.s.store.Height(b.Context()) require.NoError(b, err) assert.Equal(b, spec.heights, gotStoreHeight) } diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 64ccea439d..716924e2ae 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -436,7 +436,7 @@ func TestSyncLoopPersistState(t *testing.T) { requireEmptyChan(t, errorCh) t.Log("sync workers on instance1 completed") - require.Equal(t, myFutureDAHeight, follower1.localDAHeight.Load()) + require.Equal(t, myFutureDAHeight, follower1.localNextDAHeight.Load()) // wait for all events consumed require.NoError(t, cm.SaveToStore()) diff --git a/tools/local-da/local.go b/tools/local-da/local.go index 07d6876f9d..fb47f6c3b8 100644 --- a/tools/local-da/local.go +++ b/tools/local-da/local.go @@ -222,11 +222,21 @@ func (d *LocalDA) SubmitWithOptions(ctx context.Context, blobs []datypes.Blob, g ids := make([]datypes.ID, len(blobs)) d.height += 1 d.timestamps[d.height] = d.monotonicTime() + + nspace, _ := libshare.NewNamespaceFromBytes(ns) + rpcBlobs := make([]*blobrpc.Blob, len(blobs)) + for i, blob := range blobs { ids[i] = append(d.nextID(), d.getHash(blob)...) d.data[d.height] = append(d.data[d.height], kvp{ids[i], blob}) + + if b, err := blobrpc.NewBlobV0(nspace, blob); err == nil { + rpcBlobs[i] = b + } } + d.blobData[d.height] = rpcBlobs + d.notifySubscribers(d.height) d.logger.Info().Uint64("newHeight", d.height).Int("count", len(ids)).Msg("SubmitWithOptions successful") return ids, nil @@ -253,11 +263,21 @@ func (d *LocalDA) Submit(ctx context.Context, blobs []datypes.Blob, gasPrice flo ids := make([]datypes.ID, len(blobs)) d.height += 1 d.timestamps[d.height] = d.monotonicTime() + + nspace, _ := libshare.NewNamespaceFromBytes(ns) + rpcBlobs := make([]*blobrpc.Blob, len(blobs)) + for i, blob := range blobs { ids[i] = append(d.nextID(), d.getHash(blob)...) d.data[d.height] = append(d.data[d.height], kvp{ids[i], blob}) + + if b, err := blobrpc.NewBlobV0(nspace, blob); err == nil { + rpcBlobs[i] = b + } } + d.blobData[d.height] = rpcBlobs + d.notifySubscribers(d.height) d.logger.Info().Uint64("newHeight", d.height).Int("count", len(ids)).Msg("Submit successful") return ids, nil From 896c6ad6a027eddefd326cb0d8efdee213976c2b Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 5 Mar 2026 13:47:27 +0100 Subject: [PATCH 11/17] Review feedback --- block/internal/syncing/da_follower.go | 12 +++++++----- block/internal/syncing/da_retriever_strict_test.go | 6 +++--- test/testda/dummy.go | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/block/internal/syncing/da_follower.go b/block/internal/syncing/da_follower.go index 71b15f570e..443c3b8333 100644 --- a/block/internal/syncing/da_follower.go +++ b/block/internal/syncing/da_follower.go @@ -325,11 +325,13 @@ func (f *daFollower) runCatchup(ctx context.Context) { } // Check for priority heights from P2P hints first. - if priorityHeight := f.retriever.PopPriorityHeight(); priorityHeight > 0 { - nextHeight := f.localNextDAHeight.Load() - if priorityHeight < nextHeight { - continue - } + // We drain stale hints to avoid a tight CPU loop if many are queued. + priorityHeight := f.retriever.PopPriorityHeight() + for priorityHeight > 0 && priorityHeight < f.localNextDAHeight.Load() { + priorityHeight = f.retriever.PopPriorityHeight() + } + + if priorityHeight > 0 { f.logger.Debug(). Uint64("da_height", priorityHeight). Msg("fetching priority DA height from P2P hint") diff --git a/block/internal/syncing/da_retriever_strict_test.go b/block/internal/syncing/da_retriever_strict_test.go index af1046dba8..a6d5140d97 100644 --- a/block/internal/syncing/da_retriever_strict_test.go +++ b/block/internal/syncing/da_retriever_strict_test.go @@ -70,21 +70,21 @@ func TestDARetriever_StrictEnvelopeMode_Switch(t *testing.T) { // --- Test Scenario --- // A. Initial State: StrictMode is false. Legacy blob should be accepted. - assert.False(t, r.strictMode) + assert.False(t, r.strictMode.Load()) decodedLegacy := r.tryDecodeHeader(legacyBlob, 100) require.NotNil(t, decodedLegacy) assert.Equal(t, uint64(1), decodedLegacy.Height()) // StrictMode should still be false because it was a legacy blob - assert.False(t, r.strictMode) + assert.False(t, r.strictMode.Load()) // B. Receiving Envelope: Should be accepted and Switch StrictMode to true. decodedEnvelope := r.tryDecodeHeader(envelopeBlob, 101) require.NotNil(t, decodedEnvelope) assert.Equal(t, uint64(2), decodedEnvelope.Height()) - assert.True(t, r.strictMode, "retriever should have switched to strict mode") + assert.True(t, r.strictMode.Load(), "retriever should have switched to strict mode") // C. Receiving Legacy again: Should be REJECTED now. // We reuse the same legacyBlob (or a new one, doesn't matter, structure is legacy). diff --git a/test/testda/dummy.go b/test/testda/dummy.go index 7f29140b28..44fc565c60 100644 --- a/test/testda/dummy.go +++ b/test/testda/dummy.go @@ -70,10 +70,10 @@ func (d *DummyDA) Subscribe(ctx context.Context, _ []byte) (<-chan datypes.Subsc for i, s := range d.subscribers { if s == sub { d.subscribers = append(d.subscribers[:i], d.subscribers[i+1:]...) + close(ch) break } } - close(ch) }() return ch, nil From 191ae0127ed1e2a26cdfa8b9b5ebe57797a6460c Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Fri, 6 Mar 2026 12:38:23 +0100 Subject: [PATCH 12/17] Review feedbac --- apps/evm/cmd/run.go | 1 + block/internal/syncing/da_follower.go | 3 ++- block/internal/syncing/syncer.go | 22 ++++++++++++++-------- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/apps/evm/cmd/run.go b/apps/evm/cmd/run.go index 73b809de46..0136e1eb6c 100644 --- a/apps/evm/cmd/run.go +++ b/apps/evm/cmd/run.go @@ -64,6 +64,7 @@ var RunCmd = &cobra.Command{ if err != nil { return fmt.Errorf("failed to create blob client: %w", err) } + defer blobClient.Close() daClient := block.NewDAClient(blobClient, nodeConfig, logger) diff --git a/block/internal/syncing/da_follower.go b/block/internal/syncing/da_follower.go index 443c3b8333..99b51db593 100644 --- a/block/internal/syncing/da_follower.go +++ b/block/internal/syncing/da_follower.go @@ -111,6 +111,7 @@ func (f *daFollower) Start(ctx context.Context) error { f.wg.Add(2) go f.followLoop(ctx) + f.signalCatchup() go f.catchupLoop(ctx) f.logger.Info(). @@ -347,7 +348,7 @@ func (f *daFollower) runCatchup(ctx context.Context) { local := f.localNextDAHeight.Load() highest := f.highestSeenDAHeight.Load() - if local > highest { + if highest > 0 && local > highest { // Caught up. f.headReached.Store(true) return diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index bf99181cbb..f906be7150 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -161,11 +161,17 @@ func (s *Syncer) SetBlockSyncer(bs BlockSyncer) { } // Start begins the syncing component -func (s *Syncer) Start(ctx context.Context) error { +func (s *Syncer) Start(ctx context.Context) (err error) { ctx, cancel := context.WithCancel(ctx) s.ctx, s.cancel = ctx, cancel - if err := s.initializeState(); err != nil { + defer func() { + if err != nil { + _ = s.Stop() + } + }() + + if err = s.initializeState(); err != nil { return fmt.Errorf("failed to initialize syncer state: %w", err) } @@ -177,14 +183,16 @@ func (s *Syncer) Start(ctx context.Context) error { s.fiRetriever = da.NewForcedInclusionRetriever(s.daClient, s.logger, s.config, s.genesis.DAStartHeight, s.genesis.DAEpochForcedInclusion) s.p2pHandler = NewP2PHandler(s.headerStore, s.dataStore, s.cache, s.genesis, s.logger) - if currentHeight, err := s.store.Height(ctx); err != nil { - s.logger.Error().Err(err).Msg("failed to set initial processed height for p2p handler") + + currentHeight, initErr := s.store.Height(ctx) + if initErr != nil { + s.logger.Error().Err(initErr).Msg("failed to set initial processed height for p2p handler") } else { s.p2pHandler.SetProcessedHeight(currentHeight) } if s.raftRetriever != nil { - if err := s.raftRetriever.Start(ctx); err != nil { + if err = s.raftRetriever.Start(ctx); err != nil { return fmt.Errorf("start raft retriever: %w", err) } } @@ -207,9 +215,7 @@ func (s *Syncer) Start(ctx context.Context) error { StartDAHeight: s.daRetrieverHeight.Load(), DABlockTime: s.config.DA.BlockTime.Duration, }) - if err := s.daFollower.Start(ctx); err != nil { - s.cancel() - s.wg.Wait() + if err = s.daFollower.Start(ctx); err != nil { return fmt.Errorf("failed to start DA follower: %w", err) } From 31b52b38fcafccd3d9a7024544f3911fcc60b9df Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 9 Mar 2026 16:12:47 +0100 Subject: [PATCH 13/17] Linter --- block/components.go | 2 +- block/internal/syncing/syncer.go | 8 ++++---- block/internal/syncing/syncer_test.go | 4 ++-- node/failover.go | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/block/components.go b/block/components.go index c54f8d554d..71b6f60523 100644 --- a/block/components.go +++ b/block/components.go @@ -114,7 +114,7 @@ func (bc *Components) Stop() error { } } if bc.Syncer != nil { - if err := bc.Syncer.Stop(); err != nil { + if err := bc.Syncer.Stop(context.Background()); err != nil { errs = errors.Join(errs, fmt.Errorf("failed to stop syncer: %w", err)) } } diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index bddcaba6e4..a41f2a5478 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -165,9 +165,9 @@ func (s *Syncer) Start(ctx context.Context) (err error) { ctx, cancel := context.WithCancel(ctx) s.ctx, s.cancel = ctx, cancel - defer func() { + defer func() { //nolint: contextcheck // use new context as parent can be cancelled already if err != nil { - _ = s.Stop() + _ = s.Stop(context.Background()) } }() @@ -226,7 +226,7 @@ func (s *Syncer) Start(ctx context.Context) (err error) { } // Stop shuts down the syncing component -func (s *Syncer) Stop() error { +func (s *Syncer) Stop(ctx context.Context) error { if s.cancel == nil { return nil } @@ -244,7 +244,7 @@ func (s *Syncer) Stop() error { // Skip draining if we're shutting down due to a critical error (e.g. execution // client unavailable). if !s.hasCriticalError.Load() { - drainCtx, drainCancel := context.WithTimeout(context.Background(), 5*time.Second) + drainCtx, drainCancel := context.WithTimeout(ctx, 5*time.Second) defer drainCancel() drained := 0 diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 781562fbfc..771e71c284 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -1156,7 +1156,7 @@ func TestSyncer_Stop_SkipsDrainOnCriticalError(t *testing.T) { // Stop must complete quickly — no drain, no ExecuteTxs calls done := make(chan struct{}) go func() { - _ = s.Stop() + _ = s.Stop(t.Context()) close(done) }() @@ -1225,7 +1225,7 @@ func TestSyncer_Stop_DrainWorksWithoutCriticalError(t *testing.T) { // hasCriticalError is false (default) — drain should process events including ExecuteTxs s.wg.Go(func() {}) - _ = s.Stop() + _ = s.Stop(t.Context()) // Verify ExecuteTxs was actually called during drain mockExec.AssertExpectations(t) diff --git a/node/failover.go b/node/failover.go index f6fd09a3a0..c60e27a5f4 100644 --- a/node/failover.go +++ b/node/failover.go @@ -255,7 +255,7 @@ func (f *failoverState) runCatchupPhase(ctx context.Context) error { if err := f.bc.Syncer.Start(ctx); err != nil { return fmt.Errorf("catchup syncer start: %w", err) } - defer f.bc.Syncer.Stop() // nolint:errcheck // not critical + defer f.bc.Syncer.Stop(context.Background()) // nolint:errcheck,contextcheck // not critical caughtUp, err := f.waitForCatchup(ctx) if err != nil { From 06c78649d376eafe09b7ac16efca97468ea67437 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 5 Mar 2026 11:53:36 +0100 Subject: [PATCH 14/17] Use subscription with forced inclusion; refactorings --- block/internal/common/event.go | 21 +- block/internal/da/async_block_retriever.go | 227 +++++----- .../internal/da/async_block_retriever_test.go | 245 +++++++---- .../internal/da/forced_inclusion_retriever.go | 11 +- .../da/forced_inclusion_retriever_test.go | 30 +- block/internal/da/subscriber.go | 391 +++++++++++++++++ block/internal/syncing/da_follower.go | 402 ++++-------------- block/internal/syncing/da_retriever.go | 115 ++--- block/internal/syncing/da_retriever_mock.go | 84 ---- .../internal/syncing/da_retriever_tracing.go | 8 - .../syncing/da_retriever_tracing_test.go | 4 - block/internal/syncing/raft_retriever.go | 22 +- block/internal/syncing/syncer.go | 10 +- block/internal/syncing/syncer_backoff_test.go | 74 ++-- .../internal/syncing/syncer_benchmark_test.go | 19 +- .../syncing/syncer_forced_inclusion_test.go | 8 +- block/internal/syncing/syncer_test.go | 184 +++----- block/public.go | 3 +- pkg/sequencers/based/sequencer.go | 2 +- pkg/sequencers/based/sequencer_test.go | 6 + pkg/sequencers/single/sequencer.go | 4 +- pkg/sequencers/single/sequencer_test.go | 18 + 22 files changed, 954 insertions(+), 934 deletions(-) create mode 100644 block/internal/da/subscriber.go diff --git a/block/internal/common/event.go b/block/internal/common/event.go index f02a181de8..3cc18f4641 100644 --- a/block/internal/common/event.go +++ b/block/internal/common/event.go @@ -1,6 +1,10 @@ package common -import "github.com/evstack/ev-node/types" +import ( + "context" + + "github.com/evstack/ev-node/types" +) // EventSource represents the origin of a block event type EventSource string @@ -24,3 +28,18 @@ type DAHeightEvent struct { // Optional DA height hints from P2P. first is the DA height hint for the header, second is the DA height hint for the data DaHeightHints [2]uint64 } + +// EventSink receives parsed DA events with backpressure support. +type EventSink interface { + PipeEvent(ctx context.Context, event DAHeightEvent) error +} + +// EventSinkFunc adapts a plain function to the EventSink interface. +// Useful in tests: +// +// sink := common.EventSinkFunc(func(ctx context.Context, ev common.DAHeightEvent) error { return nil }) +type EventSinkFunc func(ctx context.Context, event DAHeightEvent) error + +func (f EventSinkFunc) PipeEvent(ctx context.Context, event DAHeightEvent) error { + return f(ctx, event) +} diff --git a/block/internal/da/async_block_retriever.go b/block/internal/da/async_block_retriever.go index 4fd81d3e9c..a0dcf7a578 100644 --- a/block/internal/da/async_block_retriever.go +++ b/block/internal/da/async_block_retriever.go @@ -4,8 +4,6 @@ import ( "context" "errors" "fmt" - "sync" - "sync/atomic" "time" ds "github.com/ipfs/go-datastore" @@ -14,14 +12,13 @@ import ( "github.com/rs/zerolog" "google.golang.org/protobuf/proto" - "github.com/evstack/ev-node/pkg/config" datypes "github.com/evstack/ev-node/pkg/da/types" pb "github.com/evstack/ev-node/types/pb/evnode/v1" ) // AsyncBlockRetriever provides background prefetching of DA blocks type AsyncBlockRetriever interface { - Start() + Start(ctx context.Context) Stop() GetCachedBlock(ctx context.Context, daHeight uint64) (*BlockData, error) UpdateCurrentHeight(height uint64) @@ -35,29 +32,23 @@ type BlockData struct { } // asyncBlockRetriever handles background prefetching of individual DA blocks -// from a specific namespace. +// from a specific namespace. Wraps a da.Subscriber for the subscription +// plumbing and implements SubscriberHandler for caching. type asyncBlockRetriever struct { - client Client - logger zerolog.Logger - namespace []byte - daStartHeight uint64 + subscriber *Subscriber + client Client + logger zerolog.Logger + namespace []byte // In-memory cache for prefetched block data cache ds.Batching - // Background fetcher control - ctx context.Context - cancel context.CancelFunc - wg sync.WaitGroup - - // Current DA height tracking (accessed atomically) - currentDAHeight atomic.Uint64 + // Current DA height tracking (accessed atomically via subscriber). + // Updated externally via UpdateCurrentHeight. + daStartHeight uint64 - // Prefetch window - how many blocks ahead to prefetch + // Prefetch window - how many blocks ahead to speculatively fetch. prefetchWindow uint64 - - // Polling interval for checking new DA heights - pollInterval time.Duration } // NewAsyncBlockRetriever creates a new async block retriever with in-memory cache. @@ -65,61 +56,69 @@ func NewAsyncBlockRetriever( client Client, logger zerolog.Logger, namespace []byte, - config config.Config, + daBlockTime time.Duration, daStartHeight uint64, prefetchWindow uint64, ) AsyncBlockRetriever { if prefetchWindow == 0 { - prefetchWindow = 10 // Default: prefetch next 10 blocks + prefetchWindow = 10 } - ctx, cancel := context.WithCancel(context.Background()) - - fetcher := &asyncBlockRetriever{ + f := &asyncBlockRetriever{ client: client, logger: logger.With().Str("component", "async_block_retriever").Logger(), namespace: namespace, daStartHeight: daStartHeight, cache: dsync.MutexWrap(ds.NewMapDatastore()), - ctx: ctx, - cancel: cancel, prefetchWindow: prefetchWindow, - pollInterval: config.DA.BlockTime.Duration, } - fetcher.currentDAHeight.Store(daStartHeight) - return fetcher + + var namespaces [][]byte + if len(namespace) > 0 { + namespaces = [][]byte{namespace} + } + + f.subscriber = NewSubscriber(SubscriberConfig{ + Client: client, + Logger: logger, + Namespaces: namespaces, + DABlockTime: daBlockTime, + Handler: f, + }) + f.subscriber.SetStartHeight(daStartHeight) + + return f } -// Start begins the background prefetching process. -func (f *asyncBlockRetriever) Start() { - f.wg.Add(1) - go f.backgroundFetchLoop() +// Start begins the subscription follow loop and catchup loop. +func (f *asyncBlockRetriever) Start(ctx context.Context) { + if err := f.subscriber.Start(ctx); err != nil { + f.logger.Warn().Err(err).Msg("failed to start subscriber") + } f.logger.Debug(). Uint64("da_start_height", f.daStartHeight). Uint64("prefetch_window", f.prefetchWindow). - Dur("poll_interval", f.pollInterval). Msg("async block retriever started") } -// Stop gracefully stops the background prefetching process. +// Stop gracefully stops the background goroutines. func (f *asyncBlockRetriever) Stop() { f.logger.Debug().Msg("stopping async block retriever") - f.cancel() - f.wg.Wait() + f.subscriber.Stop() } -// UpdateCurrentHeight updates the current DA height for prefetching. +// UpdateCurrentHeight updates the current DA height. func (f *asyncBlockRetriever) UpdateCurrentHeight(height uint64) { - // Use atomic compare-and-swap to update only if the new height is greater for { - current := f.currentDAHeight.Load() + current := f.subscriber.LocalDAHeight() if height <= current { return } - if f.currentDAHeight.CompareAndSwap(current, height) { + if f.subscriber.CompareAndSwapLocalHeight(current, height) { f.logger.Debug(). Uint64("new_height", height). Msg("updated current DA height") + f.cleanupOldBlocks(height) return } } @@ -149,7 +148,6 @@ func (f *asyncBlockRetriever) GetCachedBlock(ctx context.Context, daHeight uint6 return nil, fmt.Errorf("failed to get cached block: %w", err) } - // Deserialize the cached block var pbBlock pb.BlockData if err := proto.Unmarshal(data, &pbBlock); err != nil { return nil, fmt.Errorf("failed to unmarshal cached block: %w", err) @@ -169,138 +167,107 @@ func (f *asyncBlockRetriever) GetCachedBlock(ctx context.Context, daHeight uint6 return block, nil } -// backgroundFetchLoop runs in the background and prefetches blocks ahead of time. -func (f *asyncBlockRetriever) backgroundFetchLoop() { - defer f.wg.Done() - - ticker := time.NewTicker(f.pollInterval) - defer ticker.Stop() +// --------------------------------------------------------------------------- +// SubscriberHandler implementation +// --------------------------------------------------------------------------- - for { - select { - case <-f.ctx.Done(): - return - case <-ticker.C: - f.prefetchBlocks() - } +// HandleEvent caches blobs from the subscription inline. +func (f *asyncBlockRetriever) HandleEvent(ctx context.Context, ev datypes.SubscriptionEvent) { + if len(ev.Blobs) > 0 { + f.cacheBlock(ctx, ev.Height, ev.Blobs) } } -// prefetchBlocks prefetches blocks within the prefetch window. -func (f *asyncBlockRetriever) prefetchBlocks() { - if len(f.namespace) == 0 { - return - } - - currentHeight := f.currentDAHeight.Load() - - // Prefetch upcoming blocks - for i := uint64(0); i < f.prefetchWindow; i++ { - targetHeight := currentHeight + i - - // Check if already cached - key := newBlockDataKey(targetHeight) - _, err := f.cache.Get(f.ctx, key) - if err == nil { - // Already cached - continue +// HandleCatchup fetches a single height via Retrieve and caches it. +// Also applies the prefetch window for speculative forward fetching. +func (f *asyncBlockRetriever) HandleCatchup(ctx context.Context, height uint64) error { + f.fetchAndCacheBlock(ctx, height) + + // Speculatively prefetch ahead. + highest := f.subscriber.HighestSeenDAHeight() + target := highest + f.prefetchWindow + for h := height + 1; h <= target; h++ { + if err := ctx.Err(); err != nil { + return err } - - // Fetch and cache the block - f.fetchAndCacheBlock(targetHeight) + key := newBlockDataKey(h) + if _, err := f.cache.Get(ctx, key); err == nil { + continue // Already cached. + } + f.fetchAndCacheBlock(ctx, h) } - // Clean up old blocks from cache to prevent memory growth - f.cleanupOldBlocks(currentHeight) + return nil } -// fetchAndCacheBlock fetches a block and stores it in the cache. -func (f *asyncBlockRetriever) fetchAndCacheBlock(height uint64) { - f.logger.Debug(). - Uint64("height", height). - Msg("prefetching block") +// --------------------------------------------------------------------------- +// Cache helpers +// --------------------------------------------------------------------------- - result := f.client.Retrieve(f.ctx, height, f.namespace) +// fetchAndCacheBlock fetches a block via Retrieve and caches it. +func (f *asyncBlockRetriever) fetchAndCacheBlock(ctx context.Context, height uint64) { + f.logger.Debug().Uint64("height", height).Msg("prefetching block") - block := &BlockData{ - Height: height, - Timestamp: result.Timestamp, - Blobs: [][]byte{}, - } + result := f.client.Retrieve(ctx, height, f.namespace) switch result.Code { case datypes.StatusHeightFromFuture: - f.logger.Debug(). - Uint64("height", height). - Msg("block height not yet available - will retry") + f.logger.Debug().Uint64("height", height).Msg("block height not yet available - will retry") return case datypes.StatusNotFound: - f.logger.Debug(). - Uint64("height", height). - Msg("no blobs at height") - // Cache empty result to avoid re-fetching + f.cacheBlock(ctx, height, nil) case datypes.StatusSuccess: - // Process each blob + blobs := make([][]byte, 0, len(result.Data)) for _, blob := range result.Data { if len(blob) > 0 { - block.Blobs = append(block.Blobs, blob) + blobs = append(blobs, blob) } } - f.logger.Debug(). - Uint64("height", height). - Int("blob_count", len(result.Data)). - Msg("processed blobs for prefetch") + f.cacheBlock(ctx, height, blobs) default: f.logger.Debug(). Uint64("height", height). Str("status", result.Message). Msg("failed to retrieve block - will retry") - return + } +} + +// cacheBlock serializes and stores a block in the in-memory cache. +func (f *asyncBlockRetriever) cacheBlock(ctx context.Context, height uint64, blobs [][]byte) { + if blobs == nil { + blobs = [][]byte{} } - // Serialize and cache the block pbBlock := &pb.BlockData{ - Height: block.Height, - Timestamp: block.Timestamp.UnixNano(), - Blobs: block.Blobs, + Height: height, + Timestamp: time.Now().UnixNano(), + Blobs: blobs, } data, err := proto.Marshal(pbBlock) if err != nil { - f.logger.Error(). - Err(err). - Uint64("height", height). - Msg("failed to marshal block for caching") + f.logger.Error().Err(err).Uint64("height", height).Msg("failed to marshal block for caching") return } key := newBlockDataKey(height) - err = f.cache.Put(f.ctx, key, data) - if err != nil { - f.logger.Error(). - Err(err). - Uint64("height", height). - Msg("failed to cache block") + if err := f.cache.Put(ctx, key, data); err != nil { + f.logger.Error().Err(err).Uint64("height", height).Msg("failed to cache block") return } - f.logger.Debug(). - Uint64("height", height). - Int("blob_count", len(block.Blobs)). - Msg("successfully prefetched and cached block") + f.logger.Debug().Uint64("height", height).Int("blob_count", len(blobs)).Msg("cached block") } -// cleanupOldBlocks removes blocks older than a threshold from cache. +// cleanupOldBlocks removes blocks older than currentHeight − prefetchWindow. func (f *asyncBlockRetriever) cleanupOldBlocks(currentHeight uint64) { - // Remove blocks older than current - prefetchWindow if currentHeight < f.prefetchWindow { return } cleanupThreshold := currentHeight - f.prefetchWindow - // Query all keys query := dsq.Query{Prefix: "/block/"} - results, err := f.cache.Query(f.ctx, query) + results, err := f.cache.Query(context.Background(), query) if err != nil { f.logger.Debug().Err(err).Msg("failed to query cache for cleanup") return @@ -313,7 +280,6 @@ func (f *asyncBlockRetriever) cleanupOldBlocks(currentHeight uint64) { } key := ds.NewKey(result.Key) - // Extract height from key var height uint64 _, err := fmt.Sscanf(key.String(), "/block/%d", &height) if err != nil { @@ -321,11 +287,8 @@ func (f *asyncBlockRetriever) cleanupOldBlocks(currentHeight uint64) { } if height < cleanupThreshold { - if err := f.cache.Delete(f.ctx, key); err != nil { - f.logger.Debug(). - Err(err). - Uint64("height", height). - Msg("failed to delete old block from cache") + if err := f.cache.Delete(context.Background(), key); err != nil { + f.logger.Debug().Err(err).Uint64("height", height).Msg("failed to delete old block from cache") } } } diff --git a/block/internal/da/async_block_retriever_test.go b/block/internal/da/async_block_retriever_test.go index dcfbce84d1..d9c986e4e8 100644 --- a/block/internal/da/async_block_retriever_test.go +++ b/block/internal/da/async_block_retriever_test.go @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" - "github.com/evstack/ev-node/pkg/config" datypes "github.com/evstack/ev-node/pkg/da/types" mocks "github.com/evstack/ev-node/test/mocks" pb "github.com/evstack/ev-node/types/pb/evnode/v1" @@ -21,7 +20,7 @@ func TestAsyncBlockRetriever_GetCachedBlock_NoNamespace(t *testing.T) { client := &mocks.MockClient{} logger := zerolog.Nop() - fetcher := NewAsyncBlockRetriever(client, logger, nil, config.DefaultConfig(), 100, 10) + fetcher := NewAsyncBlockRetriever(client, logger, nil, 100*time.Millisecond, 100, 10) ctx := context.Background() block, err := fetcher.GetCachedBlock(ctx, 100) @@ -34,7 +33,7 @@ func TestAsyncBlockRetriever_GetCachedBlock_CacheMiss(t *testing.T) { fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() logger := zerolog.Nop() - fetcher := NewAsyncBlockRetriever(client, logger, fiNs, config.DefaultConfig(), 100, 10) + fetcher := NewAsyncBlockRetriever(client, logger, fiNs, 100*time.Millisecond, 100, 10) ctx := context.Background() @@ -44,141 +43,143 @@ func TestAsyncBlockRetriever_GetCachedBlock_CacheMiss(t *testing.T) { assert.Nil(t, block) // Cache miss } -func TestAsyncBlockRetriever_FetchAndCache(t *testing.T) { +func TestAsyncBlockRetriever_SubscriptionDrivenCaching(t *testing.T) { + // Test that blobs arriving via subscription are cached inline. testBlobs := [][]byte{ []byte("tx1"), []byte("tx2"), - []byte("tx3"), } client := &mocks.MockClient{} fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() - // Mock Retrieve call for height 100 - client.On("Retrieve", mock.Anything, uint64(100), fiNs).Return(datypes.ResultRetrieve{ - BaseResult: datypes.BaseResult{ - Code: datypes.StatusSuccess, - Timestamp: time.Unix(1000, 0), - }, - Data: testBlobs, - }).Once() - - // Mock other heights that will be prefetched - for height := uint64(101); height <= 109; height++ { - client.On("Retrieve", mock.Anything, height, fiNs).Return(datypes.ResultRetrieve{ - BaseResult: datypes.BaseResult{Code: datypes.StatusNotFound}, - }).Maybe() + // Create a subscription channel that delivers one event then blocks. + subCh := make(chan datypes.SubscriptionEvent, 1) + subCh <- datypes.SubscriptionEvent{ + Height: 100, + Blobs: testBlobs, } + client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() + // Catchup loop may call Retrieve for heights beyond 100 — stub those. + client.On("Retrieve", mock.Anything, mock.Anything, fiNs).Return(datypes.ResultRetrieve{ + BaseResult: datypes.BaseResult{Code: datypes.StatusHeightFromFuture}, + }).Maybe() + + // On second subscribe (after watchdog timeout) just block forever. + blockCh := make(chan datypes.SubscriptionEvent) + client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() + logger := zerolog.Nop() - // Use a short poll interval for faster test execution - cfg := config.DefaultConfig() - cfg.DA.BlockTime.Duration = 100 * time.Millisecond - fetcher := NewAsyncBlockRetriever(client, logger, fiNs, cfg, 100, 10) - fetcher.Start() - defer fetcher.Stop() + fetcher := NewAsyncBlockRetriever(client, logger, fiNs, 200*time.Millisecond, 100, 5) - // Update current height to trigger prefetch - fetcher.UpdateCurrentHeight(100) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + fetcher.Start(ctx) + defer fetcher.Stop() - // Wait for the background fetch to complete by polling the cache - ctx := context.Background() + // Wait for the subscription event to be processed. var block *BlockData var err error - - // Poll for up to 2 seconds for the block to be cached for range 40 { block, err = fetcher.GetCachedBlock(ctx, 100) require.NoError(t, err) if block != nil { break } - time.Sleep(50 * time.Millisecond) + time.Sleep(25 * time.Millisecond) } - require.NotNil(t, block, "block should be cached after background fetch") + require.NotNil(t, block, "block should be cached after subscription event") assert.Equal(t, uint64(100), block.Height) - assert.Equal(t, 3, len(block.Blobs)) - for i, tb := range testBlobs { - assert.Equal(t, tb, block.Blobs[i]) - } + assert.Equal(t, 2, len(block.Blobs)) + assert.Equal(t, []byte("tx1"), block.Blobs[0]) + assert.Equal(t, []byte("tx2"), block.Blobs[1]) } -func TestAsyncBlockRetriever_BackgroundPrefetch(t *testing.T) { - testBlobs := [][]byte{ - []byte("tx1"), - []byte("tx2"), - } +func TestAsyncBlockRetriever_CatchupFillsGaps(t *testing.T) { + // When subscription reports height 105 but current is 100, + // catchup loop should Retrieve heights 100-114 (100 + prefetch window). + testBlobs := [][]byte{[]byte("gap-tx")} client := &mocks.MockClient{} fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() - // Mock for heights 100-110 (current + prefetch window) - for height := uint64(100); height <= 110; height++ { - if height == 105 { - client.On("Retrieve", mock.Anything, height, fiNs).Return(datypes.ResultRetrieve{ - BaseResult: datypes.BaseResult{ - Code: datypes.StatusSuccess, - Timestamp: time.Unix(2000, 0), - }, - Data: testBlobs, - }).Maybe() - } else { - client.On("Retrieve", mock.Anything, height, fiNs).Return(datypes.ResultRetrieve{ - BaseResult: datypes.BaseResult{Code: datypes.StatusNotFound}, - }).Maybe() - } - } + // Subscription delivers height 105 (no blobs — just a signal). + subCh := make(chan datypes.SubscriptionEvent, 1) + subCh <- datypes.SubscriptionEvent{Height: 105} - logger := zerolog.Nop() - cfg := config.DefaultConfig() - cfg.DA.BlockTime.Duration = 100 * time.Millisecond - fetcher := NewAsyncBlockRetriever(client, logger, fiNs, cfg, 100, 10) + client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() + blockCh := make(chan datypes.SubscriptionEvent) + client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() - fetcher.Start() - defer fetcher.Stop() + // Height 102 has blobs; rest return not found or future. + client.On("Retrieve", mock.Anything, uint64(102), fiNs).Return(datypes.ResultRetrieve{ + BaseResult: datypes.BaseResult{ + Code: datypes.StatusSuccess, + Timestamp: time.Unix(2000, 0), + }, + Data: testBlobs, + }).Maybe() + client.On("Retrieve", mock.Anything, mock.Anything, fiNs).Return(datypes.ResultRetrieve{ + BaseResult: datypes.BaseResult{Code: datypes.StatusNotFound}, + }).Maybe() - // Update current height to trigger prefetch - fetcher.UpdateCurrentHeight(100) + logger := zerolog.Nop() + fetcher := NewAsyncBlockRetriever(client, logger, fiNs, 100*time.Millisecond, 100, 10) - // Wait for background prefetch to happen (wait for at least one poll cycle) - time.Sleep(250 * time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + fetcher.Start(ctx) + defer fetcher.Stop() - // Check if block was prefetched - ctx := context.Background() - block, err := fetcher.GetCachedBlock(ctx, 105) - require.NoError(t, err) - assert.NotNil(t, block) - assert.Equal(t, uint64(105), block.Height) - assert.Equal(t, 2, len(block.Blobs)) + // Wait for catchup to fill the gap. + var block *BlockData + var err error + for range 40 { + block, err = fetcher.GetCachedBlock(ctx, 102) + require.NoError(t, err) + if block != nil { + break + } + time.Sleep(50 * time.Millisecond) + } + require.NotNil(t, block, "block at 102 should be cached via catchup") + assert.Equal(t, uint64(102), block.Height) + assert.Equal(t, 1, len(block.Blobs)) + assert.Equal(t, []byte("gap-tx"), block.Blobs[0]) } func TestAsyncBlockRetriever_HeightFromFuture(t *testing.T) { client := &mocks.MockClient{} fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() - // All heights in prefetch window not available yet - for height := uint64(100); height <= 109; height++ { - client.On("Retrieve", mock.Anything, height, fiNs).Return(datypes.ResultRetrieve{ - BaseResult: datypes.BaseResult{Code: datypes.StatusHeightFromFuture}, - }).Maybe() - } + // Subscription delivers height 100 with no blobs. + subCh := make(chan datypes.SubscriptionEvent, 1) + subCh <- datypes.SubscriptionEvent{Height: 100} + + client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() + blockCh := make(chan datypes.SubscriptionEvent) + client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() + + // All Retrieve calls return HeightFromFuture. + client.On("Retrieve", mock.Anything, mock.Anything, fiNs).Return(datypes.ResultRetrieve{ + BaseResult: datypes.BaseResult{Code: datypes.StatusHeightFromFuture}, + }).Maybe() logger := zerolog.Nop() - cfg := config.DefaultConfig() - cfg.DA.BlockTime.Duration = 100 * time.Millisecond - fetcher := NewAsyncBlockRetriever(client, logger, fiNs, cfg, 100, 10) - fetcher.Start() - defer fetcher.Stop() + fetcher := NewAsyncBlockRetriever(client, logger, fiNs, 100*time.Millisecond, 100, 10) - fetcher.UpdateCurrentHeight(100) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + fetcher.Start(ctx) + defer fetcher.Stop() - // Wait for at least one poll cycle + // Wait a bit for catchup to attempt fetches. time.Sleep(250 * time.Millisecond) - // Cache should be empty - ctx := context.Background() + // Cache should be empty since all heights are from the future. block, err := fetcher.GetCachedBlock(ctx, 100) require.NoError(t, err) assert.Nil(t, block) @@ -188,16 +189,76 @@ func TestAsyncBlockRetriever_StopGracefully(t *testing.T) { client := &mocks.MockClient{} fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() + blockCh := make(chan datypes.SubscriptionEvent) + client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() + logger := zerolog.Nop() - fetcher := NewAsyncBlockRetriever(client, logger, fiNs, config.DefaultConfig(), 100, 10) + fetcher := NewAsyncBlockRetriever(client, logger, fiNs, 100*time.Millisecond, 100, 10) - fetcher.Start() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + fetcher.Start(ctx) time.Sleep(100 * time.Millisecond) // Should stop gracefully without panic fetcher.Stop() } +func TestAsyncBlockRetriever_ReconnectOnSubscriptionError(t *testing.T) { + // Verify that the follow loop reconnects after a subscription channel closes. + testBlobs := [][]byte{[]byte("reconnect-tx")} + + client := &mocks.MockClient{} + fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() + + // First subscription closes immediately (simulating error). + closedCh := make(chan datypes.SubscriptionEvent) + close(closedCh) + client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(closedCh), nil).Once() + + // Second subscription delivers a blob. + subCh := make(chan datypes.SubscriptionEvent, 1) + subCh <- datypes.SubscriptionEvent{ + Height: 100, + Blobs: testBlobs, + } + client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() + + // Third+ subscribe returns a blocking channel so it doesn't loop forever. + blockCh := make(chan datypes.SubscriptionEvent) + client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() + + // Stub Retrieve for catchup. + client.On("Retrieve", mock.Anything, mock.Anything, fiNs).Return(datypes.ResultRetrieve{ + BaseResult: datypes.BaseResult{Code: datypes.StatusHeightFromFuture}, + }).Maybe() + + logger := zerolog.Nop() + // Very short backoff so reconnect is fast in tests. + fetcher := NewAsyncBlockRetriever(client, logger, fiNs, 50*time.Millisecond, 100, 5) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + fetcher.Start(ctx) + defer fetcher.Stop() + + // Wait for reconnect + event processing. + var block *BlockData + var err error + for range 60 { + block, err = fetcher.GetCachedBlock(ctx, 100) + require.NoError(t, err) + if block != nil { + break + } + time.Sleep(50 * time.Millisecond) + } + + require.NotNil(t, block, "block should be cached after reconnect") + assert.Equal(t, 1, len(block.Blobs)) + assert.Equal(t, []byte("reconnect-tx"), block.Blobs[0]) +} + func TestBlockData_Serialization(t *testing.T) { block := &BlockData{ Height: 100, diff --git a/block/internal/da/forced_inclusion_retriever.go b/block/internal/da/forced_inclusion_retriever.go index 2ec299333a..a8051a5758 100644 --- a/block/internal/da/forced_inclusion_retriever.go +++ b/block/internal/da/forced_inclusion_retriever.go @@ -8,7 +8,6 @@ import ( "github.com/rs/zerolog" - "github.com/evstack/ev-node/pkg/config" datypes "github.com/evstack/ev-node/pkg/da/types" "github.com/evstack/ev-node/types" ) @@ -42,9 +41,11 @@ type ForcedInclusionEvent struct { // NewForcedInclusionRetriever creates a new forced inclusion retriever. // It internally creates and manages an AsyncBlockRetriever for background prefetching. func NewForcedInclusionRetriever( + ctx context.Context, client Client, logger zerolog.Logger, - cfg config.Config, + daBlockTime time.Duration, + tracingEnabled bool, daStartHeight, daEpochSize uint64, ) ForcedInclusionRetriever { retrieverLogger := logger.With().Str("component", "forced_inclusion_retriever").Logger() @@ -54,11 +55,11 @@ func NewForcedInclusionRetriever( client, logger, client.GetForcedInclusionNamespace(), - cfg, + daBlockTime, daStartHeight, daEpochSize*2, // prefetch window: 2x epoch size ) - asyncFetcher.Start() + asyncFetcher.Start(ctx) base := &forcedInclusionRetriever{ client: client, @@ -67,7 +68,7 @@ func NewForcedInclusionRetriever( daEpochSize: daEpochSize, asyncFetcher: asyncFetcher, } - if cfg.Instrumentation.IsTracingEnabled() { + if tracingEnabled { return withTracingForcedInclusionRetriever(base) } return base diff --git a/block/internal/da/forced_inclusion_retriever_test.go b/block/internal/da/forced_inclusion_retriever_test.go index 446b655bec..47864d7985 100644 --- a/block/internal/da/forced_inclusion_retriever_test.go +++ b/block/internal/da/forced_inclusion_retriever_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/mock" "gotest.tools/v3/assert" - "github.com/evstack/ev-node/pkg/config" datypes "github.com/evstack/ev-node/pkg/da/types" "github.com/evstack/ev-node/pkg/genesis" "github.com/evstack/ev-node/test/mocks" @@ -18,13 +17,14 @@ import ( func TestNewForcedInclusionRetriever(t *testing.T) { client := mocks.NewMockClient(t) client.On("GetForcedInclusionNamespace").Return(datypes.NamespaceFromString("test-fi-ns").Bytes()).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() gen := genesis.Genesis{ DAStartHeight: 100, DAEpochForcedInclusion: 10, } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) assert.Assert(t, retriever != nil) retriever.Stop() } @@ -39,7 +39,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_NoNamespace(t *testi DAEpochForcedInclusion: 10, } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -53,13 +53,14 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_NotAtEpochStart(t *t fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() gen := genesis.Genesis{ DAStartHeight: 100, DAEpochForcedInclusion: 10, } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -83,6 +84,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EpochStartSuccess(t fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, mock.Anything, fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusSuccess, IDs: []datypes.ID{[]byte("id1"), []byte("id2"), []byte("id3")}, Timestamp: time.Now()}, Data: testBlobs, @@ -93,7 +95,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EpochStartSuccess(t DAEpochForcedInclusion: 1, // Single height epoch } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -112,6 +114,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EpochStartNotAvailab fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // Mock the first height in epoch as not available client.On("Retrieve", mock.Anything, uint64(100), fiNs).Return(datypes.ResultRetrieve{ @@ -123,7 +126,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EpochStartNotAvailab DAEpochForcedInclusion: 10, } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -138,6 +141,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_NoBlobsAtHeight(t *t fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, uint64(100), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusNotFound}, }).Once() @@ -147,7 +151,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_NoBlobsAtHeight(t *t DAEpochForcedInclusion: 1, // Single height epoch } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -168,6 +172,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_MultiHeightEpoch(t * fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, uint64(102), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusSuccess, Timestamp: time.Now()}, Data: testBlobsByHeight[102], @@ -186,7 +191,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_MultiHeightEpoch(t * DAEpochForcedInclusion: 3, // Epoch: 100-102 } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -208,6 +213,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_ErrorHandling(t *tes fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, uint64(100), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{ Code: datypes.StatusError, @@ -220,7 +226,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_ErrorHandling(t *tes DAEpochForcedInclusion: 1, // Single height epoch } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -236,6 +242,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EmptyBlobsSkipped(t fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, uint64(100), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusSuccess, Timestamp: time.Now()}, Data: [][]byte{[]byte("tx1"), {}, []byte("tx2"), nil, []byte("tx3")}, @@ -246,7 +253,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EmptyBlobsSkipped(t DAEpochForcedInclusion: 1, // Single height epoch } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() @@ -272,6 +279,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_OrderPreserved(t *te fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // Return heights out of order to test ordering is preserved client.On("Retrieve", mock.Anything, uint64(102), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusSuccess, Timestamp: time.Now()}, @@ -291,7 +299,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_OrderPreserved(t *te DAEpochForcedInclusion: 3, // Epoch: 100-102 } - retriever := NewForcedInclusionRetriever(client, zerolog.Nop(), config.DefaultConfig(), gen.DAStartHeight, gen.DAEpochForcedInclusion) + retriever := NewForcedInclusionRetriever(context.Background(), client, zerolog.Nop(), 100*time.Millisecond, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) defer retriever.Stop() ctx := context.Background() diff --git a/block/internal/da/subscriber.go b/block/internal/da/subscriber.go new file mode 100644 index 0000000000..ffeb5141c9 --- /dev/null +++ b/block/internal/da/subscriber.go @@ -0,0 +1,391 @@ +package da + +import ( + "bytes" + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/rs/zerolog" + + datypes "github.com/evstack/ev-node/pkg/da/types" +) + +// SubscriberHandler is the callback interface for subscription consumers. +// Implementations drive the consumer-specific logic (caching, piping events, etc.). +type SubscriberHandler interface { + // HandleEvent processes a subscription event inline (fast path). + // Called on the followLoop goroutine for each subscription event. + HandleEvent(ctx context.Context, ev datypes.SubscriptionEvent) + + // HandleCatchup is called for each height during sequential catchup. + // The subscriber advances localDAHeight only after this returns nil. + // Returning an error triggers a backoff retry. + HandleCatchup(ctx context.Context, height uint64) error +} + +// SubscriberConfig holds configuration for creating a Subscriber. +type SubscriberConfig struct { + Client Client + Logger zerolog.Logger + Namespaces [][]byte // subscribe to all, merge into one channel + DABlockTime time.Duration + Handler SubscriberHandler +} + +// Subscriber is a shared DA subscription primitive that encapsulates the +// follow/catchup lifecycle. It subscribes to one or more DA namespaces, +// tracks the highest seen DA height, and drives sequential catchup via +// callbacks on SubscriberHandler. +// +// Used by both DAFollower (syncing) and asyncBlockRetriever (forced inclusion). +type Subscriber struct { + client Client + logger zerolog.Logger + handler SubscriberHandler + + // namespaces to subscribe on. When multiple, they are merged. + namespaces [][]byte + + // localDAHeight is only written by catchupLoop (via CAS) and read by + // followLoop to determine whether inline processing is possible. + localDAHeight atomic.Uint64 + + // highestSeenDAHeight is written by followLoop and read by catchupLoop. + highestSeenDAHeight atomic.Uint64 + + // headReached tracks whether the subscriber has caught up to DA head. + headReached atomic.Bool + + // catchupSignal wakes catchupLoop when a new height is seen above localDAHeight. + catchupSignal chan struct{} + + // daBlockTime used as backoff and watchdog base. + daBlockTime time.Duration + + // lifecycle + cancel context.CancelFunc + wg sync.WaitGroup +} + +// NewSubscriber creates a new Subscriber. +func NewSubscriber(cfg SubscriberConfig) *Subscriber { + s := &Subscriber{ + client: cfg.Client, + logger: cfg.Logger, + handler: cfg.Handler, + namespaces: cfg.Namespaces, + catchupSignal: make(chan struct{}, 1), + daBlockTime: cfg.DABlockTime, + } + return s +} + +// SetStartHeight sets the initial local DA height before Start is called. +func (s *Subscriber) SetStartHeight(height uint64) { + s.localDAHeight.Store(height) +} + +// Start begins the follow and catchup goroutines. +func (s *Subscriber) Start(ctx context.Context) error { + if len(s.namespaces) == 0 { + return nil // Nothing to subscribe to. + } + + ctx, s.cancel = context.WithCancel(ctx) + s.wg.Add(2) + go s.followLoop(ctx) + go s.catchupLoop(ctx) + return nil +} + +// Stop gracefully stops the background goroutines. +func (s *Subscriber) Stop() { + if s.cancel != nil { + s.cancel() + } + s.wg.Wait() +} + +// LocalDAHeight returns the current local DA height. +func (s *Subscriber) LocalDAHeight() uint64 { + return s.localDAHeight.Load() +} + +// HighestSeenDAHeight returns the highest DA height seen from the subscription. +func (s *Subscriber) HighestSeenDAHeight() uint64 { + return s.highestSeenDAHeight.Load() +} + +// HasReachedHead returns whether the subscriber has caught up to DA head. +func (s *Subscriber) HasReachedHead() bool { + return s.headReached.Load() +} + +// SetHeadReached marks the subscriber as having reached DA head. +func (s *Subscriber) SetHeadReached() { + s.headReached.Store(true) +} + +// CompareAndSwapLocalHeight attempts a CAS on localDAHeight. +// Used by handlers that want to claim exclusive processing of a height. +func (s *Subscriber) CompareAndSwapLocalHeight(old, new uint64) bool { + return s.localDAHeight.CompareAndSwap(old, new) +} + +// SetLocalHeight stores a new localDAHeight value. +func (s *Subscriber) SetLocalHeight(height uint64) { + s.localDAHeight.Store(height) +} + +// UpdateHighestForTest directly sets the highest seen DA height and signals catchup. +// Only for use in tests that bypass the subscription loop. +func (s *Subscriber) UpdateHighestForTest(height uint64) { + s.updateHighest(height) +} + +// RunCatchupForTest runs a single catchup pass. Only for use in tests that +// bypass the catchup loop's signal-wait mechanism. +func (s *Subscriber) RunCatchupForTest(ctx context.Context) { + s.runCatchup(ctx) +} + +// --------------------------------------------------------------------------- +// Follow loop +// --------------------------------------------------------------------------- + +// signalCatchup sends a non-blocking signal to wake catchupLoop. +func (s *Subscriber) signalCatchup() { + select { + case s.catchupSignal <- struct{}{}: + default: + } +} + +// followLoop subscribes to DA blob events and keeps highestSeenDAHeight up to date. +func (s *Subscriber) followLoop(ctx context.Context) { + defer s.wg.Done() + + s.logger.Info().Msg("starting follow loop") + defer s.logger.Info().Msg("follow loop stopped") + + for { + if err := s.runSubscription(ctx); err != nil { + if ctx.Err() != nil { + return + } + s.logger.Warn().Err(err).Msg("DA subscription failed, reconnecting") + select { + case <-ctx.Done(): + return + case <-time.After(s.backoff()): + } + } + } +} + +// runSubscription opens subscriptions on all namespaces (merging if more than one) +// and processes events until a channel is closed or the watchdog times out. +func (s *Subscriber) runSubscription(ctx context.Context) error { + subCtx, subCancel := context.WithCancel(ctx) + defer subCancel() + + ch, err := s.subscribe(subCtx) + if err != nil { + return err + } + + watchdogTimeout := s.watchdogTimeout() + watchdog := time.NewTimer(watchdogTimeout) + defer watchdog.Stop() + + for { + select { + case <-subCtx.Done(): + return subCtx.Err() + case ev, ok := <-ch: + if !ok { + return errors.New("subscription channel closed") + } + s.updateHighest(ev.Height) + s.handler.HandleEvent(ctx, ev) + watchdog.Reset(watchdogTimeout) + case <-watchdog.C: + return errors.New("subscription watchdog: no events received, reconnecting") + } + } +} + +// subscribe opens subscriptions on all configured namespaces. When there are +// multiple distinct namespaces, channels are merged via mergeSubscriptions. +func (s *Subscriber) subscribe(ctx context.Context) (<-chan datypes.SubscriptionEvent, error) { + if len(s.namespaces) == 0 { + return nil, errors.New("no namespaces configured") + } + + // Subscribe to the first namespace. + ch, err := s.client.Subscribe(ctx, s.namespaces[0]) + if err != nil { + return nil, fmt.Errorf("subscribe namespace 0: %w", err) + } + + // Subscribe to additional namespaces and merge. + for i := 1; i < len(s.namespaces); i++ { + if bytes.Equal(s.namespaces[i], s.namespaces[0]) { + continue // Same namespace, skip duplicate. + } + ch2, err := s.client.Subscribe(ctx, s.namespaces[i]) + if err != nil { + return nil, fmt.Errorf("subscribe namespace %d: %w", i, err) + } + ch = mergeSubscriptions(ctx, ch, ch2) + } + + return ch, nil +} + +// mergeSubscriptions fans two subscription channels into one. +func mergeSubscriptions( + ctx context.Context, + ch1, ch2 <-chan datypes.SubscriptionEvent, +) <-chan datypes.SubscriptionEvent { + out := make(chan datypes.SubscriptionEvent, 16) + go func() { + defer close(out) + for ch1 != nil || ch2 != nil { + var ev datypes.SubscriptionEvent + var ok bool + select { + case <-ctx.Done(): + return + case ev, ok = <-ch1: + if !ok { + ch1 = nil + continue + } + case ev, ok = <-ch2: + if !ok { + ch2 = nil + continue + } + } + select { + case out <- ev: + case <-ctx.Done(): + return + } + } + }() + return out +} + +// updateHighest atomically bumps highestSeenDAHeight and signals catchup if needed. +func (s *Subscriber) updateHighest(height uint64) { + for { + cur := s.highestSeenDAHeight.Load() + if height <= cur { + return + } + if s.highestSeenDAHeight.CompareAndSwap(cur, height) { + s.signalCatchup() + return + } + } +} + +// --------------------------------------------------------------------------- +// Catchup loop +// --------------------------------------------------------------------------- + +// catchupLoop waits for signals and sequentially catches up. +func (s *Subscriber) catchupLoop(ctx context.Context) { + defer s.wg.Done() + + s.logger.Info().Msg("starting catchup loop") + defer s.logger.Info().Msg("catchup loop stopped") + + for { + select { + case <-ctx.Done(): + return + case <-s.catchupSignal: + s.runCatchup(ctx) + } + } +} + +// runCatchup sequentially calls HandleCatchup from localDAHeight to highestSeenDAHeight. +func (s *Subscriber) runCatchup(ctx context.Context) { + for { + if ctx.Err() != nil { + return + } + + local := s.localDAHeight.Load() + highest := s.highestSeenDAHeight.Load() + + if local > highest { + s.headReached.Store(true) + return + } + + // CAS claims this height — prevents followLoop from inline-processing. + if !s.localDAHeight.CompareAndSwap(local, local+1) { + // followLoop already advanced past this height via inline processing. + continue + } + + if err := s.handler.HandleCatchup(ctx, local); err != nil { + // Roll back so we can retry after backoff. + s.localDAHeight.Store(local) + if !s.waitOnCatchupError(ctx, err, local) { + return + } + continue + } + } +} + +// ErrCaughtUp is a sentinel used to signal that the catchup loop has reached DA head. +var ErrCaughtUp = errors.New("caught up with DA head") + +// waitOnCatchupError logs the error and backs off before retrying. +func (s *Subscriber) waitOnCatchupError(ctx context.Context, err error, daHeight uint64) bool { + if errors.Is(err, ErrCaughtUp) { + s.logger.Debug().Uint64("da_height", daHeight).Msg("DA catchup reached head, waiting for subscription signal") + return false + } + if ctx.Err() != nil { + return false + } + s.logger.Warn().Err(err).Uint64("da_height", daHeight).Msg("catchup error, backing off") + select { + case <-ctx.Done(): + return false + case <-time.After(s.backoff()): + return true + } +} + +// --------------------------------------------------------------------------- +// Timing helpers +// --------------------------------------------------------------------------- + +func (s *Subscriber) backoff() time.Duration { + if s.daBlockTime > 0 { + return s.daBlockTime + } + return 2 * time.Second +} + +const subscriberWatchdogMultiplier = 3 + +func (s *Subscriber) watchdogTimeout() time.Duration { + if s.daBlockTime > 0 { + return s.daBlockTime * subscriberWatchdogMultiplier + } + return 30 * time.Second +} diff --git a/block/internal/syncing/da_follower.go b/block/internal/syncing/da_follower.go index 99b51db593..a31ffc43ca 100644 --- a/block/internal/syncing/da_follower.go +++ b/block/internal/syncing/da_follower.go @@ -1,12 +1,10 @@ package syncing import ( - "bytes" "context" "errors" - "fmt" + "slices" "sync" - "sync/atomic" "time" "github.com/rs/zerolog" @@ -16,61 +14,26 @@ import ( datypes "github.com/evstack/ev-node/pkg/da/types" ) -// DAFollower subscribes to DA blob events and drives sequential catchup. -// -// Architecture: -// - followLoop listens on the subscription channel. When caught up, it processes -// subscription blobs inline (fast path, no DA re-fetch). Otherwise, it updates -// highestSeenDAHeight and signals the catchup loop. -// - catchupLoop sequentially retrieves from localNextDAHeight → highestSeenDAHeight, -// piping events to the Syncer's heightInCh. -// -// The two goroutines share only atomic state; no mutexes needed. +// DAFollower follows DA blob events and drives sequential catchup +// using a shared da.Subscriber for the subscription plumbing. type DAFollower interface { - // Start begins the follow and catchup loops. Start(ctx context.Context) error - // Stop cancels the context and waits for goroutines. Stop() - // HasReachedHead returns true once the catchup loop has processed the - // DA head at least once. Once true, it stays true. HasReachedHead() bool + // QueuePriorityHeight queues a DA height for priority retrieval (from P2P hints). + QueuePriorityHeight(daHeight uint64) } // daFollower is the concrete implementation of DAFollower. type daFollower struct { - client da.Client - retriever DARetriever - logger zerolog.Logger - - // pipeEvent sends a DA height event to the syncer's processLoop. - pipeEvent func(ctx context.Context, event common.DAHeightEvent) error - - // Namespace to subscribe on (header namespace). - namespace []byte - // dataNamespace is the data namespace (may equal namespace when header+data - // share the same namespace). When different, we subscribe to both and merge. - dataNamespace []byte - - // localNextDAHeight is only written by catchupLoop and read by followLoop - // to determine whether a catchup is needed. - localNextDAHeight atomic.Uint64 + subscriber *da.Subscriber + retriever DARetriever + eventSink common.EventSink + logger zerolog.Logger - // highestSeenDAHeight is written by followLoop and read by catchupLoop. - highestSeenDAHeight atomic.Uint64 - - // headReached tracks whether the follower has caught up to DA head. - headReached atomic.Bool - - // catchupSignal is sent by followLoop to wake catchupLoop when a new - // height is seen that is above localNextDAHeight. - catchupSignal chan struct{} - - // daBlockTime is used as a backoff before retrying after errors. - daBlockTime time.Duration - - // lifecycle - cancel context.CancelFunc - wg sync.WaitGroup + // Priority queue for P2P hint heights (absorbed from DARetriever refactoring #2). + priorityMu sync.Mutex + priorityHeights []uint64 } // DAFollowerConfig holds configuration for creating a DAFollower. @@ -78,7 +41,7 @@ type DAFollowerConfig struct { Client da.Client Retriever DARetriever Logger zerolog.Logger - PipeEvent func(ctx context.Context, event common.DAHeightEvent) error + EventSink common.EventSink Namespace []byte DataNamespace []byte // may be nil or equal to Namespace StartDAHeight uint64 @@ -91,193 +54,74 @@ func NewDAFollower(cfg DAFollowerConfig) DAFollower { if len(dataNs) == 0 { dataNs = cfg.Namespace } + f := &daFollower{ - client: cfg.Client, - retriever: cfg.Retriever, - logger: cfg.Logger.With().Str("component", "da_follower").Logger(), - pipeEvent: cfg.PipeEvent, - namespace: cfg.Namespace, - dataNamespace: dataNs, - catchupSignal: make(chan struct{}, 1), - daBlockTime: cfg.DABlockTime, + retriever: cfg.Retriever, + eventSink: cfg.EventSink, + logger: cfg.Logger.With().Str("component", "da_follower").Logger(), + priorityHeights: make([]uint64, 0), } - f.localNextDAHeight.Store(cfg.StartDAHeight) + + f.subscriber = da.NewSubscriber(da.SubscriberConfig{ + Client: cfg.Client, + Logger: cfg.Logger, + Namespaces: [][]byte{cfg.Namespace, dataNs}, + DABlockTime: cfg.DABlockTime, + Handler: f, + }) + f.subscriber.SetStartHeight(cfg.StartDAHeight) + return f } // Start begins the follow and catchup goroutines. func (f *daFollower) Start(ctx context.Context) error { - ctx, f.cancel = context.WithCancel(ctx) - - f.wg.Add(2) - go f.followLoop(ctx) - f.signalCatchup() - go f.catchupLoop(ctx) - - f.logger.Info(). - Uint64("start_da_height", f.localNextDAHeight.Load()). - Msg("DA follower started") - return nil + return f.subscriber.Start(ctx) } -// Stop cancels and waits. +// Stop gracefully stops the background goroutines. func (f *daFollower) Stop() { - if f.cancel != nil { - f.cancel() - } - f.wg.Wait() + f.subscriber.Stop() } -// HasReachedHead returns whether the DA head has been reached at least once. +// HasReachedHead returns whether the follower has caught up to DA head. func (f *daFollower) HasReachedHead() bool { - return f.headReached.Load() -} - -// signalCatchup sends a non-blocking signal to wake catchupLoop. -func (f *daFollower) signalCatchup() { - select { - case f.catchupSignal <- struct{}{}: - default: - // Already signaled, catchupLoop will pick up the new highestSeen. - } -} - -// followLoop subscribes to DA blob events and keeps highestSeenDAHeight up to date. -// When a new height appears above localNextDAHeight, it wakes the catchup loop. -func (f *daFollower) followLoop(ctx context.Context) { - defer f.wg.Done() - - f.logger.Info().Msg("starting follow loop") - defer f.logger.Info().Msg("follow loop stopped") - - for { - if err := f.runSubscription(ctx); err != nil { - if ctx.Err() != nil { - return - } - f.logger.Warn().Err(err).Msg("DA subscription failed, reconnecting") - select { - case <-ctx.Done(): - return - case <-time.After(f.backoff()): - } - } - } -} - -// runSubscription opens subscriptions on both header and data namespaces (if -// different) and processes events until a channel is closed or an error occurs. -// A watchdog timer triggers if no events arrive within watchdogTimeout(), -// causing a reconnect. -func (f *daFollower) runSubscription(ctx context.Context) error { - // Sub-context ensures the merge goroutine is cancelled when this function returns. - subCtx, subCancel := context.WithCancel(ctx) - defer subCancel() - - headerCh, err := f.client.Subscribe(subCtx, f.namespace) - if err != nil { - return fmt.Errorf("subscribe header namespace: %w", err) - } - - // If namespaces differ, subscribe to the data namespace too and fan-in. - ch := headerCh - if !bytes.Equal(f.namespace, f.dataNamespace) { - dataCh, err := f.client.Subscribe(subCtx, f.dataNamespace) - if err != nil { - return fmt.Errorf("subscribe data namespace: %w", err) - } - ch = f.mergeSubscriptions(subCtx, headerCh, dataCh) - } - - watchdogTimeout := f.watchdogTimeout() - watchdog := time.NewTimer(watchdogTimeout) - defer watchdog.Stop() - - for { - select { - case <-subCtx.Done(): - return subCtx.Err() - case ev, ok := <-ch: - if !ok { - return errors.New("subscription channel closed") - } - f.handleSubscriptionEvent(ctx, ev) - watchdog.Reset(watchdogTimeout) - case <-watchdog.C: - return errors.New("subscription watchdog: no events received, reconnecting") - } - } + return f.subscriber.HasReachedHead() } -// mergeSubscriptions fans two subscription channels into one. -func (f *daFollower) mergeSubscriptions( - ctx context.Context, - headerCh, dataCh <-chan datypes.SubscriptionEvent, -) <-chan datypes.SubscriptionEvent { - out := make(chan datypes.SubscriptionEvent, 16) - go func() { - defer close(out) - for headerCh != nil || dataCh != nil { - var ev datypes.SubscriptionEvent - var ok bool - select { - case <-ctx.Done(): - return - case ev, ok = <-headerCh: - if !ok { - headerCh = nil - continue - } - case ev, ok = <-dataCh: - if !ok { - dataCh = nil - continue - } - } - select { - case out <- ev: - case <-ctx.Done(): - return - } - } - }() - return out -} +// --------------------------------------------------------------------------- +// SubscriberHandler implementation +// --------------------------------------------------------------------------- -// handleSubscriptionEvent processes a subscription event. When the follower is -// caught up (ev.Height == localNextDAHeight) and blobs are available, it processes -// them inline — avoiding a DA re-fetch round trip. Otherwise, it just updates -// highestSeenDAHeight and lets catchupLoop handle retrieval. +// HandleEvent processes a subscription event. When the follower is +// caught up (ev.Height == localDAHeight) and blobs are available, it processes +// them inline — avoiding a DA re-fetch round trip. Otherwise, it just lets +// the catchup loop handle retrieval. // -// Uses CAS on localNextDAHeight to claim exclusive access to processBlobs, +// Uses CAS on localDAHeight to claim exclusive access to processBlobs, // preventing concurrent map access with catchupLoop. -func (f *daFollower) handleSubscriptionEvent(ctx context.Context, ev datypes.SubscriptionEvent) { - // Always record the highest height we've seen from the subscription. - f.updateHighest(ev.Height) - +func (f *daFollower) HandleEvent(ctx context.Context, ev datypes.SubscriptionEvent) { // Fast path: try to claim this height for inline processing. // CAS(N, N+1) ensures only one goroutine (followLoop or catchupLoop) // can enter processBlobs for height N. - if len(ev.Blobs) > 0 && f.localNextDAHeight.CompareAndSwap(ev.Height, ev.Height+1) { + if len(ev.Blobs) > 0 && f.subscriber.CompareAndSwapLocalHeight(ev.Height, ev.Height+1) { events := f.retriever.ProcessBlobs(ctx, ev.Blobs, ev.Height) for _, event := range events { - if err := f.pipeEvent(ctx, event); err != nil { + if err := f.eventSink.PipeEvent(ctx, event); err != nil { // Roll back so catchupLoop can retry this height. - f.localNextDAHeight.Store(ev.Height) + f.subscriber.SetLocalHeight(ev.Height) f.logger.Warn().Err(err).Uint64("da_height", ev.Height). Msg("failed to pipe inline event, catchup will retry") return } } if len(events) != 0 { - if !f.headReached.Load() && f.localNextDAHeight.Load() > f.highestSeenDAHeight.Load() { - f.headReached.Store(true) - } + f.subscriber.SetHeadReached() f.logger.Debug().Uint64("da_height", ev.Height).Int("events", len(events)). Msg("processed subscription blobs inline (fast path)") } else { // No complete events (split namespace, waiting for other half). - f.localNextDAHeight.Store(ev.Height) + f.subscriber.SetLocalHeight(ev.Height) } return } @@ -285,105 +129,36 @@ func (f *daFollower) handleSubscriptionEvent(ctx context.Context, ev datypes.Sub // Slow path: behind, no blobs, or catchupLoop claimed this height. } -// updateHighest atomically bumps highestSeenDAHeight and signals catchup if needed. -func (f *daFollower) updateHighest(height uint64) { - for { - cur := f.highestSeenDAHeight.Load() - if height <= cur { - return - } - if f.highestSeenDAHeight.CompareAndSwap(cur, height) { - f.signalCatchup() - return - } - } -} - -// catchupLoop waits for signals and sequentially retrieves DA heights -// from localNextDAHeight up to highestSeenDAHeight. -func (f *daFollower) catchupLoop(ctx context.Context) { - defer f.wg.Done() - - f.logger.Info().Msg("starting catchup loop") - defer f.logger.Info().Msg("catchup loop stopped") - - for { - select { - case <-ctx.Done(): - return - case <-f.catchupSignal: - f.runCatchup(ctx) - } - } -} - -// runCatchup sequentially retrieves from localNextDAHeight to highestSeenDAHeight. -// It handles priority heights first, then sequential heights. -func (f *daFollower) runCatchup(ctx context.Context) { - for { - if ctx.Err() != nil { - return - } - - // Check for priority heights from P2P hints first. - // We drain stale hints to avoid a tight CPU loop if many are queued. - priorityHeight := f.retriever.PopPriorityHeight() - for priorityHeight > 0 && priorityHeight < f.localNextDAHeight.Load() { - priorityHeight = f.retriever.PopPriorityHeight() - } - - if priorityHeight > 0 { +// HandleCatchup retrieves events at a single DA height and pipes them +// to the event sink. Checks priority heights first. +func (f *daFollower) HandleCatchup(ctx context.Context, daHeight uint64) error { + // Check for priority heights from P2P hints first. + if priorityHeight := f.popPriorityHeight(); priorityHeight > 0 { + if priorityHeight >= daHeight { f.logger.Debug(). Uint64("da_height", priorityHeight). Msg("fetching priority DA height from P2P hint") if err := f.fetchAndPipeHeight(ctx, priorityHeight); err != nil { - if !f.waitOnCatchupError(ctx, err, priorityHeight) { - return - } - } - continue - } - - // Sequential catchup. - local := f.localNextDAHeight.Load() - highest := f.highestSeenDAHeight.Load() - - if highest > 0 && local > highest { - // Caught up. - f.headReached.Store(true) - return - } - - // CAS claims this height prevents followLoop from inline-processing - if !f.localNextDAHeight.CompareAndSwap(local, local+1) { - // followLoop already advanced past this height via inline processing. - continue - } - - if err := f.fetchAndPipeHeight(ctx, local); err != nil { - // Roll back so we can retry after backoff. - f.localNextDAHeight.Store(local) - if !f.waitOnCatchupError(ctx, err, local) { - return + return err } - continue } + // Re-queue the current height by rolling back (the subscriber already advanced). + f.subscriber.SetLocalHeight(daHeight) + return nil } + + return f.fetchAndPipeHeight(ctx, daHeight) } -// fetchAndPipeHeight retrieves events at a single DA height and pipes them -// to the syncer. +// fetchAndPipeHeight retrieves events at a single DA height and pipes them. func (f *daFollower) fetchAndPipeHeight(ctx context.Context, daHeight uint64) error { events, err := f.retriever.RetrieveFromDA(ctx, daHeight) if err != nil { switch { case errors.Is(err, datypes.ErrBlobNotFound): - // No blobs at this height — not an error, just skip. return nil case errors.Is(err, datypes.ErrHeightFromFuture): - // DA hasn't produced this height yet — mark head reached - // but return the error to trigger a backoff retry. - f.headReached.Store(true) + f.subscriber.SetHeadReached() return err default: return err @@ -391,7 +166,7 @@ func (f *daFollower) fetchAndPipeHeight(ctx context.Context, daHeight uint64) er } for _, event := range events { - if err := f.pipeEvent(ctx, event); err != nil { + if err := f.eventSink.PipeEvent(ctx, event); err != nil { return err } } @@ -399,44 +174,31 @@ func (f *daFollower) fetchAndPipeHeight(ctx context.Context, daHeight uint64) er return nil } -// errCaughtUp is a sentinel used to signal that the catchup loop has reached DA head. -var errCaughtUp = errors.New("caught up with DA head") +// --------------------------------------------------------------------------- +// Priority queue (absorbed from DARetriever — refactoring #2) +// --------------------------------------------------------------------------- -// waitOnCatchupError logs the error and backs off before retrying. -// It returns true if the caller should continue (retry), or false if the -// catchup loop should exit (context cancelled or caught-up sentinel). -func (f *daFollower) waitOnCatchupError(ctx context.Context, err error, daHeight uint64) bool { - if errors.Is(err, errCaughtUp) { - f.logger.Debug().Uint64("da_height", daHeight).Msg("DA catchup reached head, waiting for subscription signal") - return false - } - if ctx.Err() != nil { - return false - } - f.logger.Warn().Err(err).Uint64("da_height", daHeight).Msg("catchup error, backing off") - select { - case <-ctx.Done(): - return false - case <-time.After(f.backoff()): - return true - } -} +// QueuePriorityHeight queues a DA height for priority retrieval. +func (f *daFollower) QueuePriorityHeight(daHeight uint64) { + f.priorityMu.Lock() + defer f.priorityMu.Unlock() -// backoff returns the configured DA block time or a sane default. -func (f *daFollower) backoff() time.Duration { - if f.daBlockTime > 0 { - return f.daBlockTime + idx, found := slices.BinarySearch(f.priorityHeights, daHeight) + if found { + return } - return 2 * time.Second + f.priorityHeights = slices.Insert(f.priorityHeights, idx, daHeight) } -// watchdogTimeout returns how long to wait for a subscription event before -// assuming the subscription is stalled. Defaults to 3× the DA block time. -const watchdogMultiplier = 3 +// popPriorityHeight returns the next priority height to fetch, or 0 if none. +func (f *daFollower) popPriorityHeight() uint64 { + f.priorityMu.Lock() + defer f.priorityMu.Unlock() -func (f *daFollower) watchdogTimeout() time.Duration { - if f.daBlockTime > 0 { - return f.daBlockTime * watchdogMultiplier + if len(f.priorityHeights) == 0 { + return 0 } - return 30 * time.Second + height := f.priorityHeights[0] + f.priorityHeights = f.priorityHeights[1:] + return height } diff --git a/block/internal/syncing/da_retriever.go b/block/internal/syncing/da_retriever.go index 366d3ed30c..691b2d1bb6 100644 --- a/block/internal/syncing/da_retriever.go +++ b/block/internal/syncing/da_retriever.go @@ -5,9 +5,6 @@ import ( "context" "errors" "fmt" - "slices" - "sync" - "sync/atomic" "github.com/rs/zerolog" "google.golang.org/protobuf/proto" @@ -28,11 +25,6 @@ type DARetriever interface { // ProcessBlobs parses raw blob bytes at a given DA height into height events. // Used by the DAFollower to process subscription blobs inline without re-fetching. ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent - // QueuePriorityHeight queues a DA height for priority retrieval (from P2P hints). - // These heights take precedence over sequential fetching. - QueuePriorityHeight(daHeight uint64) - // PopPriorityHeight returns the next priority height to fetch, or 0 if none. - PopPriorityHeight() uint64 } // daRetriever handles DA retrieval operations for syncing @@ -44,19 +36,12 @@ type daRetriever struct { // transient cache, only full event need to be passed to the syncer // on restart, will be refetch as da height is updated by syncer - pendingMu sync.Mutex pendingHeaders map[uint64]*types.SignedHeader pendingData map[uint64]*types.Data // strictMode indicates if the node has seen a valid DAHeaderEnvelope // and should now reject all legacy/unsigned headers. - strictMode atomic.Bool - - // priorityMu protects priorityHeights from concurrent access - priorityMu sync.Mutex - // priorityHeights holds DA heights from P2P hints that should be fetched - // before continuing sequential retrieval. Sorted in ascending order. - priorityHeights []uint64 + strictMode bool } // NewDARetriever creates a new DA retriever @@ -67,42 +52,14 @@ func NewDARetriever( logger zerolog.Logger, ) *daRetriever { return &daRetriever{ - client: client, - cache: cache, - genesis: genesis, - logger: logger.With().Str("component", "da_retriever").Logger(), - pendingHeaders: make(map[uint64]*types.SignedHeader), - pendingData: make(map[uint64]*types.Data), - priorityHeights: make([]uint64, 0), - } -} - -// QueuePriorityHeight queues a DA height for priority retrieval. -// Heights from P2P hints take precedence over sequential fetching. -func (r *daRetriever) QueuePriorityHeight(daHeight uint64) { - r.priorityMu.Lock() - defer r.priorityMu.Unlock() - - idx, found := slices.BinarySearch(r.priorityHeights, daHeight) - if found { - return // Already queued - } - r.priorityHeights = slices.Insert(r.priorityHeights, idx, daHeight) -} - -// PopPriorityHeight returns the next priority height to fetch, or 0 if none. -func (r *daRetriever) PopPriorityHeight() uint64 { - r.priorityMu.Lock() - defer r.priorityMu.Unlock() - - if len(r.priorityHeights) == 0 { - return 0 + client: client, + cache: cache, + genesis: genesis, + logger: logger.With().Str("component", "da_retriever").Logger(), + pendingHeaders: make(map[uint64]*types.SignedHeader), + pendingData: make(map[uint64]*types.Data), + strictMode: false, } - - height := r.priorityHeights[0] - r.priorityHeights = r.priorityHeights[1:] - - return height } // RetrieveFromDA retrieves blocks from the specified DA height and returns height events @@ -199,55 +156,41 @@ func (r *daRetriever) validateBlobResponse(res datypes.ResultRetrieve, daHeight // This is the public interface used by the DAFollower for inline subscription processing. // // NOT thread-safe: the caller (DAFollower) must ensure exclusive access via CAS -// on localNextDAHeight before calling this method. +// on localDAHeight before calling this method. func (r *daRetriever) ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { return r.processBlobs(ctx, blobs, daHeight) } // processBlobs processes retrieved blobs to extract headers and data and returns height events func (r *daRetriever) processBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { - var decodedHeaders []*types.SignedHeader - var decodedData []*types.Data - - // Decode all blobs first without holding the lock + // Decode all blobs for _, bz := range blobs { if len(bz) == 0 { continue } if header := r.tryDecodeHeader(bz, daHeight); header != nil { - decodedHeaders = append(decodedHeaders, header) - continue - } - - if data := r.tryDecodeData(bz, daHeight); data != nil { - decodedData = append(decodedData, data) - } - } - - r.pendingMu.Lock() - defer r.pendingMu.Unlock() + if _, ok := r.pendingHeaders[header.Height()]; ok { + // a (malicious) node may have re-published valid header to another da height (should never happen) + // we can already discard it, only the first one is valid + r.logger.Debug().Uint64("height", header.Height()).Uint64("da_height", daHeight).Msg("header blob already exists for height, discarding") + continue + } - for _, header := range decodedHeaders { - if _, ok := r.pendingHeaders[header.Height()]; ok { - // a (malicious) node may have re-published valid header to another da height (should never happen) - // we can already discard it, only the first one is valid - r.logger.Debug().Uint64("height", header.Height()).Uint64("da_height", daHeight).Msg("header blob already exists for height, discarding") + r.pendingHeaders[header.Height()] = header continue } - r.pendingHeaders[header.Height()] = header - } + if data := r.tryDecodeData(bz, daHeight); data != nil { + if _, ok := r.pendingData[data.Height()]; ok { + // a (malicious) node may have re-published valid data to another da height (should never happen) + // we can already discard it, only the first one is valid + r.logger.Debug().Uint64("height", data.Height()).Uint64("da_height", daHeight).Msg("data blob already exists for height, discarding") + continue + } - for _, data := range decodedData { - if _, ok := r.pendingData[data.Height()]; ok { - // a (malicious) node may have re-published valid data to another da height (should never happen) - // we can already discard it, only the first one is valid - r.logger.Debug().Uint64("height", data.Height()).Uint64("da_height", daHeight).Msg("data blob already exists for height, discarding") - continue + r.pendingData[data.Height()] = data } - - r.pendingData[data.Height()] = data } var events []common.DAHeightEvent @@ -309,7 +252,7 @@ func (r *daRetriever) tryDecodeHeader(bz []byte, daHeight uint64) *types.SignedH // Attempt to unmarshal as DAHeaderEnvelope and get the envelope signature if envelopeSignature, err := header.UnmarshalDAEnvelope(bz); err != nil { // If in strict mode, we REQUIRE an envelope. - if r.strictMode.Load() { + if r.strictMode { r.logger.Warn().Err(err).Msg("strict mode is enabled, rejecting non-envelope blob") return nil } @@ -343,14 +286,14 @@ func (r *daRetriever) tryDecodeHeader(bz []byte, daHeight uint64) *types.SignedH isValidEnvelope = true } } - if r.strictMode.Load() && !isValidEnvelope { + if r.strictMode && !isValidEnvelope { r.logger.Warn().Msg("strict mode: rejecting block that is not a fully valid envelope") return nil } // Mode Switch Logic - if isValidEnvelope && !r.strictMode.Load() { + if isValidEnvelope && !r.strictMode { r.logger.Info().Uint64("height", header.Height()).Msg("valid DA envelope detected, switching to STRICT MODE") - r.strictMode.Store(true) + r.strictMode = true } // Legacy blob support implies: strictMode == false AND (!isValidEnvelope). diff --git a/block/internal/syncing/da_retriever_mock.go b/block/internal/syncing/da_retriever_mock.go index dc6926485e..7e8cc47be1 100644 --- a/block/internal/syncing/da_retriever_mock.go +++ b/block/internal/syncing/da_retriever_mock.go @@ -38,90 +38,6 @@ func (_m *MockDARetriever) EXPECT() *MockDARetriever_Expecter { return &MockDARetriever_Expecter{mock: &_m.Mock} } -// PopPriorityHeight provides a mock function for the type MockDARetriever -func (_mock *MockDARetriever) PopPriorityHeight() uint64 { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for PopPriorityHeight") - } - - var r0 uint64 - if returnFunc, ok := ret.Get(0).(func() uint64); ok { - r0 = returnFunc() - } else { - r0 = ret.Get(0).(uint64) - } - return r0 -} - -// MockDARetriever_PopPriorityHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PopPriorityHeight' -type MockDARetriever_PopPriorityHeight_Call struct { - *mock.Call -} - -// PopPriorityHeight is a helper method to define mock.On call -func (_e *MockDARetriever_Expecter) PopPriorityHeight() *MockDARetriever_PopPriorityHeight_Call { - return &MockDARetriever_PopPriorityHeight_Call{Call: _e.mock.On("PopPriorityHeight")} -} - -func (_c *MockDARetriever_PopPriorityHeight_Call) Run(run func()) *MockDARetriever_PopPriorityHeight_Call { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *MockDARetriever_PopPriorityHeight_Call) Return(v uint64) *MockDARetriever_PopPriorityHeight_Call { - _c.Call.Return(v) - return _c -} - -func (_c *MockDARetriever_PopPriorityHeight_Call) RunAndReturn(run func() uint64) *MockDARetriever_PopPriorityHeight_Call { - _c.Call.Return(run) - return _c -} - -// QueuePriorityHeight provides a mock function for the type MockDARetriever -func (_mock *MockDARetriever) QueuePriorityHeight(daHeight uint64) { - _mock.Called(daHeight) - return -} - -// MockDARetriever_QueuePriorityHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'QueuePriorityHeight' -type MockDARetriever_QueuePriorityHeight_Call struct { - *mock.Call -} - -// QueuePriorityHeight is a helper method to define mock.On call -// - daHeight uint64 -func (_e *MockDARetriever_Expecter) QueuePriorityHeight(daHeight interface{}) *MockDARetriever_QueuePriorityHeight_Call { - return &MockDARetriever_QueuePriorityHeight_Call{Call: _e.mock.On("QueuePriorityHeight", daHeight)} -} - -func (_c *MockDARetriever_QueuePriorityHeight_Call) Run(run func(daHeight uint64)) *MockDARetriever_QueuePriorityHeight_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 uint64 - if args[0] != nil { - arg0 = args[0].(uint64) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *MockDARetriever_QueuePriorityHeight_Call) Return() *MockDARetriever_QueuePriorityHeight_Call { - _c.Call.Return() - return _c -} - -func (_c *MockDARetriever_QueuePriorityHeight_Call) RunAndReturn(run func(daHeight uint64)) *MockDARetriever_QueuePriorityHeight_Call { - _c.Run(run) - return _c -} - // RetrieveFromDA provides a mock function for the type MockDARetriever func (_mock *MockDARetriever) RetrieveFromDA(ctx context.Context, daHeight uint64) ([]common.DAHeightEvent, error) { ret := _mock.Called(ctx, daHeight) diff --git a/block/internal/syncing/da_retriever_tracing.go b/block/internal/syncing/da_retriever_tracing.go index 1e1a9ea7c0..d41418a1d8 100644 --- a/block/internal/syncing/da_retriever_tracing.go +++ b/block/internal/syncing/da_retriever_tracing.go @@ -56,14 +56,6 @@ func (t *tracedDARetriever) RetrieveFromDA(ctx context.Context, daHeight uint64) return events, nil } -func (t *tracedDARetriever) QueuePriorityHeight(daHeight uint64) { - t.inner.QueuePriorityHeight(daHeight) -} - -func (t *tracedDARetriever) PopPriorityHeight() uint64 { - return t.inner.PopPriorityHeight() -} - func (t *tracedDARetriever) ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { return t.inner.ProcessBlobs(ctx, blobs, daHeight) } diff --git a/block/internal/syncing/da_retriever_tracing_test.go b/block/internal/syncing/da_retriever_tracing_test.go index 930fc43617..83bd864bfa 100644 --- a/block/internal/syncing/da_retriever_tracing_test.go +++ b/block/internal/syncing/da_retriever_tracing_test.go @@ -27,10 +27,6 @@ func (m *mockDARetriever) RetrieveFromDA(ctx context.Context, daHeight uint64) ( return nil, nil } -func (m *mockDARetriever) QueuePriorityHeight(daHeight uint64) {} - -func (m *mockDARetriever) PopPriorityHeight() uint64 { return 0 } - func (m *mockDARetriever) ProcessBlobs(_ context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { return nil } diff --git a/block/internal/syncing/raft_retriever.go b/block/internal/syncing/raft_retriever.go index 0b976acab9..cfa55662bd 100644 --- a/block/internal/syncing/raft_retriever.go +++ b/block/internal/syncing/raft_retriever.go @@ -14,20 +14,6 @@ import ( "github.com/evstack/ev-node/types" ) -// eventProcessor handles DA height events. Used for syncing. -type eventProcessor interface { - // handle processes a single DA height event. - handle(ctx context.Context, event common.DAHeightEvent) error -} - -// eventProcessorFn adapts a function to an eventProcessor. -type eventProcessorFn func(ctx context.Context, event common.DAHeightEvent) error - -// handle calls the wrapped function. -func (e eventProcessorFn) handle(ctx context.Context, event common.DAHeightEvent) error { - return e(ctx, event) -} - // raftStatePreProcessor is called before processing a raft block state type raftStatePreProcessor func(ctx context.Context, state *raft.RaftBlockState) error @@ -37,7 +23,7 @@ type raftRetriever struct { wg sync.WaitGroup logger zerolog.Logger genesis genesis.Genesis - eventProcessor eventProcessor + eventSink common.EventSink raftBlockPreProcessor raftStatePreProcessor mtx sync.Mutex @@ -49,14 +35,14 @@ func newRaftRetriever( raftNode common.RaftNode, genesis genesis.Genesis, logger zerolog.Logger, - eventProcessor eventProcessor, + eventSink common.EventSink, raftBlockPreProcessor raftStatePreProcessor, ) *raftRetriever { return &raftRetriever{ raftNode: raftNode, genesis: genesis, logger: logger, - eventProcessor: eventProcessor, + eventSink: eventSink, raftBlockPreProcessor: raftBlockPreProcessor, } } @@ -153,7 +139,7 @@ func (r *raftRetriever) consumeRaftBlock(ctx context.Context, state *raft.RaftBl Data: &data, DaHeight: 0, // raft events don't have DA height context, yet as DA submission is asynchronous } - return r.eventProcessor.handle(ctx, event) + return r.eventSink.PipeEvent(ctx, event) } // Height returns the current height of the raft node's state. diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index a41f2a5478..91d231527e 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -143,7 +143,7 @@ func NewSyncer( } s.blockSyncer = s if raftNode != nil && !reflect.ValueOf(raftNode).IsNil() { - s.raftRetriever = newRaftRetriever(raftNode, genesis, logger, eventProcessorFn(s.pipeEvent), + s.raftRetriever = newRaftRetriever(raftNode, genesis, logger, s, func(ctx context.Context, state *raft.RaftBlockState) error { s.logger.Debug().Uint64("header_height", state.LastSubmittedDaHeaderHeight).Uint64("data_height", state.LastSubmittedDaDataHeight).Msg("received raft block state") cache.SetLastSubmittedHeaderHeight(ctx, state.LastSubmittedDaHeaderHeight) @@ -181,7 +181,7 @@ func (s *Syncer) Start(ctx context.Context) (err error) { s.daRetriever = WithTracingDARetriever(s.daRetriever) } - s.fiRetriever = da.NewForcedInclusionRetriever(s.daClient, s.logger, s.config, s.genesis.DAStartHeight, s.genesis.DAEpochForcedInclusion) + s.fiRetriever = da.NewForcedInclusionRetriever(ctx, s.daClient, s.logger, s.config.DA.BlockTime.Duration, s.config.Instrumentation.IsTracingEnabled(), s.genesis.DAStartHeight, s.genesis.DAEpochForcedInclusion) s.p2pHandler = NewP2PHandler(s.headerStore, s.dataStore, s.cache, s.genesis, s.logger) currentHeight, initErr := s.store.Height(ctx) @@ -209,7 +209,7 @@ func (s *Syncer) Start(ctx context.Context) (err error) { Client: s.daClient, Retriever: s.daRetriever, Logger: s.logger, - PipeEvent: s.pipeEvent, + EventSink: s, Namespace: s.daClient.GetHeaderNamespace(), DataNamespace: s.daClient.GetDataNamespace(), StartDAHeight: s.daRetrieverHeight.Load(), @@ -492,7 +492,7 @@ func (s *Syncer) waitForGenesis() bool { return true } -func (s *Syncer) pipeEvent(ctx context.Context, event common.DAHeightEvent) error { +func (s *Syncer) PipeEvent(ctx context.Context, event common.DAHeightEvent) error { select { case s.heightInCh <- event: return nil @@ -608,7 +608,7 @@ func (s *Syncer) processHeightEvent(ctx context.Context, event *common.DAHeightE Msg("P2P event with DA height hint, queuing priority DA retrieval") // Queue priority DA retrieval - will be processed in fetchDAUntilCaughtUp - s.daRetriever.QueuePriorityHeight(daHeightHint) + s.daFollower.QueuePriorityHeight(daHeightHint) } } } diff --git a/block/internal/syncing/syncer_backoff_test.go b/block/internal/syncing/syncer_backoff_test.go index b8ebf269e9..113cf173d9 100644 --- a/block/internal/syncing/syncer_backoff_test.go +++ b/block/internal/syncing/syncer_backoff_test.go @@ -60,21 +60,26 @@ func TestDAFollower_BackoffOnCatchupError(t *testing.T) { defer cancel() daRetriever := NewMockDARetriever(t) - daRetriever.On("PopPriorityHeight").Return(uint64(0)).Maybe() pipeEvent := func(_ context.Context, _ common.DAHeightEvent) error { return nil } follower := NewDAFollower(DAFollowerConfig{ Retriever: daRetriever, Logger: zerolog.Nop(), - PipeEvent: pipeEvent, + EventSink: common.EventSinkFunc(pipeEvent), Namespace: []byte("ns"), StartDAHeight: 100, DABlockTime: tc.daBlockTime, }).(*daFollower) - ctx, follower.cancel = context.WithCancel(ctx) - follower.highestSeenDAHeight.Store(102) + // Set up the subscriber for direct testing. + sub := follower.subscriber + sub.SetStartHeight(100) + ctx, subCancel := context.WithCancel(ctx) + defer subCancel() + + // Set highest to trigger catchup. + sub.UpdateHighestForTest(102) var callTimes []time.Time callCount := 0 @@ -92,7 +97,7 @@ func TestDAFollower_BackoffOnCatchupError(t *testing.T) { Run(func(args mock.Arguments) { callTimes = append(callTimes, time.Now()) callCount++ - cancel() + subCancel() }). Return(nil, datypes.ErrBlobNotFound).Once() } else { @@ -100,12 +105,12 @@ func TestDAFollower_BackoffOnCatchupError(t *testing.T) { Run(func(args mock.Arguments) { callTimes = append(callTimes, time.Now()) callCount++ - cancel() + subCancel() }). Return(nil, datypes.ErrBlobNotFound).Once() } - go follower.runCatchup(ctx) + go sub.RunCatchupForTest(ctx) <-ctx.Done() if tc.expectsBackoff { @@ -144,21 +149,22 @@ func TestDAFollower_BackoffResetOnSuccess(t *testing.T) { gen := backoffTestGenesis(addr) daRetriever := NewMockDARetriever(t) - daRetriever.On("PopPriorityHeight").Return(uint64(0)).Maybe() pipeEvent := func(_ context.Context, _ common.DAHeightEvent) error { return nil } follower := NewDAFollower(DAFollowerConfig{ Retriever: daRetriever, Logger: zerolog.Nop(), - PipeEvent: pipeEvent, + EventSink: common.EventSinkFunc(pipeEvent), Namespace: []byte("ns"), StartDAHeight: 100, DABlockTime: 1 * time.Second, }).(*daFollower) - ctx, follower.cancel = context.WithCancel(ctx) - follower.highestSeenDAHeight.Store(105) + sub := follower.subscriber + ctx, subCancel := context.WithCancel(ctx) + defer subCancel() + sub.UpdateHighestForTest(105) var callTimes []time.Time @@ -194,11 +200,11 @@ func TestDAFollower_BackoffResetOnSuccess(t *testing.T) { daRetriever.On("RetrieveFromDA", mock.Anything, uint64(101)). Run(func(args mock.Arguments) { callTimes = append(callTimes, time.Now()) - cancel() + subCancel() }). Return(nil, datypes.ErrBlobNotFound).Once() - go follower.runCatchup(ctx) + go sub.RunCatchupForTest(ctx) <-ctx.Done() require.Len(t, callTimes, 3, "should make exactly 3 calls") @@ -221,20 +227,20 @@ func TestDAFollower_CatchupThenReachHead(t *testing.T) { defer cancel() daRetriever := NewMockDARetriever(t) - daRetriever.On("PopPriorityHeight").Return(uint64(0)).Maybe() pipeEvent := func(_ context.Context, _ common.DAHeightEvent) error { return nil } follower := NewDAFollower(DAFollowerConfig{ Retriever: daRetriever, Logger: zerolog.Nop(), - PipeEvent: pipeEvent, + EventSink: common.EventSinkFunc(pipeEvent), Namespace: []byte("ns"), StartDAHeight: 3, DABlockTime: 500 * time.Millisecond, }).(*daFollower) - follower.highestSeenDAHeight.Store(5) + sub := follower.subscriber + sub.UpdateHighestForTest(5) var fetchedHeights []uint64 @@ -246,7 +252,7 @@ func TestDAFollower_CatchupThenReachHead(t *testing.T) { Return(nil, datypes.ErrBlobNotFound).Once() } - follower.runCatchup(ctx) + sub.RunCatchupForTest(ctx) assert.True(t, follower.HasReachedHead(), "should have reached DA head") // Heights 3, 4, 5 processed; local now at 6 which > highest (5) → caught up @@ -255,7 +261,7 @@ func TestDAFollower_CatchupThenReachHead(t *testing.T) { } // TestDAFollower_InlineProcessing verifies the fast path: when the subscription -// delivers blobs at the current localNextDAHeight, handleSubscriptionEvent processes +// delivers blobs at the current localDAHeight, HandleEvent processes // them inline via ProcessBlobs (not RetrieveFromDA). func TestDAFollower_InlineProcessing(t *testing.T) { t.Run("processes_blobs_inline_when_caught_up", func(t *testing.T) { @@ -270,7 +276,7 @@ func TestDAFollower_InlineProcessing(t *testing.T) { follower := NewDAFollower(DAFollowerConfig{ Retriever: daRetriever, Logger: zerolog.Nop(), - PipeEvent: pipeEvent, + EventSink: common.EventSinkFunc(pipeEvent), Namespace: []byte("ns"), StartDAHeight: 10, DABlockTime: 500 * time.Millisecond, @@ -285,8 +291,8 @@ func TestDAFollower_InlineProcessing(t *testing.T) { daRetriever.On("ProcessBlobs", mock.Anything, blobs, uint64(10)). Return(expectedEvents).Once() - // Simulate subscription event at the current localNextDAHeight - follower.handleSubscriptionEvent(t.Context(), datypes.SubscriptionEvent{ + // Simulate subscription event at the current localDAHeight + follower.HandleEvent(t.Context(), datypes.SubscriptionEvent{ Height: 10, Blobs: blobs, }) @@ -294,7 +300,7 @@ func TestDAFollower_InlineProcessing(t *testing.T) { // Verify: ProcessBlobs was called, events were piped, height advanced require.Len(t, pipedEvents, 1, "should pipe 1 event from inline processing") assert.Equal(t, uint64(10), pipedEvents[0].DaHeight) - assert.Equal(t, uint64(11), follower.localNextDAHeight.Load(), "localNextDAHeight should advance past processed height") + assert.Equal(t, uint64(11), follower.subscriber.LocalDAHeight(), "localDAHeight should advance past processed height") assert.True(t, follower.HasReachedHead(), "should mark head as reached after inline processing") }) @@ -306,26 +312,24 @@ func TestDAFollower_InlineProcessing(t *testing.T) { follower := NewDAFollower(DAFollowerConfig{ Retriever: daRetriever, Logger: zerolog.Nop(), - PipeEvent: pipeEvent, + EventSink: common.EventSinkFunc(pipeEvent), Namespace: []byte("ns"), StartDAHeight: 10, DABlockTime: 500 * time.Millisecond, }).(*daFollower) - ctx := t.Context() - ctx, follower.cancel = context.WithCancel(ctx) - defer follower.cancel() - // Subscription reports height 15 but local is at 10 — should NOT process inline - follower.handleSubscriptionEvent(ctx, datypes.SubscriptionEvent{ + // In production, the subscriber calls updateHighest before HandleEvent. + follower.subscriber.UpdateHighestForTest(15) + follower.HandleEvent(t.Context(), datypes.SubscriptionEvent{ Height: 15, Blobs: [][]byte{[]byte("blob")}, }) // ProcessBlobs should NOT have been called daRetriever.AssertNotCalled(t, "ProcessBlobs", mock.Anything, mock.Anything, mock.Anything) - assert.Equal(t, uint64(10), follower.localNextDAHeight.Load(), "localNextDAHeight should not change") - assert.Equal(t, uint64(15), follower.highestSeenDAHeight.Load(), "highestSeen should be updated") + assert.Equal(t, uint64(10), follower.subscriber.LocalDAHeight(), "localDAHeight should not change") + assert.Equal(t, uint64(15), follower.subscriber.HighestSeenDAHeight(), "highestSeen should be updated") }) t.Run("falls_through_when_no_blobs", func(t *testing.T) { @@ -336,22 +340,24 @@ func TestDAFollower_InlineProcessing(t *testing.T) { follower := NewDAFollower(DAFollowerConfig{ Retriever: daRetriever, Logger: zerolog.Nop(), - PipeEvent: pipeEvent, + EventSink: common.EventSinkFunc(pipeEvent), Namespace: []byte("ns"), StartDAHeight: 10, DABlockTime: 500 * time.Millisecond, }).(*daFollower) // Subscription at current height but no blobs — should fall through - follower.handleSubscriptionEvent(t.Context(), datypes.SubscriptionEvent{ + // In production, the subscriber calls updateHighest before HandleEvent. + follower.subscriber.UpdateHighestForTest(10) + follower.HandleEvent(t.Context(), datypes.SubscriptionEvent{ Height: 10, Blobs: nil, }) // ProcessBlobs should NOT have been called daRetriever.AssertNotCalled(t, "ProcessBlobs", mock.Anything, mock.Anything, mock.Anything) - assert.Equal(t, uint64(10), follower.localNextDAHeight.Load(), "localNextDAHeight should not change") - assert.Equal(t, uint64(10), follower.highestSeenDAHeight.Load(), "highestSeen should be updated") + assert.Equal(t, uint64(10), follower.subscriber.LocalDAHeight(), "localDAHeight should not change") + assert.Equal(t, uint64(10), follower.subscriber.HighestSeenDAHeight(), "highestSeen should be updated") }) } diff --git a/block/internal/syncing/syncer_benchmark_test.go b/block/internal/syncing/syncer_benchmark_test.go index 9536808657..b0bccc6f74 100644 --- a/block/internal/syncing/syncer_benchmark_test.go +++ b/block/internal/syncing/syncer_benchmark_test.go @@ -43,25 +43,26 @@ func BenchmarkSyncerIO(b *testing.B) { fixt := newBenchFixture(b, spec.heights, spec.shuffledTx, spec.daDelay, spec.execDelay, true) // run both loops - runCtx := fixt.s.ctx - go fixt.s.processLoop(runCtx) + ctx := b.Context() + go fixt.s.processLoop(ctx) // Create a DAFollower to drive DA retrieval. follower := NewDAFollower(DAFollowerConfig{ Retriever: fixt.s.daRetriever, Logger: zerolog.Nop(), - PipeEvent: fixt.s.pipeEvent, + EventSink: fixt.s, Namespace: []byte("ns"), StartDAHeight: fixt.s.daRetrieverHeight.Load(), DABlockTime: 0, }).(*daFollower) - follower.highestSeenDAHeight.Store(spec.heights + daHeightOffset) - go follower.runCatchup(runCtx) + sub := follower.subscriber + sub.UpdateHighestForTest(spec.heights + daHeightOffset) + go sub.RunCatchupForTest(ctx) - fixt.s.startSyncWorkers(runCtx) + fixt.s.startSyncWorkers(ctx) require.Eventually(b, func() bool { - processedHeight, _ := fixt.s.store.Height(b.Context()) + processedHeight, _ := fixt.s.store.Height(ctx) return processedHeight == spec.heights }, 5*time.Second, 50*time.Microsecond) fixt.s.cancel() @@ -75,7 +76,7 @@ func BenchmarkSyncerIO(b *testing.B) { require.Len(b, fixt.s.heightInCh, 0) assert.Equal(b, spec.heights+daHeightOffset, fixt.s.daRetrieverHeight) - gotStoreHeight, err := fixt.s.store.Height(b.Context()) + gotStoreHeight, err := fixt.s.store.Height(ctx) require.NoError(b, err) assert.Equal(b, spec.heights, gotStoreHeight) } @@ -151,7 +152,7 @@ func newBenchFixture(b *testing.B, totalHeights uint64, shuffledTx bool, daDelay // Mock DA retriever to emit exactly totalHeights events, then HFF and cancel daR := NewMockDARetriever(b) - daR.On("PopPriorityHeight").Return(uint64(0)).Maybe() + for i := range totalHeights { daHeight := i + daHeightOffset daR.On("RetrieveFromDA", mock.Anything, daHeight). diff --git a/block/internal/syncing/syncer_forced_inclusion_test.go b/block/internal/syncing/syncer_forced_inclusion_test.go index bcb96d5bb1..110fa05609 100644 --- a/block/internal/syncing/syncer_forced_inclusion_test.go +++ b/block/internal/syncing/syncer_forced_inclusion_test.go @@ -74,9 +74,11 @@ func newForcedInclusionSyncer(t *testing.T, daStart, epochSize uint64) (*Syncer, client.On("GetDataNamespace").Return([]byte(cfg.DA.DataNamespace)).Maybe() client.On("GetForcedInclusionNamespace").Return([]byte(cfg.DA.ForcedInclusionNamespace)).Maybe() client.On("HasForcedInclusionNamespace").Return(true).Maybe() + subCh := make(chan datypes.SubscriptionEvent) + client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Maybe() daRetriever := NewDARetriever(client, cm, gen, zerolog.Nop()) - fiRetriever := da.NewForcedInclusionRetriever(client, zerolog.Nop(), cfg, gen.DAStartHeight, gen.DAEpochForcedInclusion) + fiRetriever := da.NewForcedInclusionRetriever(t.Context(), client, zerolog.Nop(), cfg.DA.BlockTime.Duration, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) t.Cleanup(fiRetriever.Stop) s := NewSyncer( @@ -145,8 +147,10 @@ func TestVerifyForcedInclusionTxs_NamespaceNotConfigured(t *testing.T) { client.On("GetDataNamespace").Return([]byte(cfg.DA.DataNamespace)).Maybe() client.On("GetForcedInclusionNamespace").Return([]byte(nil)).Maybe() client.On("HasForcedInclusionNamespace").Return(false).Maybe() + subCh := make(chan datypes.SubscriptionEvent) + client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Maybe() - fiRetriever := da.NewForcedInclusionRetriever(client, zerolog.Nop(), cfg, gen.DAStartHeight, gen.DAEpochForcedInclusion) + fiRetriever := da.NewForcedInclusionRetriever(t.Context(), client, zerolog.Nop(), cfg.DA.BlockTime.Duration, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) t.Cleanup(fiRetriever.Stop) s := NewSyncer( diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 771e71c284..757d4fb283 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -271,13 +271,13 @@ func TestSequentialBlockSync(t *testing.T) { assert.Equal(t, uint64(2), finalState.LastBlockHeight) // Verify DA inclusion markers are set - _, ok := cm.GetHeaderDAIncludedByHash(hdr1.Hash().String()) + _, ok := cm.GetHeaderDAIncluded(hdr1.Hash().String()) assert.True(t, ok) - _, ok = cm.GetHeaderDAIncludedByHash(hdr2.Hash().String()) + _, ok = cm.GetHeaderDAIncluded(hdr2.Hash().String()) assert.True(t, ok) - _, ok = cm.GetDataDAIncludedByHash(data1.DACommitment().String()) + _, ok = cm.GetDataDAIncluded(data1.DACommitment().String()) assert.True(t, ok) - _, ok = cm.GetDataDAIncludedByHash(data2.DACommitment().String()) + _, ok = cm.GetDataDAIncluded(data2.DACommitment().String()) assert.True(t, ok) requireEmptyChan(t, errChan) } @@ -375,8 +375,15 @@ func TestSyncLoopPersistState(t *testing.T) { daRtrMock, p2pHndlMock := NewMockDARetriever(t), newMockp2pHandler(t) p2pHndlMock.On("ProcessHeight", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() p2pHndlMock.On("SetProcessedHeight", mock.Anything).Return().Maybe() - daRtrMock.On("PopPriorityHeight").Return(uint64(0)).Maybe() + syncerInst1.daRetriever, syncerInst1.p2pHandler = daRtrMock, p2pHndlMock + syncerInst1.daFollower = NewDAFollower(DAFollowerConfig{ + Retriever: daRtrMock, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(syncerInst1.PipeEvent), + Namespace: []byte("ns"), + StartDAHeight: syncerInst1.daRetrieverHeight.Load(), + }) // with n da blobs fetched var prevHeaderHash, prevAppHash []byte @@ -422,21 +429,20 @@ func TestSyncLoopPersistState(t *testing.T) { follower1 := NewDAFollower(DAFollowerConfig{ Retriever: daRtrMock, Logger: zerolog.Nop(), - PipeEvent: syncerInst1.pipeEvent, + EventSink: common.EventSinkFunc(syncerInst1.PipeEvent), Namespace: []byte("ns"), StartDAHeight: syncerInst1.daRetrieverHeight.Load(), DABlockTime: cfg.DA.BlockTime.Duration, }).(*daFollower) - ctx, follower1.cancel = context.WithCancel(ctx) - // Set highest so catchup runs through all mocked heights. - follower1.highestSeenDAHeight.Store(myFutureDAHeight) - go follower1.runCatchup(ctx) + sub1 := follower1.subscriber + sub1.UpdateHighestForTest(myFutureDAHeight) + go sub1.RunCatchupForTest(ctx) syncerInst1.startSyncWorkers(ctx) syncerInst1.wg.Wait() requireEmptyChan(t, errorCh) t.Log("sync workers on instance1 completed") - require.Equal(t, myFutureDAHeight, follower1.localNextDAHeight.Load()) + require.Equal(t, myFutureDAHeight, follower1.subscriber.LocalDAHeight()) // wait for all events consumed require.NoError(t, cm.SaveToStore()) @@ -481,8 +487,15 @@ func TestSyncLoopPersistState(t *testing.T) { daRtrMock, p2pHndlMock = NewMockDARetriever(t), newMockp2pHandler(t) p2pHndlMock.On("ProcessHeight", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() p2pHndlMock.On("SetProcessedHeight", mock.Anything).Return().Maybe() - daRtrMock.On("PopPriorityHeight").Return(uint64(0)).Maybe() + syncerInst2.daRetriever, syncerInst2.p2pHandler = daRtrMock, p2pHndlMock + syncerInst2.daFollower = NewDAFollower(DAFollowerConfig{ + Retriever: daRtrMock, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(syncerInst2.PipeEvent), + Namespace: []byte("ns"), + StartDAHeight: syncerInst2.daRetrieverHeight.Load(), + }) daRtrMock.On("RetrieveFromDA", mock.Anything, mock.Anything). Run(func(arg mock.Arguments) { @@ -499,14 +512,14 @@ func TestSyncLoopPersistState(t *testing.T) { follower2 := NewDAFollower(DAFollowerConfig{ Retriever: daRtrMock, Logger: zerolog.Nop(), - PipeEvent: syncerInst2.pipeEvent, + EventSink: common.EventSinkFunc(syncerInst2.PipeEvent), Namespace: []byte("ns"), StartDAHeight: syncerInst2.daRetrieverHeight.Load(), DABlockTime: cfg.DA.BlockTime.Duration, }).(*daFollower) - ctx, follower2.cancel = context.WithCancel(ctx) - follower2.highestSeenDAHeight.Store(syncerInst2.daRetrieverHeight.Load() + 1) - go follower2.runCatchup(ctx) + sub2 := follower2.subscriber + sub2.UpdateHighestForTest(syncerInst2.daRetrieverHeight.Load() + 1) + go sub2.RunCatchupForTest(ctx) syncerInst2.startSyncWorkers(ctx) syncerInst2.wg.Wait() @@ -719,6 +732,13 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { // Create a real daRetriever to test priority queue s.daRetriever = NewDARetriever(nil, cm, gen, zerolog.Nop()) + s.daFollower = NewDAFollower(DAFollowerConfig{ + Retriever: s.daRetriever, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(func(_ context.Context, _ common.DAHeightEvent) error { return nil }), + Namespace: []byte("ns"), + StartDAHeight: 0, + }) // Create event with DA height hint evt := common.DAHeightEvent{ @@ -737,7 +757,7 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { s.processHeightEvent(t.Context(), &evt) // Verify that the priority height was queued in the daRetriever - priorityHeight := s.daRetriever.PopPriorityHeight() + priorityHeight := s.daFollower.(*daFollower).popPriorityHeight() assert.Equal(t, uint64(100), priorityHeight) } @@ -776,6 +796,13 @@ func TestProcessHeightEvent_RejectsUnreasonableDAHint(t *testing.T) { require.NoError(t, s.initializeState()) s.ctx = context.Background() s.daRetriever = NewDARetriever(nil, cm, gen, zerolog.Nop()) + s.daFollower = NewDAFollower(DAFollowerConfig{ + Retriever: s.daRetriever, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(func(_ context.Context, _ common.DAHeightEvent) error { return nil }), + Namespace: []byte("ns"), + StartDAHeight: 0, + }) // Set store height to 1 so event at height 2 is "next" batch, err := st.NewBatch(context.Background()) @@ -794,7 +821,7 @@ func TestProcessHeightEvent_RejectsUnreasonableDAHint(t *testing.T) { s.processHeightEvent(t.Context(), &evt) // Verify that NO priority height was queued — the hint was rejected - priorityHeight := s.daRetriever.PopPriorityHeight() + priorityHeight := s.daFollower.(*daFollower).popPriorityHeight() assert.Equal(t, uint64(0), priorityHeight, "unreasonable DA hint should be rejected") } @@ -833,6 +860,13 @@ func TestProcessHeightEvent_AcceptsValidDAHint(t *testing.T) { require.NoError(t, s.initializeState()) s.ctx = context.Background() s.daRetriever = NewDARetriever(nil, cm, gen, zerolog.Nop()) + s.daFollower = NewDAFollower(DAFollowerConfig{ + Retriever: s.daRetriever, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(func(_ context.Context, _ common.DAHeightEvent) error { return nil }), + Namespace: []byte("ns"), + StartDAHeight: 0, + }) // Set store height to 1 so event at height 2 is "next" batch, err := st.NewBatch(context.Background()) @@ -851,104 +885,11 @@ func TestProcessHeightEvent_AcceptsValidDAHint(t *testing.T) { s.processHeightEvent(t.Context(), &evt) // Verify that the priority height was queued — the hint is valid - priorityHeight := s.daRetriever.PopPriorityHeight() + priorityHeight := s.daFollower.(*daFollower).popPriorityHeight() assert.Equal(t, uint64(50), priorityHeight, "valid DA hint should be queued") } -// TestProcessHeightEvent_SkipsDAHintWhenAlreadyDAIncluded verifies that when the -// DA-inclusion cache already has an entry for the block height carried by a P2P -// event, the DA height hint is NOT queued for priority retrieval. -func TestProcessHeightEvent_SkipsDAHintWhenAlreadyDAIncluded(t *testing.T) { - ds := dssync.MutexWrap(datastore.NewMapDatastore()) - st := store.New(ds) - cm, err := cache.NewManager(config.DefaultConfig(), st, zerolog.Nop()) - require.NoError(t, err) - - addr, _, _ := buildSyncTestSigner(t) - cfg := config.DefaultConfig() - gen := genesis.Genesis{ChainID: "tchain", InitialHeight: 1, StartTime: time.Now().Add(-time.Second), ProposerAddress: addr} - - mockExec := testmocks.NewMockExecutor(t) - mockExec.EXPECT().InitChain(mock.Anything, mock.Anything, uint64(1), "tchain").Return([]byte("app0"), nil).Once() - - mockDAClient := testmocks.NewMockClient(t) - - s := NewSyncer( - st, - mockExec, - mockDAClient, - cm, - common.NopMetrics(), - cfg, - gen, - extmocks.NewMockStore[*types.P2PSignedHeader](t), - extmocks.NewMockStore[*types.P2PData](t), - zerolog.Nop(), - common.DefaultBlockOptions(), - make(chan error, 1), - nil, - ) - require.NoError(t, s.initializeState()) - s.ctx = context.Background() - s.daRetriever = NewDARetriever(nil, cm, gen, zerolog.Nop()) - - // Set the store height to 1 so the event at height 2 is "next". - batch, err := st.NewBatch(context.Background()) - require.NoError(t, err) - require.NoError(t, batch.SetHeight(1)) - require.NoError(t, batch.Commit()) - - // Simulate the DA retriever having already confirmed header and data at height 2. - cm.SetHeaderDAIncluded("somehash-hdr2", 42, 2) - cm.SetDataDAIncluded("somehash-data2", 42, 2) - - // Both hints point to DA height 100, which is above daRetrieverHeight (0), - // so the only reason NOT to queue them is the cache hit. - evt := common.DAHeightEvent{ - Header: &types.SignedHeader{Header: types.Header{BaseHeader: types.BaseHeader{ChainID: gen.ChainID, Height: 2}}}, - Data: &types.Data{Metadata: &types.Metadata{ChainID: gen.ChainID, Height: 2}}, - Source: common.SourceP2P, - DaHeightHints: [2]uint64{100, 100}, - } - - s.processHeightEvent(t.Context(), &evt) - - // Neither hint should be queued — both are already DA-included in the cache. - priorityHeight := s.daRetriever.PopPriorityHeight() - assert.Equal(t, uint64(0), priorityHeight, - "DA hint must not be queued when header and data are already DA-included in cache") - - // ── partial case: only header confirmed ────────────────────────────────── - // Reset with a fresh cache that has only the header entry for height 3. - cm2, err := cache.NewManager(config.DefaultConfig(), st, zerolog.Nop()) - require.NoError(t, err) - s.cache = cm2 - cm2.SetHeaderDAIncluded("somehash-hdr3", 55, 3) - // data at height 3 is NOT in cache - - batch, err = st.NewBatch(context.Background()) - require.NoError(t, err) - require.NoError(t, batch.SetHeight(2)) - require.NoError(t, batch.Commit()) - - evt3 := common.DAHeightEvent{ - Header: &types.SignedHeader{Header: types.Header{BaseHeader: types.BaseHeader{ChainID: gen.ChainID, Height: 3}}}, - Data: &types.Data{Metadata: &types.Metadata{ChainID: gen.ChainID, Height: 3}, Txs: types.Txs{types.Tx("tx2")}}, - Source: common.SourceP2P, - DaHeightHints: [2]uint64{150, 151}, // within daHintMaxDrift(200) of daRetrieverHeight(0) — no DA validation needed - } - - s.processHeightEvent(t.Context(), &evt3) - - // Header hint (150) must be skipped; data hint (151) must be queued. - priorityHeight = s.daRetriever.PopPriorityHeight() - assert.Equal(t, uint64(151), priorityHeight, - "data hint must be queued when only the header is already DA-included") - assert.Equal(t, uint64(0), s.daRetriever.PopPriorityHeight(), - "no further hints should be queued") -} - -func TestProcessHeightEvent_SkipsDAHintWhenBelowRetrieverCursor(t *testing.T) { +func TestProcessHeightEvent_SkipsDAHintWhenAlreadyFetched(t *testing.T) { ds := dssync.MutexWrap(datastore.NewMapDatastore()) st := store.New(ds) cm, err := cache.NewManager(config.DefaultConfig(), st, zerolog.Nop()) @@ -985,9 +926,15 @@ func TestProcessHeightEvent_SkipsDAHintWhenBelowRetrieverCursor(t *testing.T) { // Create a real daRetriever to test priority queue s.daRetriever = NewDARetriever(nil, cm, gen, zerolog.Nop()) + s.daFollower = NewDAFollower(DAFollowerConfig{ + Retriever: s.daRetriever, + Logger: zerolog.Nop(), + EventSink: common.EventSinkFunc(func(_ context.Context, _ common.DAHeightEvent) error { return nil }), + Namespace: []byte("ns"), + StartDAHeight: 0, + }) - // Set DA retriever height to 150 - the sequential DA scan cursor is already past - // the hint (100), so there is no need to queue a priority retrieval. + // Set DA retriever height to 150 - simulating we've already fetched past height 100 s.daRetrieverHeight.Store(150) // Set the store height to 1 so the event can be processed @@ -1006,9 +953,8 @@ func TestProcessHeightEvent_SkipsDAHintWhenBelowRetrieverCursor(t *testing.T) { s.processHeightEvent(t.Context(), &evt) - // Verify that no priority height was queued: the hint (100) is below the - // retriever cursor (150), so sequential scanning will cover it naturally. - priorityHeight := s.daRetriever.PopPriorityHeight() + // Verify that no priority height was queued since we've already fetched past it + priorityHeight := s.daFollower.(*daFollower).popPriorityHeight() assert.Equal(t, uint64(0), priorityHeight, "should not queue DA hint that is below current daRetrieverHeight") // Now test with a hint that is ABOVE the current daRetrieverHeight @@ -1028,7 +974,7 @@ func TestProcessHeightEvent_SkipsDAHintWhenBelowRetrieverCursor(t *testing.T) { s.processHeightEvent(t.Context(), &evt2) // Verify that the priority height WAS queued since it's above daRetrieverHeight - priorityHeight = s.daRetriever.PopPriorityHeight() + priorityHeight = s.daFollower.(*daFollower).popPriorityHeight() assert.Equal(t, uint64(200), priorityHeight, "should queue DA hint that is above current daRetrieverHeight") } diff --git a/block/public.go b/block/public.go index 39f6c3e308..988760211b 100644 --- a/block/public.go +++ b/block/public.go @@ -84,12 +84,13 @@ type ForcedInclusionRetriever interface { // It internally creates and manages an AsyncBlockRetriever for background prefetching. // Tracing is automatically enabled when configured. func NewForcedInclusionRetriever( + ctx context.Context, client DAClient, cfg config.Config, logger zerolog.Logger, daStartHeight, daEpochSize uint64, ) ForcedInclusionRetriever { - return da.NewForcedInclusionRetriever(client, logger, cfg, daStartHeight, daEpochSize) + return da.NewForcedInclusionRetriever(ctx, client, logger, cfg.DA.BlockTime.Duration, cfg.Instrumentation.IsTracingEnabled(), daStartHeight, daEpochSize) } // Expose Raft types for consensus integration diff --git a/pkg/sequencers/based/sequencer.go b/pkg/sequencers/based/sequencer.go index f079d8ccab..22bb8947f5 100644 --- a/pkg/sequencers/based/sequencer.go +++ b/pkg/sequencers/based/sequencer.go @@ -97,7 +97,7 @@ func NewBasedSequencer( } } - bs.fiRetriever = block.NewForcedInclusionRetriever(daClient, cfg, logger, genesis.DAStartHeight, genesis.DAEpochForcedInclusion) + bs.fiRetriever = block.NewForcedInclusionRetriever(context.Background(), daClient, cfg, logger, genesis.DAStartHeight, genesis.DAEpochForcedInclusion) return bs, nil } diff --git a/pkg/sequencers/based/sequencer_test.go b/pkg/sequencers/based/sequencer_test.go index c51a7673fe..7c65b8ad0a 100644 --- a/pkg/sequencers/based/sequencer_test.go +++ b/pkg/sequencers/based/sequencer_test.go @@ -71,6 +71,7 @@ func createTestSequencer(t *testing.T, mockRetriever *common.MockForcedInclusion // Mock the forced inclusion namespace call mockDAClient.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockExec := createDefaultMockExecutor(t) @@ -469,6 +470,7 @@ func TestBasedSequencer_CheckpointPersistence(t *testing.T) { } mockDAClient.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // Create first sequencer mockExec1 := createDefaultMockExecutor(t) @@ -496,6 +498,7 @@ func TestBasedSequencer_CheckpointPersistence(t *testing.T) { } mockDAClient2.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient2.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDAClient2.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockExec2 := createDefaultMockExecutor(t) seq2, err := NewBasedSequencer(mockDAClient2, config.DefaultConfig(), db, gen, zerolog.Nop(), mockExec2) require.NoError(t, err) @@ -542,6 +545,7 @@ func TestBasedSequencer_CrashRecoveryMidEpoch(t *testing.T) { mockDAClient.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // Create mock executor that postpones tx2 on first call filterCallCount := 0 @@ -602,6 +606,7 @@ func TestBasedSequencer_CrashRecoveryMidEpoch(t *testing.T) { } mockDAClient2.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient2.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDAClient2.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() seq2, err := NewBasedSequencer(mockDAClient2, config.DefaultConfig(), db, gen, zerolog.Nop(), mockExec) require.NoError(t, err) @@ -927,6 +932,7 @@ func TestBasedSequencer_GetNextBatch_GasFilteringPreservesUnprocessedTxs(t *test } mockDAClient.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() seq, err := NewBasedSequencer(mockDAClient, config.DefaultConfig(), db, gen, zerolog.New(zerolog.NewTestWriter(t)), mockExec) require.NoError(t, err) diff --git a/pkg/sequencers/single/sequencer.go b/pkg/sequencers/single/sequencer.go index 16cdf12b92..44bbb5b311 100644 --- a/pkg/sequencers/single/sequencer.go +++ b/pkg/sequencers/single/sequencer.go @@ -130,7 +130,7 @@ func NewSequencer( // Determine initial DA height for forced inclusion initialDAHeight := s.getInitialDAStartHeight(context.Background()) - s.fiRetriever = block.NewForcedInclusionRetriever(daClient, cfg, logger, initialDAHeight, genesis.DAEpochForcedInclusion) + s.fiRetriever = block.NewForcedInclusionRetriever(context.Background(), daClient, cfg, logger, initialDAHeight, genesis.DAEpochForcedInclusion) return s, nil } @@ -202,7 +202,7 @@ func (c *Sequencer) GetNextBatch(ctx context.Context, req coresequencer.GetNextB if c.fiRetriever != nil { c.fiRetriever.Stop() } - c.fiRetriever = block.NewForcedInclusionRetriever(c.daClient, c.cfg, c.logger, c.getInitialDAStartHeight(ctx), c.genesis.DAEpochForcedInclusion) + c.fiRetriever = block.NewForcedInclusionRetriever(ctx, c.daClient, c.cfg, c.logger, c.getInitialDAStartHeight(ctx), c.genesis.DAEpochForcedInclusion) } // If we have no cached transactions or we've consumed all from the current epoch, diff --git a/pkg/sequencers/single/sequencer_test.go b/pkg/sequencers/single/sequencer_test.go index b0bbe247c5..6392694ef5 100644 --- a/pkg/sequencers/single/sequencer_test.go +++ b/pkg/sequencers/single/sequencer_test.go @@ -380,6 +380,7 @@ func TestSequencer_GetNextBatch_ForcedInclusionAndBatch_MaxBytes(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — same as sequencer start, no catch-up needed mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(100), nil).Maybe() @@ -471,6 +472,7 @@ func TestSequencer_GetNextBatch_ForcedInclusion_ExceedsMaxBytes(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — same as sequencer start, no catch-up needed mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(100), nil).Maybe() @@ -554,6 +556,7 @@ func TestSequencer_GetNextBatch_AlwaysCheckPendingForcedInclusion(t *testing.T) mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — same as sequencer start, no catch-up needed mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(100), nil).Maybe() @@ -895,6 +898,7 @@ func TestSequencer_CheckpointPersistence_CrashRecovery(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 101 — close to sequencer start (100), no catch-up needed. // Use Maybe() since two sequencer instances share this mock. @@ -998,6 +1002,7 @@ func TestSequencer_GetNextBatch_EmptyDABatch_IncreasesDAHeight(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — same as sequencer start, no catch-up needed mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(100), nil).Maybe() @@ -1091,6 +1096,7 @@ func TestSequencer_GetNextBatch_WithGasFiltering(t *testing.T) { mockDA.MockClient.On("GetBlobsAtHeight", mock.Anything, mock.Anything, mock.Anything). Return(forcedTxs, nil).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return([]byte("forced")).Maybe() mockDA.MockClient.On("MaxBlobSize", mock.Anything).Return(uint64(1000000), nil).Maybe() @@ -1190,6 +1196,7 @@ func TestSequencer_GetNextBatch_GasFilterError(t *testing.T) { mockDA := newMockFullDAClient(t) mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return([]byte("forced")).Maybe() mockDA.MockClient.On("MaxBlobSize", mock.Anything).Return(uint64(1000000), nil).Maybe() @@ -1254,6 +1261,7 @@ func TestSequencer_CatchUp_DetectsOldEpoch(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at height 105 — sequencer starts at 100 with epoch size 1, // so it has missed 5 epochs (>1), triggering catch-up. @@ -1324,6 +1332,7 @@ func TestSequencer_CatchUp_SkipsMempoolDuringCatchUp(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — sequencer starts at 100 with epoch size 1, // so it has missed multiple epochs, triggering catch-up. @@ -1416,6 +1425,7 @@ func TestSequencer_CatchUp_UsesDATimestamp(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — multiple epochs ahead, triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(105), nil).Once() @@ -1475,6 +1485,7 @@ func TestSequencer_CatchUp_ExitsCatchUpAtDAHead(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — multiple epochs ahead, triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(105), nil).Once() @@ -1556,6 +1567,7 @@ func TestSequencer_CatchUp_HeightFromFutureExitsCatchUp(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — multiple epochs ahead, triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(105), nil).Once() @@ -1622,6 +1634,7 @@ func TestSequencer_CatchUp_NoCatchUpWhenRecentEpoch(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — sequencer starts at 100 with epoch size 1, // so it is within the same epoch (0 missed). No catch-up. @@ -1688,6 +1701,7 @@ func TestSequencer_CatchUp_MultiEpochReplay(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 106 — sequencer starts at 100 with epoch size 1, // so it has missed 6 epochs (>1), triggering catch-up. @@ -1839,6 +1853,7 @@ func TestSequencer_CatchUp_CheckpointAdvancesDuringCatchUp(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — multiple epochs ahead, triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(105), nil).Once() @@ -1931,6 +1946,7 @@ func TestSequencer_CatchUp_MonotonicTimestamps(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is far ahead — triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(110), nil).Once() @@ -2057,6 +2073,7 @@ func TestSequencer_CatchUp_MonotonicTimestamps_EmptyEpoch(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(110), nil).Once() @@ -2133,6 +2150,7 @@ func TestSequencer_GetNextBatch_GasFilteringPreservesUnprocessedTxs(t *testing.T mockDA := newMockFullDAClient(t) mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return([]byte("forced")).Maybe() mockDA.MockClient.On("MaxBlobSize", mock.Anything).Return(uint64(1000000), nil).Maybe() From 4b234de4c400acaa4b6b71b72030af13c13d5dfa Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 9 Mar 2026 12:44:54 +0100 Subject: [PATCH 15/17] Fix compile errors in tests after rebase --- block/internal/syncing/da_retriever_strict_test.go | 6 +++--- block/internal/syncing/syncer_test.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/block/internal/syncing/da_retriever_strict_test.go b/block/internal/syncing/da_retriever_strict_test.go index a6d5140d97..af1046dba8 100644 --- a/block/internal/syncing/da_retriever_strict_test.go +++ b/block/internal/syncing/da_retriever_strict_test.go @@ -70,21 +70,21 @@ func TestDARetriever_StrictEnvelopeMode_Switch(t *testing.T) { // --- Test Scenario --- // A. Initial State: StrictMode is false. Legacy blob should be accepted. - assert.False(t, r.strictMode.Load()) + assert.False(t, r.strictMode) decodedLegacy := r.tryDecodeHeader(legacyBlob, 100) require.NotNil(t, decodedLegacy) assert.Equal(t, uint64(1), decodedLegacy.Height()) // StrictMode should still be false because it was a legacy blob - assert.False(t, r.strictMode.Load()) + assert.False(t, r.strictMode) // B. Receiving Envelope: Should be accepted and Switch StrictMode to true. decodedEnvelope := r.tryDecodeHeader(envelopeBlob, 101) require.NotNil(t, decodedEnvelope) assert.Equal(t, uint64(2), decodedEnvelope.Height()) - assert.True(t, r.strictMode.Load(), "retriever should have switched to strict mode") + assert.True(t, r.strictMode, "retriever should have switched to strict mode") // C. Receiving Legacy again: Should be REJECTED now. // We reuse the same legacyBlob (or a new one, doesn't matter, structure is legacy). diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 757d4fb283..537b3e8ad9 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -271,13 +271,13 @@ func TestSequentialBlockSync(t *testing.T) { assert.Equal(t, uint64(2), finalState.LastBlockHeight) // Verify DA inclusion markers are set - _, ok := cm.GetHeaderDAIncluded(hdr1.Hash().String()) + _, ok := cm.GetHeaderDAIncludedByHash(hdr1.Hash().String()) assert.True(t, ok) - _, ok = cm.GetHeaderDAIncluded(hdr2.Hash().String()) + _, ok = cm.GetHeaderDAIncludedByHash(hdr2.Hash().String()) assert.True(t, ok) - _, ok = cm.GetDataDAIncluded(data1.DACommitment().String()) + _, ok = cm.GetDataDAIncludedByHash(data1.DACommitment().String()) assert.True(t, ok) - _, ok = cm.GetDataDAIncluded(data2.DACommitment().String()) + _, ok = cm.GetDataDAIncludedByHash(data2.DACommitment().String()) assert.True(t, ok) requireEmptyChan(t, errChan) } From d08236b3ffa0ab53e59fb07c4ff09118817bc448 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 9 Mar 2026 16:00:37 +0100 Subject: [PATCH 16/17] Include timestamp for subscription events --- apps/evm/server/force_inclusion_test.go | 2 +- block/internal/da/async_block_retriever.go | 43 ++++---- .../internal/da/async_block_retriever_test.go | 20 ++-- block/internal/da/client.go | 23 +++- .../da/forced_inclusion_retriever_test.go | 18 ++-- block/internal/da/interface.go | 3 +- block/internal/da/subscriber.go | 32 +++--- block/internal/da/tracing.go | 4 +- block/internal/da/tracing_test.go | 6 +- block/internal/syncing/da_retriever_mock.go | 102 +++++++++--------- .../syncing/syncer_forced_inclusion_test.go | 4 +- pkg/da/jsonrpc/types.go | 5 +- pkg/da/types/types.go | 3 + pkg/sequencers/based/sequencer_test.go | 12 +-- pkg/sequencers/single/sequencer_test.go | 36 +++---- test/mocks/da.go | 30 +++--- test/testda/dummy.go | 2 +- 17 files changed, 184 insertions(+), 161 deletions(-) diff --git a/apps/evm/server/force_inclusion_test.go b/apps/evm/server/force_inclusion_test.go index 6be6e6ca76..a107a10d11 100644 --- a/apps/evm/server/force_inclusion_test.go +++ b/apps/evm/server/force_inclusion_test.go @@ -50,7 +50,7 @@ func (m *mockDA) Get(ctx context.Context, ids []da.ID, namespace []byte) ([]da.B return nil, nil } -func (m *mockDA) Subscribe(_ context.Context, _ []byte) (<-chan da.SubscriptionEvent, error) { +func (m *mockDA) Subscribe(_ context.Context, _ []byte, _ bool) (<-chan da.SubscriptionEvent, error) { // Not needed in these tests; return a closed channel. ch := make(chan da.SubscriptionEvent) close(ch) diff --git a/block/internal/da/async_block_retriever.go b/block/internal/da/async_block_retriever.go index a0dcf7a578..6789051514 100644 --- a/block/internal/da/async_block_retriever.go +++ b/block/internal/da/async_block_retriever.go @@ -79,11 +79,12 @@ func NewAsyncBlockRetriever( } f.subscriber = NewSubscriber(SubscriberConfig{ - Client: client, - Logger: logger, - Namespaces: namespaces, - DABlockTime: daBlockTime, - Handler: f, + Client: client, + Logger: logger, + Namespaces: namespaces, + DABlockTime: daBlockTime, + Handler: f, + FetchBlockTimestamp: true, }) f.subscriber.SetStartHeight(daStartHeight) @@ -118,7 +119,7 @@ func (f *asyncBlockRetriever) UpdateCurrentHeight(height uint64) { f.logger.Debug(). Uint64("new_height", height). Msg("updated current DA height") - f.cleanupOldBlocks(height) + f.cleanupOldBlocks(context.Background(), height) return } } @@ -167,14 +168,10 @@ func (f *asyncBlockRetriever) GetCachedBlock(ctx context.Context, daHeight uint6 return block, nil } -// --------------------------------------------------------------------------- -// SubscriberHandler implementation -// --------------------------------------------------------------------------- - // HandleEvent caches blobs from the subscription inline. func (f *asyncBlockRetriever) HandleEvent(ctx context.Context, ev datypes.SubscriptionEvent) { if len(ev.Blobs) > 0 { - f.cacheBlock(ctx, ev.Height, ev.Blobs) + f.cacheBlock(ctx, ev.Height, ev.Timestamp, ev.Blobs) } } @@ -215,7 +212,7 @@ func (f *asyncBlockRetriever) fetchAndCacheBlock(ctx context.Context, height uin f.logger.Debug().Uint64("height", height).Msg("block height not yet available - will retry") return case datypes.StatusNotFound: - f.cacheBlock(ctx, height, nil) + f.cacheBlock(ctx, height, result.Timestamp, nil) case datypes.StatusSuccess: blobs := make([][]byte, 0, len(result.Data)) for _, blob := range result.Data { @@ -223,7 +220,7 @@ func (f *asyncBlockRetriever) fetchAndCacheBlock(ctx context.Context, height uin blobs = append(blobs, blob) } } - f.cacheBlock(ctx, height, blobs) + f.cacheBlock(ctx, height, result.Timestamp, blobs) default: f.logger.Debug(). Uint64("height", height). @@ -233,33 +230,33 @@ func (f *asyncBlockRetriever) fetchAndCacheBlock(ctx context.Context, height uin } // cacheBlock serializes and stores a block in the in-memory cache. -func (f *asyncBlockRetriever) cacheBlock(ctx context.Context, height uint64, blobs [][]byte) { +func (f *asyncBlockRetriever) cacheBlock(ctx context.Context, daHeight uint64, daTimestamp time.Time, blobs [][]byte) { if blobs == nil { blobs = [][]byte{} } pbBlock := &pb.BlockData{ - Height: height, - Timestamp: time.Now().UnixNano(), + Height: daHeight, + Timestamp: daTimestamp.UnixNano(), Blobs: blobs, } data, err := proto.Marshal(pbBlock) if err != nil { - f.logger.Error().Err(err).Uint64("height", height).Msg("failed to marshal block for caching") + f.logger.Error().Err(err).Uint64("height", daHeight).Msg("failed to marshal block for caching") return } - key := newBlockDataKey(height) + key := newBlockDataKey(daHeight) if err := f.cache.Put(ctx, key, data); err != nil { - f.logger.Error().Err(err).Uint64("height", height).Msg("failed to cache block") + f.logger.Error().Err(err).Uint64("height", daHeight).Msg("failed to cache block") return } - f.logger.Debug().Uint64("height", height).Int("blob_count", len(blobs)).Msg("cached block") + f.logger.Debug().Uint64("height", daHeight).Int("blob_count", len(blobs)).Msg("cached block") } // cleanupOldBlocks removes blocks older than currentHeight − prefetchWindow. -func (f *asyncBlockRetriever) cleanupOldBlocks(currentHeight uint64) { +func (f *asyncBlockRetriever) cleanupOldBlocks(ctx context.Context, currentHeight uint64) { if currentHeight < f.prefetchWindow { return } @@ -267,7 +264,7 @@ func (f *asyncBlockRetriever) cleanupOldBlocks(currentHeight uint64) { cleanupThreshold := currentHeight - f.prefetchWindow query := dsq.Query{Prefix: "/block/"} - results, err := f.cache.Query(context.Background(), query) + results, err := f.cache.Query(ctx, query) if err != nil { f.logger.Debug().Err(err).Msg("failed to query cache for cleanup") return @@ -287,7 +284,7 @@ func (f *asyncBlockRetriever) cleanupOldBlocks(currentHeight uint64) { } if height < cleanupThreshold { - if err := f.cache.Delete(context.Background(), key); err != nil { + if err := f.cache.Delete(ctx, key); err != nil { f.logger.Debug().Err(err).Uint64("height", height).Msg("failed to delete old block from cache") } } diff --git a/block/internal/da/async_block_retriever_test.go b/block/internal/da/async_block_retriever_test.go index d9c986e4e8..1ecaca9a88 100644 --- a/block/internal/da/async_block_retriever_test.go +++ b/block/internal/da/async_block_retriever_test.go @@ -60,7 +60,7 @@ func TestAsyncBlockRetriever_SubscriptionDrivenCaching(t *testing.T) { Blobs: testBlobs, } - client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() // Catchup loop may call Retrieve for heights beyond 100 — stub those. client.On("Retrieve", mock.Anything, mock.Anything, fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusHeightFromFuture}, @@ -68,7 +68,7 @@ func TestAsyncBlockRetriever_SubscriptionDrivenCaching(t *testing.T) { // On second subscribe (after watchdog timeout) just block forever. blockCh := make(chan datypes.SubscriptionEvent) - client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() logger := zerolog.Nop() fetcher := NewAsyncBlockRetriever(client, logger, fiNs, 200*time.Millisecond, 100, 5) @@ -109,9 +109,9 @@ func TestAsyncBlockRetriever_CatchupFillsGaps(t *testing.T) { subCh := make(chan datypes.SubscriptionEvent, 1) subCh <- datypes.SubscriptionEvent{Height: 105} - client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() blockCh := make(chan datypes.SubscriptionEvent) - client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() // Height 102 has blobs; rest return not found or future. client.On("Retrieve", mock.Anything, uint64(102), fiNs).Return(datypes.ResultRetrieve{ @@ -159,9 +159,9 @@ func TestAsyncBlockRetriever_HeightFromFuture(t *testing.T) { subCh := make(chan datypes.SubscriptionEvent, 1) subCh <- datypes.SubscriptionEvent{Height: 100} - client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() blockCh := make(chan datypes.SubscriptionEvent) - client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() // All Retrieve calls return HeightFromFuture. client.On("Retrieve", mock.Anything, mock.Anything, fiNs).Return(datypes.ResultRetrieve{ @@ -190,7 +190,7 @@ func TestAsyncBlockRetriever_StopGracefully(t *testing.T) { fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() blockCh := make(chan datypes.SubscriptionEvent) - client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() logger := zerolog.Nop() fetcher := NewAsyncBlockRetriever(client, logger, fiNs, 100*time.Millisecond, 100, 10) @@ -214,7 +214,7 @@ func TestAsyncBlockRetriever_ReconnectOnSubscriptionError(t *testing.T) { // First subscription closes immediately (simulating error). closedCh := make(chan datypes.SubscriptionEvent) close(closedCh) - client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(closedCh), nil).Once() + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(closedCh), nil).Once() // Second subscription delivers a blob. subCh := make(chan datypes.SubscriptionEvent, 1) @@ -222,11 +222,11 @@ func TestAsyncBlockRetriever_ReconnectOnSubscriptionError(t *testing.T) { Height: 100, Blobs: testBlobs, } - client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Once() // Third+ subscribe returns a blocking channel so it doesn't loop forever. blockCh := make(chan datypes.SubscriptionEvent) - client.On("Subscribe", mock.Anything, fiNs).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() + client.On("Subscribe", mock.Anything, fiNs, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(blockCh), nil).Maybe() // Stub Retrieve for catchup. client.On("Retrieve", mock.Anything, mock.Anything, fiNs).Return(datypes.ResultRetrieve{ diff --git a/block/internal/da/client.go b/block/internal/da/client.go index bb0f118436..0b30ed6466 100644 --- a/block/internal/da/client.go +++ b/block/internal/da/client.go @@ -355,7 +355,10 @@ func (c *client) HasForcedInclusionNamespace() bool { // DA block containing a matching blob. The channel is closed when ctx is // cancelled. The caller must drain the channel after cancellation to avoid // goroutine leaks. -func (c *client) Subscribe(ctx context.Context, namespace []byte) (<-chan datypes.SubscriptionEvent, error) { +// Timestamps are included from the header if available (celestia-node v0.29.1+§), otherwise +// fetched via a separate call when includeTimestamp is true. Be aware that fetching timestamps +// separately is an additional call to the celestia node for each event. +func (c *client) Subscribe(ctx context.Context, namespace []byte, includeTimestamp bool) (<-chan datypes.SubscriptionEvent, error) { ns, err := share.NewNamespaceFromBytes(namespace) if err != nil { return nil, fmt.Errorf("invalid namespace: %w", err) @@ -380,10 +383,24 @@ func (c *client) Subscribe(ctx context.Context, namespace []byte) (<-chan datype if resp == nil { continue } + var blockTime time.Time + // Use header time if available (celestia-node v0.21.0+) + if resp.Header != nil && !resp.Header.Time.IsZero() { + blockTime = resp.Header.Time + } else if includeTimestamp { + // Fallback to fetching timestamp for older nodes + blockTime, err = c.getBlockTimestamp(ctx, resp.Height) + if err != nil { + c.logger.Error().Uint64("height", resp.Height).Err(err).Msg("failed to get DA block timestamp for subscription event") + blockTime = time.Now() + // TODO: we should retry fetching the timestamp. Current time may mess block time consistency for based sequencers. + } + } select { case out <- datypes.SubscriptionEvent{ - Height: resp.Height, - Blobs: extractBlobData(resp), + Height: resp.Height, + Timestamp: blockTime, + Blobs: extractBlobData(resp), }: case <-ctx.Done(): return diff --git a/block/internal/da/forced_inclusion_retriever_test.go b/block/internal/da/forced_inclusion_retriever_test.go index 47864d7985..09a5e5b077 100644 --- a/block/internal/da/forced_inclusion_retriever_test.go +++ b/block/internal/da/forced_inclusion_retriever_test.go @@ -17,7 +17,7 @@ import ( func TestNewForcedInclusionRetriever(t *testing.T) { client := mocks.NewMockClient(t) client.On("GetForcedInclusionNamespace").Return(datypes.NamespaceFromString("test-fi-ns").Bytes()).Maybe() - client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() gen := genesis.Genesis{ DAStartHeight: 100, @@ -53,7 +53,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_NotAtEpochStart(t *t fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() - client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() gen := genesis.Genesis{ DAStartHeight: 100, @@ -84,7 +84,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EpochStartSuccess(t fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() - client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, mock.Anything, fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusSuccess, IDs: []datypes.ID{[]byte("id1"), []byte("id2"), []byte("id3")}, Timestamp: time.Now()}, Data: testBlobs, @@ -114,7 +114,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EpochStartNotAvailab fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() - client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // Mock the first height in epoch as not available client.On("Retrieve", mock.Anything, uint64(100), fiNs).Return(datypes.ResultRetrieve{ @@ -141,7 +141,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_NoBlobsAtHeight(t *t fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() - client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, uint64(100), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusNotFound}, }).Once() @@ -172,7 +172,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_MultiHeightEpoch(t * fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() - client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, uint64(102), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusSuccess, Timestamp: time.Now()}, Data: testBlobsByHeight[102], @@ -213,7 +213,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_ErrorHandling(t *tes fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() - client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, uint64(100), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{ Code: datypes.StatusError, @@ -242,7 +242,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_EmptyBlobsSkipped(t fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() - client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() client.On("Retrieve", mock.Anything, uint64(100), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusSuccess, Timestamp: time.Now()}, Data: [][]byte{[]byte("tx1"), {}, []byte("tx2"), nil, []byte("tx3")}, @@ -279,7 +279,7 @@ func TestForcedInclusionRetriever_RetrieveForcedIncludedTxs_OrderPreserved(t *te fiNs := datypes.NamespaceFromString("test-fi-ns").Bytes() client.On("HasForcedInclusionNamespace").Return(true).Maybe() client.On("GetForcedInclusionNamespace").Return(fiNs).Maybe() - client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // Return heights out of order to test ordering is preserved client.On("Retrieve", mock.Anything, uint64(102), fiNs).Return(datypes.ResultRetrieve{ BaseResult: datypes.BaseResult{Code: datypes.StatusSuccess, Timestamp: time.Now()}, diff --git a/block/internal/da/interface.go b/block/internal/da/interface.go index 3cc0677580..812d12847b 100644 --- a/block/internal/da/interface.go +++ b/block/internal/da/interface.go @@ -20,7 +20,8 @@ type Client interface { // Subscribe returns a channel that emits one SubscriptionEvent per DA block // that contains a blob in the given namespace. The channel is closed when ctx // is cancelled. Callers MUST drain the channel after cancellation. - Subscribe(ctx context.Context, namespace []byte) (<-chan datypes.SubscriptionEvent, error) + // The fetchTimestamp param is going to be removed with https://github.com/evstack/ev-node/issues/3142 as the timestamp is going to be included by default + Subscribe(ctx context.Context, namespace []byte, fetchTimestamp bool) (<-chan datypes.SubscriptionEvent, error) // GetLatestDAHeight returns the latest height available on the DA layer. GetLatestDAHeight(ctx context.Context) (uint64, error) diff --git a/block/internal/da/subscriber.go b/block/internal/da/subscriber.go index ffeb5141c9..779ff559f3 100644 --- a/block/internal/da/subscriber.go +++ b/block/internal/da/subscriber.go @@ -34,6 +34,8 @@ type SubscriberConfig struct { Namespaces [][]byte // subscribe to all, merge into one channel DABlockTime time.Duration Handler SubscriberHandler + // Deprecated: Remove with https://github.com/evstack/ev-node/issues/3142 + FetchBlockTimestamp bool // the timestamp comes with an extra api call before Celestia v0.29.1-mocha. } // Subscriber is a shared DA subscription primitive that encapsulates the @@ -66,6 +68,9 @@ type Subscriber struct { // daBlockTime used as backoff and watchdog base. daBlockTime time.Duration + // Deprecated: Remove with https://github.com/evstack/ev-node/issues/3142 + fetchBlockTimestamp bool + // lifecycle cancel context.CancelFunc wg sync.WaitGroup @@ -74,12 +79,13 @@ type Subscriber struct { // NewSubscriber creates a new Subscriber. func NewSubscriber(cfg SubscriberConfig) *Subscriber { s := &Subscriber{ - client: cfg.Client, - logger: cfg.Logger, - handler: cfg.Handler, - namespaces: cfg.Namespaces, - catchupSignal: make(chan struct{}, 1), - daBlockTime: cfg.DABlockTime, + client: cfg.Client, + logger: cfg.Logger, + handler: cfg.Handler, + namespaces: cfg.Namespaces, + catchupSignal: make(chan struct{}, 1), + daBlockTime: cfg.DABlockTime, + fetchBlockTimestamp: cfg.FetchBlockTimestamp, } return s } @@ -92,7 +98,7 @@ func (s *Subscriber) SetStartHeight(height uint64) { // Start begins the follow and catchup goroutines. func (s *Subscriber) Start(ctx context.Context) error { if len(s.namespaces) == 0 { - return nil // Nothing to subscribe to. + return errors.New("no namespaces configured") } ctx, s.cancel = context.WithCancel(ctx) @@ -227,7 +233,7 @@ func (s *Subscriber) subscribe(ctx context.Context) (<-chan datypes.Subscription } // Subscribe to the first namespace. - ch, err := s.client.Subscribe(ctx, s.namespaces[0]) + ch, err := s.client.Subscribe(ctx, s.namespaces[0], s.fetchBlockTimestamp) if err != nil { return nil, fmt.Errorf("subscribe namespace 0: %w", err) } @@ -237,7 +243,7 @@ func (s *Subscriber) subscribe(ctx context.Context) (<-chan datypes.Subscription if bytes.Equal(s.namespaces[i], s.namespaces[0]) { continue // Same namespace, skip duplicate. } - ch2, err := s.client.Subscribe(ctx, s.namespaces[i]) + ch2, err := s.client.Subscribe(ctx, s.namespaces[i], s.fetchBlockTimestamp) if err != nil { return nil, fmt.Errorf("subscribe namespace %d: %w", i, err) } @@ -296,10 +302,6 @@ func (s *Subscriber) updateHighest(height uint64) { } } -// --------------------------------------------------------------------------- -// Catchup loop -// --------------------------------------------------------------------------- - // catchupLoop waits for signals and sequentially catches up. func (s *Subscriber) catchupLoop(ctx context.Context) { defer s.wg.Done() @@ -370,10 +372,6 @@ func (s *Subscriber) waitOnCatchupError(ctx context.Context, err error, daHeight } } -// --------------------------------------------------------------------------- -// Timing helpers -// --------------------------------------------------------------------------- - func (s *Subscriber) backoff() time.Duration { if s.daBlockTime > 0 { return s.daBlockTime diff --git a/block/internal/da/tracing.go b/block/internal/da/tracing.go index 161293c2c3..3da79feddb 100644 --- a/block/internal/da/tracing.go +++ b/block/internal/da/tracing.go @@ -145,8 +145,8 @@ func (t *tracedClient) GetForcedInclusionNamespace() []byte { func (t *tracedClient) HasForcedInclusionNamespace() bool { return t.inner.HasForcedInclusionNamespace() } -func (t *tracedClient) Subscribe(ctx context.Context, namespace []byte) (<-chan datypes.SubscriptionEvent, error) { - return t.inner.Subscribe(ctx, namespace) +func (t *tracedClient) Subscribe(ctx context.Context, namespace []byte, includeTimestamp bool) (<-chan datypes.SubscriptionEvent, error) { + return t.inner.Subscribe(ctx, namespace, includeTimestamp) } type submitError struct{ msg string } diff --git a/block/internal/da/tracing_test.go b/block/internal/da/tracing_test.go index e20bf3b84b..fc1ae72f96 100644 --- a/block/internal/da/tracing_test.go +++ b/block/internal/da/tracing_test.go @@ -22,14 +22,14 @@ type mockFullClient struct { getFn func(ctx context.Context, ids []datypes.ID, namespace []byte) ([]datypes.Blob, error) getProofsFn func(ctx context.Context, ids []datypes.ID, namespace []byte) ([]datypes.Proof, error) validateFn func(ctx context.Context, ids []datypes.ID, proofs []datypes.Proof, namespace []byte) ([]bool, error) - subscribeFn func(ctx context.Context, namespace []byte) (<-chan datypes.SubscriptionEvent, error) + subscribeFn func(ctx context.Context, namespace []byte, ts bool) (<-chan datypes.SubscriptionEvent, error) } -func (m *mockFullClient) Subscribe(ctx context.Context, namespace []byte) (<-chan datypes.SubscriptionEvent, error) { +func (m *mockFullClient) Subscribe(ctx context.Context, namespace []byte, ts bool) (<-chan datypes.SubscriptionEvent, error) { if m.subscribeFn == nil { panic("not expected to be called") } - return m.subscribeFn(ctx, namespace) + return m.subscribeFn(ctx, namespace, ts) } func (m *mockFullClient) Submit(ctx context.Context, data [][]byte, gasPrice float64, namespace []byte, options []byte) datypes.ResultSubmit { diff --git a/block/internal/syncing/da_retriever_mock.go b/block/internal/syncing/da_retriever_mock.go index 7e8cc47be1..10e08bbd90 100644 --- a/block/internal/syncing/da_retriever_mock.go +++ b/block/internal/syncing/da_retriever_mock.go @@ -38,135 +38,135 @@ func (_m *MockDARetriever) EXPECT() *MockDARetriever_Expecter { return &MockDARetriever_Expecter{mock: &_m.Mock} } -// RetrieveFromDA provides a mock function for the type MockDARetriever -func (_mock *MockDARetriever) RetrieveFromDA(ctx context.Context, daHeight uint64) ([]common.DAHeightEvent, error) { - ret := _mock.Called(ctx, daHeight) +// ProcessBlobs provides a mock function for the type MockDARetriever +func (_mock *MockDARetriever) ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { + ret := _mock.Called(ctx, blobs, daHeight) if len(ret) == 0 { - panic("no return value specified for RetrieveFromDA") + panic("no return value specified for ProcessBlobs") } var r0 []common.DAHeightEvent - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, uint64) ([]common.DAHeightEvent, error)); ok { - return returnFunc(ctx, daHeight) - } - if returnFunc, ok := ret.Get(0).(func(context.Context, uint64) []common.DAHeightEvent); ok { - r0 = returnFunc(ctx, daHeight) + if returnFunc, ok := ret.Get(0).(func(context.Context, [][]byte, uint64) []common.DAHeightEvent); ok { + r0 = returnFunc(ctx, blobs, daHeight) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]common.DAHeightEvent) } } - if returnFunc, ok := ret.Get(1).(func(context.Context, uint64) error); ok { - r1 = returnFunc(ctx, daHeight) - } else { - r1 = ret.Error(1) - } - return r0, r1 + return r0 } -// MockDARetriever_RetrieveFromDA_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RetrieveFromDA' -type MockDARetriever_RetrieveFromDA_Call struct { +// MockDARetriever_ProcessBlobs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ProcessBlobs' +type MockDARetriever_ProcessBlobs_Call struct { *mock.Call } -// RetrieveFromDA is a helper method to define mock.On call +// ProcessBlobs is a helper method to define mock.On call // - ctx context.Context +// - blobs [][]byte // - daHeight uint64 -func (_e *MockDARetriever_Expecter) RetrieveFromDA(ctx interface{}, daHeight interface{}) *MockDARetriever_RetrieveFromDA_Call { - return &MockDARetriever_RetrieveFromDA_Call{Call: _e.mock.On("RetrieveFromDA", ctx, daHeight)} +func (_e *MockDARetriever_Expecter) ProcessBlobs(ctx interface{}, blobs interface{}, daHeight interface{}) *MockDARetriever_ProcessBlobs_Call { + return &MockDARetriever_ProcessBlobs_Call{Call: _e.mock.On("ProcessBlobs", ctx, blobs, daHeight)} } -func (_c *MockDARetriever_RetrieveFromDA_Call) Run(run func(ctx context.Context, daHeight uint64)) *MockDARetriever_RetrieveFromDA_Call { +func (_c *MockDARetriever_ProcessBlobs_Call) Run(run func(ctx context.Context, blobs [][]byte, daHeight uint64)) *MockDARetriever_ProcessBlobs_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } - var arg1 uint64 + var arg1 [][]byte if args[1] != nil { - arg1 = args[1].(uint64) + arg1 = args[1].([][]byte) + } + var arg2 uint64 + if args[2] != nil { + arg2 = args[2].(uint64) } run( arg0, arg1, + arg2, ) }) return _c } -func (_c *MockDARetriever_RetrieveFromDA_Call) Return(dAHeightEvents []common.DAHeightEvent, err error) *MockDARetriever_RetrieveFromDA_Call { - _c.Call.Return(dAHeightEvents, err) +func (_c *MockDARetriever_ProcessBlobs_Call) Return(dAHeightEvents []common.DAHeightEvent) *MockDARetriever_ProcessBlobs_Call { + _c.Call.Return(dAHeightEvents) return _c } -func (_c *MockDARetriever_RetrieveFromDA_Call) RunAndReturn(run func(ctx context.Context, daHeight uint64) ([]common.DAHeightEvent, error)) *MockDARetriever_RetrieveFromDA_Call { +func (_c *MockDARetriever_ProcessBlobs_Call) RunAndReturn(run func(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent) *MockDARetriever_ProcessBlobs_Call { _c.Call.Return(run) return _c } -// ProcessBlobs provides a mock function for the type MockDARetriever -func (_mock *MockDARetriever) ProcessBlobs(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent { - ret := _mock.Called(ctx, blobs, daHeight) +// RetrieveFromDA provides a mock function for the type MockDARetriever +func (_mock *MockDARetriever) RetrieveFromDA(ctx context.Context, daHeight uint64) ([]common.DAHeightEvent, error) { + ret := _mock.Called(ctx, daHeight) if len(ret) == 0 { - panic("no return value specified for ProcessBlobs") + panic("no return value specified for RetrieveFromDA") } var r0 []common.DAHeightEvent - if returnFunc, ok := ret.Get(0).(func(context.Context, [][]byte, uint64) []common.DAHeightEvent); ok { - r0 = returnFunc(ctx, blobs, daHeight) + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, uint64) ([]common.DAHeightEvent, error)); ok { + return returnFunc(ctx, daHeight) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, uint64) []common.DAHeightEvent); ok { + r0 = returnFunc(ctx, daHeight) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]common.DAHeightEvent) } } - return r0 + if returnFunc, ok := ret.Get(1).(func(context.Context, uint64) error); ok { + r1 = returnFunc(ctx, daHeight) + } else { + r1 = ret.Error(1) + } + return r0, r1 } -// MockDARetriever_ProcessBlobs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ProcessBlobs' -type MockDARetriever_ProcessBlobs_Call struct { +// MockDARetriever_RetrieveFromDA_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RetrieveFromDA' +type MockDARetriever_RetrieveFromDA_Call struct { *mock.Call } -// ProcessBlobs is a helper method to define mock.On call +// RetrieveFromDA is a helper method to define mock.On call // - ctx context.Context -// - blobs [][]byte // - daHeight uint64 -func (_e *MockDARetriever_Expecter) ProcessBlobs(ctx interface{}, blobs interface{}, daHeight interface{}) *MockDARetriever_ProcessBlobs_Call { - return &MockDARetriever_ProcessBlobs_Call{Call: _e.mock.On("ProcessBlobs", ctx, blobs, daHeight)} +func (_e *MockDARetriever_Expecter) RetrieveFromDA(ctx interface{}, daHeight interface{}) *MockDARetriever_RetrieveFromDA_Call { + return &MockDARetriever_RetrieveFromDA_Call{Call: _e.mock.On("RetrieveFromDA", ctx, daHeight)} } -func (_c *MockDARetriever_ProcessBlobs_Call) Run(run func(ctx context.Context, blobs [][]byte, daHeight uint64)) *MockDARetriever_ProcessBlobs_Call { +func (_c *MockDARetriever_RetrieveFromDA_Call) Run(run func(ctx context.Context, daHeight uint64)) *MockDARetriever_RetrieveFromDA_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } - var arg1 [][]byte + var arg1 uint64 if args[1] != nil { - arg1 = args[1].([][]byte) - } - var arg2 uint64 - if args[2] != nil { - arg2 = args[2].(uint64) + arg1 = args[1].(uint64) } run( arg0, arg1, - arg2, ) }) return _c } -func (_c *MockDARetriever_ProcessBlobs_Call) Return(dAHeightEvents []common.DAHeightEvent) *MockDARetriever_ProcessBlobs_Call { - _c.Call.Return(dAHeightEvents) +func (_c *MockDARetriever_RetrieveFromDA_Call) Return(dAHeightEvents []common.DAHeightEvent, err error) *MockDARetriever_RetrieveFromDA_Call { + _c.Call.Return(dAHeightEvents, err) return _c } -func (_c *MockDARetriever_ProcessBlobs_Call) RunAndReturn(run func(ctx context.Context, blobs [][]byte, daHeight uint64) []common.DAHeightEvent) *MockDARetriever_ProcessBlobs_Call { +func (_c *MockDARetriever_RetrieveFromDA_Call) RunAndReturn(run func(ctx context.Context, daHeight uint64) ([]common.DAHeightEvent, error)) *MockDARetriever_RetrieveFromDA_Call { _c.Call.Return(run) return _c } diff --git a/block/internal/syncing/syncer_forced_inclusion_test.go b/block/internal/syncing/syncer_forced_inclusion_test.go index 110fa05609..daba0322dd 100644 --- a/block/internal/syncing/syncer_forced_inclusion_test.go +++ b/block/internal/syncing/syncer_forced_inclusion_test.go @@ -75,7 +75,7 @@ func newForcedInclusionSyncer(t *testing.T, daStart, epochSize uint64) (*Syncer, client.On("GetForcedInclusionNamespace").Return([]byte(cfg.DA.ForcedInclusionNamespace)).Maybe() client.On("HasForcedInclusionNamespace").Return(true).Maybe() subCh := make(chan datypes.SubscriptionEvent) - client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Maybe() daRetriever := NewDARetriever(client, cm, gen, zerolog.Nop()) fiRetriever := da.NewForcedInclusionRetriever(t.Context(), client, zerolog.Nop(), cfg.DA.BlockTime.Duration, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) @@ -148,7 +148,7 @@ func TestVerifyForcedInclusionTxs_NamespaceNotConfigured(t *testing.T) { client.On("GetForcedInclusionNamespace").Return([]byte(nil)).Maybe() client.On("HasForcedInclusionNamespace").Return(false).Maybe() subCh := make(chan datypes.SubscriptionEvent) - client.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Maybe() + client.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(subCh), nil).Maybe() fiRetriever := da.NewForcedInclusionRetriever(t.Context(), client, zerolog.Nop(), cfg.DA.BlockTime.Duration, false, gen.DAStartHeight, gen.DAEpochForcedInclusion) t.Cleanup(fiRetriever.Stop) diff --git a/pkg/da/jsonrpc/types.go b/pkg/da/jsonrpc/types.go index b7377e950e..daecb9ec71 100644 --- a/pkg/da/jsonrpc/types.go +++ b/pkg/da/jsonrpc/types.go @@ -8,6 +8,7 @@ type CommitmentProof struct { // SubscriptionResponse mirrors celestia-node's blob.SubscriptionResponse. type SubscriptionResponse struct { - Blobs []*Blob `json:"blobs"` - Height uint64 `json:"height"` + Blobs []*Blob `json:"blobs"` + Height uint64 `json:"height"` + Header *RawHeader `json:"header,omitempty"` // Available in celestia-node v0.29.1+ } diff --git a/pkg/da/types/types.go b/pkg/da/types/types.go index 8c11ce5cb5..24bcceea84 100644 --- a/pkg/da/types/types.go +++ b/pkg/da/types/types.go @@ -88,6 +88,9 @@ func SplitID(id []byte) (uint64, []byte, error) { type SubscriptionEvent struct { // Height is the DA layer height at which the blob was finalized. Height uint64 + // Timestamp is the DA block timestamp at the given height. + // This is an optional field that is not set by default. Enable via SubscribeExtra method + Timestamp time.Time // Blobs contains the raw blob data from the subscription response. // When non-nil, followers can process blobs inline without re-fetching from DA. Blobs [][]byte diff --git a/pkg/sequencers/based/sequencer_test.go b/pkg/sequencers/based/sequencer_test.go index 7c65b8ad0a..b4690d527d 100644 --- a/pkg/sequencers/based/sequencer_test.go +++ b/pkg/sequencers/based/sequencer_test.go @@ -71,7 +71,7 @@ func createTestSequencer(t *testing.T, mockRetriever *common.MockForcedInclusion // Mock the forced inclusion namespace call mockDAClient.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockExec := createDefaultMockExecutor(t) @@ -470,7 +470,7 @@ func TestBasedSequencer_CheckpointPersistence(t *testing.T) { } mockDAClient.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // Create first sequencer mockExec1 := createDefaultMockExecutor(t) @@ -498,7 +498,7 @@ func TestBasedSequencer_CheckpointPersistence(t *testing.T) { } mockDAClient2.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient2.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDAClient2.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDAClient2.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockExec2 := createDefaultMockExecutor(t) seq2, err := NewBasedSequencer(mockDAClient2, config.DefaultConfig(), db, gen, zerolog.Nop(), mockExec2) require.NoError(t, err) @@ -545,7 +545,7 @@ func TestBasedSequencer_CrashRecoveryMidEpoch(t *testing.T) { mockDAClient.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // Create mock executor that postpones tx2 on first call filterCallCount := 0 @@ -606,7 +606,7 @@ func TestBasedSequencer_CrashRecoveryMidEpoch(t *testing.T) { } mockDAClient2.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient2.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDAClient2.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDAClient2.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() seq2, err := NewBasedSequencer(mockDAClient2, config.DefaultConfig(), db, gen, zerolog.Nop(), mockExec) require.NoError(t, err) @@ -932,7 +932,7 @@ func TestBasedSequencer_GetNextBatch_GasFilteringPreservesUnprocessedTxs(t *test } mockDAClient.MockClient.On("GetForcedInclusionNamespace").Return([]byte("test-forced-inclusion-ns")).Maybe() mockDAClient.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDAClient.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() seq, err := NewBasedSequencer(mockDAClient, config.DefaultConfig(), db, gen, zerolog.New(zerolog.NewTestWriter(t)), mockExec) require.NoError(t, err) diff --git a/pkg/sequencers/single/sequencer_test.go b/pkg/sequencers/single/sequencer_test.go index 6392694ef5..0a393d4275 100644 --- a/pkg/sequencers/single/sequencer_test.go +++ b/pkg/sequencers/single/sequencer_test.go @@ -380,7 +380,7 @@ func TestSequencer_GetNextBatch_ForcedInclusionAndBatch_MaxBytes(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — same as sequencer start, no catch-up needed mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(100), nil).Maybe() @@ -472,7 +472,7 @@ func TestSequencer_GetNextBatch_ForcedInclusion_ExceedsMaxBytes(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — same as sequencer start, no catch-up needed mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(100), nil).Maybe() @@ -556,7 +556,7 @@ func TestSequencer_GetNextBatch_AlwaysCheckPendingForcedInclusion(t *testing.T) mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — same as sequencer start, no catch-up needed mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(100), nil).Maybe() @@ -898,7 +898,7 @@ func TestSequencer_CheckpointPersistence_CrashRecovery(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 101 — close to sequencer start (100), no catch-up needed. // Use Maybe() since two sequencer instances share this mock. @@ -1002,7 +1002,7 @@ func TestSequencer_GetNextBatch_EmptyDABatch_IncreasesDAHeight(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — same as sequencer start, no catch-up needed mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(100), nil).Maybe() @@ -1096,7 +1096,7 @@ func TestSequencer_GetNextBatch_WithGasFiltering(t *testing.T) { mockDA.MockClient.On("GetBlobsAtHeight", mock.Anything, mock.Anything, mock.Anything). Return(forcedTxs, nil).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return([]byte("forced")).Maybe() mockDA.MockClient.On("MaxBlobSize", mock.Anything).Return(uint64(1000000), nil).Maybe() @@ -1196,7 +1196,7 @@ func TestSequencer_GetNextBatch_GasFilterError(t *testing.T) { mockDA := newMockFullDAClient(t) mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return([]byte("forced")).Maybe() mockDA.MockClient.On("MaxBlobSize", mock.Anything).Return(uint64(1000000), nil).Maybe() @@ -1261,7 +1261,7 @@ func TestSequencer_CatchUp_DetectsOldEpoch(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at height 105 — sequencer starts at 100 with epoch size 1, // so it has missed 5 epochs (>1), triggering catch-up. @@ -1332,7 +1332,7 @@ func TestSequencer_CatchUp_SkipsMempoolDuringCatchUp(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — sequencer starts at 100 with epoch size 1, // so it has missed multiple epochs, triggering catch-up. @@ -1425,7 +1425,7 @@ func TestSequencer_CatchUp_UsesDATimestamp(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — multiple epochs ahead, triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(105), nil).Once() @@ -1485,7 +1485,7 @@ func TestSequencer_CatchUp_ExitsCatchUpAtDAHead(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — multiple epochs ahead, triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(105), nil).Once() @@ -1567,7 +1567,7 @@ func TestSequencer_CatchUp_HeightFromFutureExitsCatchUp(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — multiple epochs ahead, triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(105), nil).Once() @@ -1634,7 +1634,7 @@ func TestSequencer_CatchUp_NoCatchUpWhenRecentEpoch(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 100 — sequencer starts at 100 with epoch size 1, // so it is within the same epoch (0 missed). No catch-up. @@ -1701,7 +1701,7 @@ func TestSequencer_CatchUp_MultiEpochReplay(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 106 — sequencer starts at 100 with epoch size 1, // so it has missed 6 epochs (>1), triggering catch-up. @@ -1853,7 +1853,7 @@ func TestSequencer_CatchUp_CheckpointAdvancesDuringCatchUp(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is at 105 — multiple epochs ahead, triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(105), nil).Once() @@ -1946,7 +1946,7 @@ func TestSequencer_CatchUp_MonotonicTimestamps(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() // DA head is far ahead — triggers catch-up mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(110), nil).Once() @@ -2073,7 +2073,7 @@ func TestSequencer_CatchUp_MonotonicTimestamps_EmptyEpoch(t *testing.T) { mockDA.MockClient.On("GetDataNamespace").Return([]byte("data")).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return(forcedInclusionNS).Maybe() mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockDA.MockClient.On("GetLatestDAHeight", mock.Anything).Return(uint64(110), nil).Once() @@ -2150,7 +2150,7 @@ func TestSequencer_GetNextBatch_GasFilteringPreservesUnprocessedTxs(t *testing.T mockDA := newMockFullDAClient(t) mockDA.MockClient.On("HasForcedInclusionNamespace").Return(true).Maybe() - mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() + mockDA.MockClient.On("Subscribe", mock.Anything, mock.Anything, mock.Anything).Return((<-chan datypes.SubscriptionEvent)(make(chan datypes.SubscriptionEvent)), nil).Maybe() mockDA.MockClient.On("GetForcedInclusionNamespace").Return([]byte("forced")).Maybe() mockDA.MockClient.On("MaxBlobSize", mock.Anything).Return(uint64(1000000), nil).Maybe() diff --git a/test/mocks/da.go b/test/mocks/da.go index 27ce9badf3..f3a4e960db 100644 --- a/test/mocks/da.go +++ b/test/mocks/da.go @@ -493,8 +493,8 @@ func (_c *MockClient_Submit_Call) RunAndReturn(run func(ctx context.Context, dat } // Subscribe provides a mock function for the type MockClient -func (_mock *MockClient) Subscribe(ctx context.Context, namespace []byte) (<-chan da.SubscriptionEvent, error) { - ret := _mock.Called(ctx, namespace) +func (_mock *MockClient) Subscribe(ctx context.Context, namespace []byte, fetchTimestamp bool) (<-chan da.SubscriptionEvent, error) { + ret := _mock.Called(ctx, namespace, fetchTimestamp) if len(ret) == 0 { panic("no return value specified for Subscribe") @@ -502,18 +502,18 @@ func (_mock *MockClient) Subscribe(ctx context.Context, namespace []byte) (<-cha var r0 <-chan da.SubscriptionEvent var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, []byte) (<-chan da.SubscriptionEvent, error)); ok { - return returnFunc(ctx, namespace) + if returnFunc, ok := ret.Get(0).(func(context.Context, []byte, bool) (<-chan da.SubscriptionEvent, error)); ok { + return returnFunc(ctx, namespace, fetchTimestamp) } - if returnFunc, ok := ret.Get(0).(func(context.Context, []byte) <-chan da.SubscriptionEvent); ok { - r0 = returnFunc(ctx, namespace) + if returnFunc, ok := ret.Get(0).(func(context.Context, []byte, bool) <-chan da.SubscriptionEvent); ok { + r0 = returnFunc(ctx, namespace, fetchTimestamp) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(<-chan da.SubscriptionEvent) } } - if returnFunc, ok := ret.Get(1).(func(context.Context, []byte) error); ok { - r1 = returnFunc(ctx, namespace) + if returnFunc, ok := ret.Get(1).(func(context.Context, []byte, bool) error); ok { + r1 = returnFunc(ctx, namespace, fetchTimestamp) } else { r1 = ret.Error(1) } @@ -528,11 +528,12 @@ type MockClient_Subscribe_Call struct { // Subscribe is a helper method to define mock.On call // - ctx context.Context // - namespace []byte -func (_e *MockClient_Expecter) Subscribe(ctx interface{}, namespace interface{}) *MockClient_Subscribe_Call { - return &MockClient_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, namespace)} +// - fetchTimestamp bool +func (_e *MockClient_Expecter) Subscribe(ctx interface{}, namespace interface{}, fetchTimestamp interface{}) *MockClient_Subscribe_Call { + return &MockClient_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, namespace, fetchTimestamp)} } -func (_c *MockClient_Subscribe_Call) Run(run func(ctx context.Context, namespace []byte)) *MockClient_Subscribe_Call { +func (_c *MockClient_Subscribe_Call) Run(run func(ctx context.Context, namespace []byte, fetchTimestamp bool)) *MockClient_Subscribe_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -542,9 +543,14 @@ func (_c *MockClient_Subscribe_Call) Run(run func(ctx context.Context, namespace if args[1] != nil { arg1 = args[1].([]byte) } + var arg2 bool + if args[2] != nil { + arg2 = args[2].(bool) + } run( arg0, arg1, + arg2, ) }) return _c @@ -555,7 +561,7 @@ func (_c *MockClient_Subscribe_Call) Return(subscriptionEventCh <-chan da.Subscr return _c } -func (_c *MockClient_Subscribe_Call) RunAndReturn(run func(ctx context.Context, namespace []byte) (<-chan da.SubscriptionEvent, error)) *MockClient_Subscribe_Call { +func (_c *MockClient_Subscribe_Call) RunAndReturn(run func(ctx context.Context, namespace []byte, fetchTimestamp bool) (<-chan da.SubscriptionEvent, error)) *MockClient_Subscribe_Call { _c.Call.Return(run) return _c } diff --git a/test/testda/dummy.go b/test/testda/dummy.go index 44fc565c60..2f92e47e60 100644 --- a/test/testda/dummy.go +++ b/test/testda/dummy.go @@ -54,7 +54,7 @@ type DummyDA struct { // Subscribe returns a channel that emits a SubscriptionEvent for every new DA // height produced by Submit or StartHeightTicker. The channel is closed when // ctx is cancelled or Reset is called. -func (d *DummyDA) Subscribe(ctx context.Context, _ []byte) (<-chan datypes.SubscriptionEvent, error) { +func (d *DummyDA) Subscribe(ctx context.Context, _ []byte, _ bool) (<-chan datypes.SubscriptionEvent, error) { ch := make(chan datypes.SubscriptionEvent, 64) sub := &subscriber{ch: ch, ctx: ctx} From b07b397c0cfcd9e4bacab79a06c051ef47708133 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 9 Mar 2026 16:29:33 +0100 Subject: [PATCH 17/17] Doc update (cherry picked from commit 7a2adbd8a24df57e6e5db8aeaf8d3a57396ff750) --- docs/guides/da-layers/celestia.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guides/da-layers/celestia.md b/docs/guides/da-layers/celestia.md index 88bbd48e4c..307dde63e9 100644 --- a/docs/guides/da-layers/celestia.md +++ b/docs/guides/da-layers/celestia.md @@ -72,10 +72,10 @@ You can use the same namespace for both headers and data, or separate them for o ### Set DA Address -Default Celestia light node port is 26658: +Default Celestia light node port is 26658. **Note:** Connection to a celestia-node DA requires a websocket connection: ```bash -DA_ADDRESS=http://localhost:26658 +DA_ADDRESS=ws://localhost:26658 ``` ## Running Your Chain