From 1988e9e9d04af5e5ae4e36a46cc64997ad27ddc4 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 4 Dec 2025 17:22:54 +0800 Subject: [PATCH 01/16] alert when blob missing from cl --- cmd/es-node/config.go | 10 +++++++++- ethstorage/archiver/service.go | 2 +- ethstorage/downloader/config.go | 3 +++ ethstorage/downloader/downloader.go | 25 +++++++++++++++++++------ ethstorage/node/node.go | 1 + 5 files changed, 33 insertions(+), 8 deletions(-) diff --git a/cmd/es-node/config.go b/cmd/es-node/config.go index 9c778939..4eb7f2c9 100644 --- a/cmd/es-node/config.go +++ b/cmd/es-node/config.go @@ -255,9 +255,17 @@ func NewL1EndpointConfig(ctx *cli.Context, lg log.Logger) (*eth.L1EndpointConfig } func NewDownloaderConfig(ctx *cli.Context) *downloader.Config { - return &downloader.Config{ + dlCfg := &downloader.Config{ DownloadStart: ctx.GlobalInt64(flags.DownloadStart.Name), DownloadDump: ctx.GlobalString(flags.DownloadDump.Name), DownloadThreadNum: ctx.GlobalInt(flags.DownloadThreadNum.Name), } + + emailConfig, err := email.GetEmailConfig(ctx) + if err != nil { + // email is nice to have but not required by downloader + return dlCfg + } + dlCfg.EmailConfig = *emailConfig + return dlCfg } diff --git a/ethstorage/archiver/service.go b/ethstorage/archiver/service.go index 6d702ef5..3462d71c 100644 --- a/ethstorage/archiver/service.go +++ b/ethstorage/archiver/service.go @@ -119,7 +119,7 @@ func (a *APIService) Start(ctx context.Context) error { return err } r := mux.NewRouter() - // Deprecated + // Deprecated by Fusaka but still used by OP Stack r.HandleFunc("/eth/v1/beacon/blob_sidecars/{id}", a.blobSidecarHandler) // Fusaka r.HandleFunc("/eth/v1/beacon/blobs/{id}", a.blobsHandler) diff --git a/ethstorage/downloader/config.go b/ethstorage/downloader/config.go index b094f199..8d8f1d95 100644 --- a/ethstorage/downloader/config.go +++ b/ethstorage/downloader/config.go @@ -3,8 +3,11 @@ package downloader +import "github.com/ethstorage/go-ethstorage/ethstorage/email" + type Config struct { DownloadStart int64 // which block should we download the blobs from DownloadDump string // where to dump the download blobs DownloadThreadNum int // how many threads that will be used to download the blobs into storage file + EmailConfig email.EmailConfig } diff --git a/ethstorage/downloader/downloader.go b/ethstorage/downloader/downloader.go index f93ff1ee..054f62bf 100644 --- a/ethstorage/downloader/downloader.go +++ b/ethstorage/downloader/downloader.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/rpc" "github.com/ethstorage/go-ethstorage/ethstorage" + "github.com/ethstorage/go-ethstorage/ethstorage/email" "github.com/ethstorage/go-ethstorage/ethstorage/eth" ) @@ -70,10 +71,11 @@ type Downloader struct { dlLatestReq chan struct{} dlFinalizedReq chan struct{} - lg log.Logger - done chan struct{} - wg sync.WaitGroup - mu sync.Mutex + emailConfig *email.EmailConfig + lg log.Logger + done chan struct{} + wg sync.WaitGroup + mu sync.Mutex } type blob struct { @@ -109,6 +111,7 @@ func NewDownloader( downloadDump string, minDurationForBlobsRequest uint64, downloadThreadNum int, + emailConfig email.EmailConfig, lg log.Logger, ) *Downloader { sm.DownloadThreadNum = downloadThreadNum @@ -127,6 +130,7 @@ func NewDownloader( done: make(chan struct{}), lastDownloadBlock: downloadStart, downloadedBlobs: 0, + emailConfig: &emailConfig, } } @@ -411,8 +415,17 @@ func (s *Downloader) downloadRange(start int64, end int64, toCache bool) ([]blob for _, elBlob := range elBlock.blobs { clBlob, exists := clBlobs[elBlob.hash] if !exists { - s.lg.Error("Did not find the event specified blob in the CL") - + if s.emailConfig != nil { + msg := fmt.Sprintf("Did not find the event specified blob in the CL, blockNumber: %d, kvIndex: %d\n", elBlock.number, elBlob.kvIndex) + msg += "This may indicate that the blob has not been published to the consensus layer yet, or there is an issue with the consensus layer blob availability.\n" + email.SendEmail( + "Blob missing in CL for downloader", + msg, + *s.emailConfig, + s.lg, + ) + } + s.lg.Crit("Did not find the event specified blob in the CL", "blockNumber", elBlock.number, "kvIndex", elBlob.kvIndex) } // encode blobs so that miner can do sampling directly from cache elBlob.data = s.sm.EncodeBlob(clBlob.Data, elBlob.hash, elBlob.kvIndex.Uint64(), s.sm.MaxKvSize()) diff --git a/ethstorage/node/node.go b/ethstorage/node/node.go index 4f20bd49..c8200a0c 100644 --- a/ethstorage/node/node.go +++ b/ethstorage/node/node.go @@ -143,6 +143,7 @@ func (n *EsNode) initL2(ctx context.Context, cfg *Config) error { cfg.Downloader.DownloadDump, cfg.L1.L1MinDurationForBlobsRequest, cfg.Downloader.DownloadThreadNum, + cfg.Downloader.EmailConfig, n.lg, ) return nil From 8303198e62c87ad8736321a2c814af012c706aff Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 4 Dec 2025 17:47:34 +0800 Subject: [PATCH 02/16] update content --- ethstorage/downloader/downloader.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ethstorage/downloader/downloader.go b/ethstorage/downloader/downloader.go index 054f62bf..429e9dcf 100644 --- a/ethstorage/downloader/downloader.go +++ b/ethstorage/downloader/downloader.go @@ -416,10 +416,11 @@ func (s *Downloader) downloadRange(start int64, end int64, toCache bool) ([]blob clBlob, exists := clBlobs[elBlob.hash] if !exists { if s.emailConfig != nil { - msg := fmt.Sprintf("Did not find the event specified blob in the CL, blockNumber: %d, kvIndex: %d\n", elBlock.number, elBlob.kvIndex) - msg += "This may indicate that the blob has not been published to the consensus layer yet, or there is an issue with the consensus layer blob availability.\n" + msg := fmt.Sprintf("From EL event: blockNumber=%d, kvIndex=%d, hash=%s\n", elBlock.number, elBlob.kvIndex, elBlob.hash.Hex()) + msg += "Downloader did not find the blob in CL. \n" + msg += "This may indicate that there is an issue with the consensus layer blob availability.\n" email.SendEmail( - "Blob missing in CL for downloader", + "🛑 Fatal error from es-node: blob missing in CL for downloader", msg, *s.emailConfig, s.lg, From c75fe84318fc2c98b0f74f56f4b7afe9755903aa Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 4 Dec 2025 17:53:29 +0800 Subject: [PATCH 03/16] update content --- ethstorage/downloader/downloader.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ethstorage/downloader/downloader.go b/ethstorage/downloader/downloader.go index 429e9dcf..8c8feeda 100644 --- a/ethstorage/downloader/downloader.go +++ b/ethstorage/downloader/downloader.go @@ -416,8 +416,11 @@ func (s *Downloader) downloadRange(start int64, end int64, toCache bool) ([]blob clBlob, exists := clBlobs[elBlob.hash] if !exists { if s.emailConfig != nil { - msg := fmt.Sprintf("From EL event: blockNumber=%d, kvIndex=%d, hash=%s\n", elBlock.number, elBlob.kvIndex, elBlob.hash.Hex()) - msg += "Downloader did not find the blob in CL. \n" + msg := "Downloader did not find the specified blob in CL: \n" + msg += "From the EL event: \n" + msg += fmt.Sprintf(" blockNumber=%d\n", elBlock.number) + msg += fmt.Sprintf(" kvIndex=%d\n", elBlob.kvIndex) + msg += fmt.Sprintf(" hash=%s\n", elBlob.hash.Hex()) msg += "This may indicate that there is an issue with the consensus layer blob availability.\n" email.SendEmail( "🛑 Fatal error from es-node: blob missing in CL for downloader", From aeb3dad0db20f96cc65cac1221441795b7486628 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 4 Dec 2025 18:11:25 +0800 Subject: [PATCH 04/16] update content --- ethstorage/downloader/downloader.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ethstorage/downloader/downloader.go b/ethstorage/downloader/downloader.go index 8c8feeda..65c30692 100644 --- a/ethstorage/downloader/downloader.go +++ b/ethstorage/downloader/downloader.go @@ -416,14 +416,14 @@ func (s *Downloader) downloadRange(start int64, end int64, toCache bool) ([]blob clBlob, exists := clBlobs[elBlob.hash] if !exists { if s.emailConfig != nil { - msg := "Downloader did not find the specified blob in CL: \n" - msg += "From the EL event: \n" - msg += fmt.Sprintf(" blockNumber=%d\n", elBlock.number) - msg += fmt.Sprintf(" kvIndex=%d\n", elBlob.kvIndex) - msg += fmt.Sprintf(" hash=%s\n", elBlob.hash.Hex()) - msg += "This may indicate that there is an issue with the consensus layer blob availability.\n" + msg := "The downloader couldn't locate the specified blob in the consensus layer. The node is stopped pending resolution. \n" + msg += "Details from the EL event: \n" + msg += fmt.Sprintf(" - blockNumber: %d\n", elBlock.number) + msg += fmt.Sprintf(" - kvIndex: %d\n", elBlob.kvIndex) + msg += fmt.Sprintf(" - hash: %s\n", elBlob.hash.Hex()) + msg += "This may indicate a potential issue with blob availability on the consensus layer. \n" email.SendEmail( - "🛑 Fatal error from es-node: blob missing in CL for downloader", + "🛑 Fatal Error from es-node: Downloader Failed to Locate Blob in CL", msg, *s.emailConfig, s.lg, From f5d4f7de3ba6da25ad0ab12ef041748e976c8257 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 4 Dec 2025 18:25:40 +0800 Subject: [PATCH 05/16] update content --- ethstorage/downloader/downloader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethstorage/downloader/downloader.go b/ethstorage/downloader/downloader.go index 65c30692..e3331bce 100644 --- a/ethstorage/downloader/downloader.go +++ b/ethstorage/downloader/downloader.go @@ -416,7 +416,7 @@ func (s *Downloader) downloadRange(start int64, end int64, toCache bool) ([]blob clBlob, exists := clBlobs[elBlob.hash] if !exists { if s.emailConfig != nil { - msg := "The downloader couldn't locate the specified blob in the consensus layer. The node is stopped pending resolution. \n" + msg := "The downloader couldn't locate the specified blob in the consensus layer. The node is stopped pending resolution. " msg += "Details from the EL event: \n" msg += fmt.Sprintf(" - blockNumber: %d\n", elBlock.number) msg += fmt.Sprintf(" - kvIndex: %d\n", elBlob.kvIndex) From d5fdad86a53d779ec7232eb48a9edf4533a3c9db Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 4 Dec 2025 18:37:36 +0800 Subject: [PATCH 06/16] check email config --- ethstorage/downloader/downloader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethstorage/downloader/downloader.go b/ethstorage/downloader/downloader.go index e3331bce..92bad7b8 100644 --- a/ethstorage/downloader/downloader.go +++ b/ethstorage/downloader/downloader.go @@ -415,7 +415,7 @@ func (s *Downloader) downloadRange(start int64, end int64, toCache bool) ([]blob for _, elBlob := range elBlock.blobs { clBlob, exists := clBlobs[elBlob.hash] if !exists { - if s.emailConfig != nil { + if s.emailConfig != nil && s.emailConfig.Check() == nil { msg := "The downloader couldn't locate the specified blob in the consensus layer. The node is stopped pending resolution. " msg += "Details from the EL event: \n" msg += fmt.Sprintf(" - blockNumber: %d\n", elBlock.number) From c4dc8b44ab7177faf91ddb9b291c35e770381919 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 4 Dec 2025 18:55:35 +0800 Subject: [PATCH 07/16] check email config --- ethstorage/email/email.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ethstorage/email/email.go b/ethstorage/email/email.go index a647dd37..dea744c7 100644 --- a/ethstorage/email/email.go +++ b/ethstorage/email/email.go @@ -22,6 +22,7 @@ type EmailConfig struct { } func (c EmailConfig) Check() error { + fmt.Println("Checking email config:", c.String()) if c.Username == "" { return fmt.Errorf("email username is empty") } From 8be4488401b4239510d6ad960da8cf31af5531ee Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 4 Dec 2025 18:59:39 +0800 Subject: [PATCH 08/16] check email config --- cmd/es-node/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/es-node/config.go b/cmd/es-node/config.go index 4eb7f2c9..37c624a5 100644 --- a/cmd/es-node/config.go +++ b/cmd/es-node/config.go @@ -55,6 +55,7 @@ func NewConfig(ctx *cli.Context, lg log.Logger) (*node.Config, error) { } dlConfig := NewDownloaderConfig(ctx) + fmt.Println("Downloader config. EmailConfig:", dlConfig.EmailConfig) minerConfig, err := NewMinerConfig(ctx, client, storageConfig.L1Contract, storageConfig.Miner, lg) if err != nil { return nil, fmt.Errorf("failed to load miner config: %w", err) From a1b0be6ec43fe909e5e4b053fe623177d0159a37 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 4 Dec 2025 19:08:24 +0800 Subject: [PATCH 09/16] check email config --- cmd/es-node/config.go | 3 +-- ethstorage/downloader/config.go | 2 +- ethstorage/downloader/downloader.go | 2 +- ethstorage/email/email.go | 1 - ethstorage/node/node.go | 2 +- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/es-node/config.go b/cmd/es-node/config.go index 37c624a5..1dc40302 100644 --- a/cmd/es-node/config.go +++ b/cmd/es-node/config.go @@ -55,7 +55,6 @@ func NewConfig(ctx *cli.Context, lg log.Logger) (*node.Config, error) { } dlConfig := NewDownloaderConfig(ctx) - fmt.Println("Downloader config. EmailConfig:", dlConfig.EmailConfig) minerConfig, err := NewMinerConfig(ctx, client, storageConfig.L1Contract, storageConfig.Miner, lg) if err != nil { return nil, fmt.Errorf("failed to load miner config: %w", err) @@ -267,6 +266,6 @@ func NewDownloaderConfig(ctx *cli.Context) *downloader.Config { // email is nice to have but not required by downloader return dlCfg } - dlCfg.EmailConfig = *emailConfig + dlCfg.EmailConfig = emailConfig return dlCfg } diff --git a/ethstorage/downloader/config.go b/ethstorage/downloader/config.go index 8d8f1d95..4ce62f44 100644 --- a/ethstorage/downloader/config.go +++ b/ethstorage/downloader/config.go @@ -9,5 +9,5 @@ type Config struct { DownloadStart int64 // which block should we download the blobs from DownloadDump string // where to dump the download blobs DownloadThreadNum int // how many threads that will be used to download the blobs into storage file - EmailConfig email.EmailConfig + EmailConfig *email.EmailConfig } diff --git a/ethstorage/downloader/downloader.go b/ethstorage/downloader/downloader.go index 92bad7b8..e3331bce 100644 --- a/ethstorage/downloader/downloader.go +++ b/ethstorage/downloader/downloader.go @@ -415,7 +415,7 @@ func (s *Downloader) downloadRange(start int64, end int64, toCache bool) ([]blob for _, elBlob := range elBlock.blobs { clBlob, exists := clBlobs[elBlob.hash] if !exists { - if s.emailConfig != nil && s.emailConfig.Check() == nil { + if s.emailConfig != nil { msg := "The downloader couldn't locate the specified blob in the consensus layer. The node is stopped pending resolution. " msg += "Details from the EL event: \n" msg += fmt.Sprintf(" - blockNumber: %d\n", elBlock.number) diff --git a/ethstorage/email/email.go b/ethstorage/email/email.go index dea744c7..a647dd37 100644 --- a/ethstorage/email/email.go +++ b/ethstorage/email/email.go @@ -22,7 +22,6 @@ type EmailConfig struct { } func (c EmailConfig) Check() error { - fmt.Println("Checking email config:", c.String()) if c.Username == "" { return fmt.Errorf("email username is empty") } diff --git a/ethstorage/node/node.go b/ethstorage/node/node.go index c8200a0c..3b77df23 100644 --- a/ethstorage/node/node.go +++ b/ethstorage/node/node.go @@ -143,7 +143,7 @@ func (n *EsNode) initL2(ctx context.Context, cfg *Config) error { cfg.Downloader.DownloadDump, cfg.L1.L1MinDurationForBlobsRequest, cfg.Downloader.DownloadThreadNum, - cfg.Downloader.EmailConfig, + *cfg.Downloader.EmailConfig, n.lg, ) return nil From 38395124c9e5bfa61c5803174eaccdfb7df4b31f Mon Sep 17 00:00:00 2001 From: syntrust Date: Fri, 5 Dec 2025 11:12:25 +0800 Subject: [PATCH 10/16] fix config --- cmd/es-node/config.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/cmd/es-node/config.go b/cmd/es-node/config.go index 1dc40302..1aaa1efc 100644 --- a/cmd/es-node/config.go +++ b/cmd/es-node/config.go @@ -54,11 +54,24 @@ func NewConfig(ctx *cli.Context, lg log.Logger) (*node.Config, error) { return nil, fmt.Errorf("failed to load storage config: %w", err) } + emailConfig, err := email.GetEmailConfig(ctx) + if err != nil { + lg.Warn("Failed to load email config, email notifications will be disabled", "error", err) + } dlConfig := NewDownloaderConfig(ctx) + if emailConfig != nil { + dlConfig.EmailConfig = emailConfig + } minerConfig, err := NewMinerConfig(ctx, client, storageConfig.L1Contract, storageConfig.Miner, lg) if err != nil { return nil, fmt.Errorf("failed to load miner config: %w", err) } + if minerConfig.EmailEnabled { + if emailConfig == nil { + return nil, fmt.Errorf("email config is required by miner but not loaded") + } + minerConfig.EmailConfig = *emailConfig + } chainId := new(big.Int).SetUint64(ctx.GlobalUint64(flags.ChainId.Name)) lg.Info("Read chain ID of EthStorage network", "chainID", chainId) if minerConfig != nil { @@ -134,13 +147,6 @@ func NewMinerConfig(ctx *cli.Context, client *ethclient.Client, l1Contract, mine if err != nil { return nil, err } - if minerConfig.EmailEnabled { - emailConfig, err := email.GetEmailConfig(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get email config: %w", err) - } - minerConfig.EmailConfig = *emailConfig - } cctx := context.Background() cr := newContractReader(cctx, client, l1Contract, lg) From efc5dc9d694f0d45a817c358e8ed853306545754 Mon Sep 17 00:00:00 2001 From: syntrust Date: Fri, 5 Dec 2025 11:15:18 +0800 Subject: [PATCH 11/16] fix config --- cmd/es-node/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/es-node/config.go b/cmd/es-node/config.go index 1aaa1efc..935ae938 100644 --- a/cmd/es-node/config.go +++ b/cmd/es-node/config.go @@ -56,7 +56,7 @@ func NewConfig(ctx *cli.Context, lg log.Logger) (*node.Config, error) { emailConfig, err := email.GetEmailConfig(ctx) if err != nil { - lg.Warn("Failed to load email config, email notifications will be disabled", "error", err) + lg.Warn("Failed to load email config, email notifications will be disabled.", "error", err) } dlConfig := NewDownloaderConfig(ctx) if emailConfig != nil { @@ -66,7 +66,7 @@ func NewConfig(ctx *cli.Context, lg log.Logger) (*node.Config, error) { if err != nil { return nil, fmt.Errorf("failed to load miner config: %w", err) } - if minerConfig.EmailEnabled { + if minerConfig != nil && minerConfig.EmailEnabled { if emailConfig == nil { return nil, fmt.Errorf("email config is required by miner but not loaded") } From b593a435a20527266c9db0bb3492c807a1c281e2 Mon Sep 17 00:00:00 2001 From: syntrust Date: Fri, 5 Dec 2025 11:26:00 +0800 Subject: [PATCH 12/16] fix config --- ethstorage/downloader/downloader.go | 13 +++++-------- ethstorage/node/node.go | 5 +---- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/ethstorage/downloader/downloader.go b/ethstorage/downloader/downloader.go index e3331bce..33eac3fd 100644 --- a/ethstorage/downloader/downloader.go +++ b/ethstorage/downloader/downloader.go @@ -107,14 +107,11 @@ func NewDownloader( db ethdb.Database, sm *ethstorage.StorageManager, cache BlobCache, - downloadStart int64, - downloadDump string, minDurationForBlobsRequest uint64, - downloadThreadNum int, - emailConfig email.EmailConfig, + downloadConfig Config, lg log.Logger, ) *Downloader { - sm.DownloadThreadNum = downloadThreadNum + sm.DownloadThreadNum = downloadConfig.DownloadThreadNum return &Downloader{ Cache: cache, l1Source: l1Source, @@ -122,15 +119,15 @@ func NewDownloader( daClient: daClient, db: db, sm: sm, - dumpDir: downloadDump, + dumpDir: downloadConfig.DownloadDump, minDurationForBlobsRequest: minDurationForBlobsRequest, dlLatestReq: make(chan struct{}, 1), dlFinalizedReq: make(chan struct{}, 1), lg: lg, done: make(chan struct{}), - lastDownloadBlock: downloadStart, + lastDownloadBlock: downloadConfig.DownloadStart, downloadedBlobs: 0, - emailConfig: &emailConfig, + emailConfig: downloadConfig.EmailConfig, } } diff --git a/ethstorage/node/node.go b/ethstorage/node/node.go index 3b77df23..180ba628 100644 --- a/ethstorage/node/node.go +++ b/ethstorage/node/node.go @@ -139,11 +139,8 @@ func (n *EsNode) initL2(ctx context.Context, cfg *Config) error { n.db, n.storageManager, n.blobCache, - cfg.Downloader.DownloadStart, - cfg.Downloader.DownloadDump, cfg.L1.L1MinDurationForBlobsRequest, - cfg.Downloader.DownloadThreadNum, - *cfg.Downloader.EmailConfig, + cfg.Downloader, n.lg, ) return nil From 7b75b7f7c4286bd931c3b9af68156f1c427c911f Mon Sep 17 00:00:00 2001 From: syntrust Date: Fri, 5 Dec 2025 11:33:27 +0800 Subject: [PATCH 13/16] fix config --- cmd/es-node/config.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/cmd/es-node/config.go b/cmd/es-node/config.go index 935ae938..09ecfc55 100644 --- a/cmd/es-node/config.go +++ b/cmd/es-node/config.go @@ -261,17 +261,9 @@ func NewL1EndpointConfig(ctx *cli.Context, lg log.Logger) (*eth.L1EndpointConfig } func NewDownloaderConfig(ctx *cli.Context) *downloader.Config { - dlCfg := &downloader.Config{ + return &downloader.Config{ DownloadStart: ctx.GlobalInt64(flags.DownloadStart.Name), DownloadDump: ctx.GlobalString(flags.DownloadDump.Name), DownloadThreadNum: ctx.GlobalInt(flags.DownloadThreadNum.Name), } - - emailConfig, err := email.GetEmailConfig(ctx) - if err != nil { - // email is nice to have but not required by downloader - return dlCfg - } - dlCfg.EmailConfig = emailConfig - return dlCfg } From e7ce75fa4b5203a97dfcded15f30c606ad9ac693 Mon Sep 17 00:00:00 2001 From: syntrust Date: Fri, 5 Dec 2025 11:48:35 +0800 Subject: [PATCH 14/16] refactor --- ethstorage/downloader/downloader.go | 34 +++++++++++++++++------------ 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/ethstorage/downloader/downloader.go b/ethstorage/downloader/downloader.go index 33eac3fd..322a9b36 100644 --- a/ethstorage/downloader/downloader.go +++ b/ethstorage/downloader/downloader.go @@ -412,20 +412,7 @@ func (s *Downloader) downloadRange(start int64, end int64, toCache bool) ([]blob for _, elBlob := range elBlock.blobs { clBlob, exists := clBlobs[elBlob.hash] if !exists { - if s.emailConfig != nil { - msg := "The downloader couldn't locate the specified blob in the consensus layer. The node is stopped pending resolution. " - msg += "Details from the EL event: \n" - msg += fmt.Sprintf(" - blockNumber: %d\n", elBlock.number) - msg += fmt.Sprintf(" - kvIndex: %d\n", elBlob.kvIndex) - msg += fmt.Sprintf(" - hash: %s\n", elBlob.hash.Hex()) - msg += "This may indicate a potential issue with blob availability on the consensus layer. \n" - email.SendEmail( - "🛑 Fatal Error from es-node: Downloader Failed to Locate Blob in CL", - msg, - *s.emailConfig, - s.lg, - ) - } + s.notifyBlobMissing(elBlock.number, elBlob.kvIndex.Uint64(), elBlob.hash) s.lg.Crit("Did not find the event specified blob in the CL", "blockNumber", elBlock.number, "kvIndex", elBlob.kvIndex) } // encode blobs so that miner can do sampling directly from cache @@ -498,3 +485,22 @@ func (s *Downloader) eventsToBlocks(events []types.Log) ([]*blockBlobs, error) { return blocks, nil } + +func (s *Downloader) notifyBlobMissing(blockNumber uint64, kvIndex uint64, hash common.Hash) { + if s.emailConfig == nil { + return + } + + msg := "The downloader couldn't locate the specified blob in the consensus layer. The node is stopped pending resolution. " + msg += "Details from the EL event: \n" + msg += fmt.Sprintf(" - blockNumber: %d\n", blockNumber) + msg += fmt.Sprintf(" - kvIndex: %d\n", kvIndex) + msg += fmt.Sprintf(" - hash: %s\n", hash.Hex()) + msg += "This may indicate a potential issue with blob availability on the consensus layer. \n" + email.SendEmail( + "🛑 Fatal Error from es-node: Downloader Failed to Locate Blob in CL", + msg, + *s.emailConfig, + s.lg, + ) +} From f8d0371d96d1de23cbfd76392fd1bceb0b6d2ce1 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 9 Dec 2025 17:57:13 +0800 Subject: [PATCH 15/16] check shard exist --- ethstorage/downloader/downloader.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ethstorage/downloader/downloader.go b/ethstorage/downloader/downloader.go index 322a9b36..36249460 100644 --- a/ethstorage/downloader/downloader.go +++ b/ethstorage/downloader/downloader.go @@ -12,6 +12,7 @@ import ( "math/big" "os" "path/filepath" + "slices" "sync" "time" @@ -410,6 +411,11 @@ func (s *Downloader) downloadRange(start int64, end int64, toCache bool) ([]blob } for _, elBlob := range elBlock.blobs { + shard := elBlob.kvIndex.Uint64() >> s.sm.KvEntriesBits() + if !slices.Contains(s.sm.Shards(), shard) { + s.lg.Warn("Shard not initialized locally for the kvIndex, skip this blob", "kvIndex", elBlob.kvIndex.Uint64(), "shard", shard) + continue + } clBlob, exists := clBlobs[elBlob.hash] if !exists { s.notifyBlobMissing(elBlock.number, elBlob.kvIndex.Uint64(), elBlob.hash) From 7aa6373a7ef124f8fd2cdd956bb90fd74932d492 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 18 Dec 2025 17:46:04 +0800 Subject: [PATCH 16/16] retry download blob --- ethstorage/downloader/downloader.go | 67 ++++++++++++++++++++--------- ethstorage/eth/beacon_client.go | 8 ++-- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/ethstorage/downloader/downloader.go b/ethstorage/downloader/downloader.go index 36249460..39e37808 100644 --- a/ethstorage/downloader/downloader.go +++ b/ethstorage/downloader/downloader.go @@ -388,26 +388,10 @@ func (s *Downloader) downloadRange(start int64, end int64, toCache bool) ([]blob ) } - var clBlobs map[common.Hash]eth.Blob - if s.l1Beacon != nil { - clBlobs, err = s.l1Beacon.DownloadBlobs(s.l1Beacon.Timestamp2Slot(elBlock.timestamp)) - if err != nil { - s.lg.Error("L1 beacon download blob error", "err", err) - return nil, err - } - } else if s.daClient != nil { - var hashes []common.Hash - for _, blob := range elBlock.blobs { - hashes = append(hashes, blob.hash) - } - - clBlobs, err = s.daClient.DownloadBlobs(hashes) - if err != nil { - s.lg.Error("DA client download blob error", "err", err) - return nil, err - } - } else { - return nil, fmt.Errorf("no beacon client or DA client is available") + clBlobs, err := s.downloadBlobsWithRetry(elBlock, 3) + if err != nil { + s.lg.Error("Failed to download blobs for the block after 3 attempts", "block", elBlock.number, "err", err) + // Empty CL blob will be handled later in the EL blob loop } for _, elBlob := range elBlock.blobs { @@ -441,6 +425,49 @@ func (s *Downloader) downloadRange(start int64, end int64, toCache bool) ([]blob return blobs, nil } +func (s *Downloader) downloadBlobsWithRetry(elBlock *blockBlobs, maxAttempts int) (map[common.Hash]eth.Blob, error) { + var lastErr error + for attempt := 1; attempt <= maxAttempts; attempt++ { + clBlobs, err := s.downloadBlobs(elBlock) + if err == nil { + return clBlobs, nil + } + lastErr = err + if attempt < maxAttempts { + time.Sleep(3 * time.Second) + } + } + return nil, lastErr +} + +func (s *Downloader) downloadBlobs(elBlock *blockBlobs) (map[common.Hash]eth.Blob, error) { + if s.l1Beacon != nil { + slot := s.l1Beacon.Timestamp2Slot(elBlock.timestamp) + clBlobs, err := s.l1Beacon.DownloadBlobs(slot) + if err != nil { + s.lg.Error("L1 beacon download blob error", "block", elBlock.number, "slot", slot, "err", err) + return nil, err + } + return clBlobs, nil + } + + if s.daClient != nil { + hashes := make([]common.Hash, 0, len(elBlock.blobs)) + for _, b := range elBlock.blobs { + hashes = append(hashes, b.hash) + } + + clBlobs, err := s.daClient.DownloadBlobs(hashes) + if err != nil { + s.lg.Error("DA client download blob error", "err", err) + return nil, err + } + return clBlobs, nil + } + + return nil, fmt.Errorf("no beacon client or DA client is available") +} + func (s *Downloader) dumpBlobsIfNeeded(blobs []blob) { if s.dumpDir != "" { for _, blob := range blobs { diff --git a/ethstorage/eth/beacon_client.go b/ethstorage/eth/beacon_client.go index 880be075..46f74445 100644 --- a/ethstorage/eth/beacon_client.go +++ b/ethstorage/eth/beacon_client.go @@ -77,13 +77,13 @@ func (c *BeaconClient) DownloadBlobs(slot uint64) (map[common.Hash]Blob, error) } resp, err := http.Get(beaconUrl) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to query beacon blobs with url %s: %w", beaconUrl, err) } defer resp.Body.Close() var blobsResp blobs.BeaconBlobs if err := json.NewDecoder(resp.Body).Decode(&blobsResp); err != nil { - return nil, err + return nil, fmt.Errorf("failed to decode beacon blobs response from url %s: %w", beaconUrl, err) } res := map[common.Hash]Blob{} @@ -91,11 +91,11 @@ func (c *BeaconClient) DownloadBlobs(slot uint64) (map[common.Hash]Blob, error) // decode hex string to bytes asciiBytes, err := hex.DecodeString(beaconBlob[2:]) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to decode beacon blob hex string %s: %w", beaconBlob, err) } hash, err := blobs.BlobToVersionedHash(asciiBytes) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to compute versioned hash for blob: %w", err) } res[hash] = Blob{VersionedHash: hash, Data: asciiBytes} }