diff --git a/cmd/service/bidcollect.go b/cmd/service/bidcollect.go index d9117ab..56e3b60 100644 --- a/cmd/service/bidcollect.go +++ b/cmd/service/bidcollect.go @@ -36,6 +36,8 @@ var ( runWebserverOnly bool // provides a SSE stream of new bids WebserverListenAddr string + + withDuplicates bool ) func init() { @@ -70,6 +72,7 @@ func init() { bidCollectCmd.Flags().BoolVar(&buildWebsite, "build-website", false, "build file listing website") bidCollectCmd.Flags().BoolVar(&buildWebsiteUpload, "build-website-upload", false, "upload after building") bidCollectCmd.Flags().StringVar(&buildWebsiteOutDir, "build-website-out", "build", "output directory for website") + bidCollectCmd.Flags().BoolVar(&withDuplicates, "with-duplicates", false, "store all bids, including duplicates from different relays") } var bidCollectCmd = &cobra.Command{ @@ -129,6 +132,7 @@ var bidCollectCmd = &cobra.Command{ OutputTSV: outputTSV, RedisAddr: redisAddr, UseRedis: useRedis, + WithDuplicates: withDuplicates, } bidCollector, err := bidcollect.NewBidCollector(&opts) diff --git a/docs/2024-06_bidcollect.md b/docs/2024-06_bidcollect.md index bd43462..5f72d65 100644 --- a/docs/2024-06_bidcollect.md +++ b/docs/2024-06_bidcollect.md @@ -74,6 +74,9 @@ Different data sources have different limitations: - Bids are deduplicated based on this key: - `fmt.Sprintf("%d-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.BuilderPubkey, bid.Value)` - this means only the first bid for a given key is stored, even if - for instance - other relays also deliver the same bid +- To store the same bid delivered by different relays use the `--with-duplicates` flag. This will change the deduplication key to: + - `fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value)` + - this is helpful to measure builder to relay latency. - Bids can be published to Redis (to be consumed by whatever, i.e. a webserver). The channel is called `bidcollect/bids`. - Enable publishing to Redis with the `--redis` flag - You can start a webserver that publishes the data via a SSE stream with `--webserver` diff --git a/services/bidcollect/bid-processor.go b/services/bidcollect/bid-processor.go index c55517c..56ca67a 100644 --- a/services/bidcollect/bid-processor.go +++ b/services/bidcollect/bid-processor.go @@ -22,12 +22,13 @@ import ( // - One CSV for top bids only type BidProcessorOpts struct { - Log *logrus.Entry - UID string - OutDir string - OutputTSV bool - RedisAddr string - UseRedis bool + Log *logrus.Entry + UID string + OutDir string + OutputTSV bool + RedisAddr string + UseRedis bool + WithDuplicates bool } type OutFiles struct { @@ -114,10 +115,18 @@ func (c *BidProcessor) processBids(bids []*types.CommonBid) { } } - // process regular bids only once per unique key (slot+blockhash+parenthash+builderpubkey+value) - if _, ok := c.bidCache[bid.Slot][bid.UniqueKey()]; !ok { + var uniqueKey string + if c.opts.WithDuplicates { + // process regular bids only once per unique key (slot+blockhash+parenthash+relay+builderpubkey+value) + uniqueKey = bid.UniqueKeyWithRelay() + } else { + // process regular bids only once per unique key (slot+blockhash+parenthash+builderpubkey+value) + uniqueKey = bid.UniqueKey() + } + + if _, ok := c.bidCache[bid.Slot][uniqueKey]; !ok { // yet unknown bid, save it - c.bidCache[bid.Slot][bid.UniqueKey()] = bid + c.bidCache[bid.Slot][uniqueKey] = bid isNewBid = true } diff --git a/services/bidcollect/bidcollector.go b/services/bidcollect/bidcollector.go index 4efac2b..f59e839 100644 --- a/services/bidcollect/bidcollector.go +++ b/services/bidcollect/bidcollector.go @@ -23,6 +23,8 @@ type BidCollectorOpts struct { RedisAddr string UseRedis bool + + WithDuplicates bool } type BidCollector struct { @@ -53,12 +55,13 @@ func NewBidCollector(opts *BidCollectorOpts) (c *BidCollector, err error) { // output c.processor, err = NewBidProcessor(&BidProcessorOpts{ - Log: opts.Log, - UID: opts.UID, - OutDir: opts.OutDir, - OutputTSV: opts.OutputTSV, - RedisAddr: opts.RedisAddr, - UseRedis: opts.UseRedis, + Log: opts.Log, + UID: opts.UID, + OutDir: opts.OutDir, + OutputTSV: opts.OutputTSV, + RedisAddr: opts.RedisAddr, + UseRedis: opts.UseRedis, + WithDuplicates: opts.WithDuplicates, }) return c, err } diff --git a/services/bidcollect/types/types.go b/services/bidcollect/types/types.go index 0efccd4..bba9c5d 100644 --- a/services/bidcollect/types/types.go +++ b/services/bidcollect/types/types.go @@ -60,6 +60,10 @@ func (bid *CommonBid) UniqueKey() string { return fmt.Sprintf("%d-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.BuilderPubkey, bid.Value) } +func (bid *CommonBid) UniqueKeyWithRelay() string { + return fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value) +} + func (bid *CommonBid) ValueAsBigInt() *big.Int { value := new(big.Int) value.SetString(bid.Value, 10)