feat(benchmarking): adding gas burner test (#3115)

* refactor: move spamoor benchmark into testify suite in test/e2e/benchmark

- Create test/e2e/benchmark/ subpackage with SpamoorSuite (testify/suite)
- Move spamoor smoke test into suite as TestSpamoorSmoke
- Split helpers into focused files: traces.go, output.go, metrics.go
- Introduce resultWriter for defer-based benchmark JSON output
- Export shared symbols from evm_test_common.go for cross-package use
- Restructure CI to fan-out benchmark jobs and fan-in publishing
- Run benchmarks on PRs only when benchmark-related files change

* fix: correct BENCH_JSON_OUTPUT path for spamoor benchmark

go test sets the working directory to the package under test, so the
env var should be relative to test/e2e/benchmark/, not test/e2e/.

* fix: place package pattern before test binary flags in benchmark CI

go test treats all arguments after an unknown flag (--evm-binary) as
test binary args, so ./benchmark/ was never recognized as a package
pattern.

* fix: adjust evm-binary path for benchmark subpackage working directory

go test sets the cwd to the package directory (test/e2e/benchmark/),
so the binary path needs an extra parent traversal.

* wip: erc20 benchmark test

* fix: exclude benchmark subpackage from make test-e2e

The benchmark package doesn't define the --binary flag that test-e2e
passes. It has its own CI workflow so it doesn't need to run here.

* fix: replace FilterLogs with header iteration and optimize spamoor config

collectBlockMetrics hit reth's 20K FilterLogs limit at high tx volumes.
Replace with direct header iteration over [startBlock, endBlock] and add
Phase 1 metrics: non-empty ratio, block interval p50/p99, gas/block and
tx/block p50/p99.

Optimize spamoor configuration for 100ms block time:
- --slot-duration 100ms, --startup-delay 0 on daemon
- throughput=50 per 100ms slot (500 tx/s per spammer)
- max_pending=50000 to avoid 3s block poll backpressure
- 5 staggered spammers with 50K txs each

Results: 55 MGas/s, 1414 TPS, 19.8% non-empty blocks (up from 6%).

* fix: improve benchmark measurement window and reliability

- Move startBlock capture after spammer creation to exclude warm-up
- Replace 20s drain sleep with smart poll (waitForDrain)
- Add deleteAllSpammers cleanup to handle stale spamoor DB entries
- Lower trace sample rate to 10% to prevent Jaeger OOM

* fix: address PR review feedback for benchmark suite

- make reth tag configurable via EV_RETH_TAG env var (default pr-140)
- fix OTLP config: remove duplicate env vars, use http/protobuf protocol
- use require.Eventually for host readiness polling
- rename requireHTTP to requireHostUp
- use non-fatal logging in resultWriter.flush deferred context
- fix stale doc comment (setupCommonEVMEnv -> SetupCommonEVMEnv)
- rename loop variable to avoid shadowing testing.TB convention
- add block/internal/executing/** to CI path trigger
- remove unused require import from output.go

* chore: specify http

* chore: filter out benchmark tests from test-e2e

* refactor: centralize reth config and lower ERC20 spammer count

move EV_RETH_TAG resolution and rpc connection limits into setupEnv
so all benchmark tests share the same reth configuration. lower ERC20
spammer count from 5 to 2 to reduce resource contention on local
hardware while keeping the loop for easy scaling on dedicated infra.

* chore: collect all traces at once

* chore: self review

* refactor: extract benchmark helpers to slim down ERC20 test body

- add blockMetricsSummary with summarize(), log(), and entries() methods
- add evNodeOverhead() for computing ProduceBlock vs ExecuteTxs overhead
- add collectTraces() suite method to deduplicate trace collection pattern
- add addEntries() convenience method on resultWriter
- slim TestERC20Throughput from ~217 to ~119 lines
- reuse collectTraces in TestSpamoorSmoke

* docs: add detailed documentation to benchmark helper methods

* ci: add ERC20 throughput benchmark job

* chore: remove span assertions

* chore: adding gas burner test

Author: chatton

test/e2e/benchmark/gasburner_test.go

@@ -0,0 +1,121 @@
+//go:build evm
+
+package benchmark
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/celestiaorg/tastora/framework/docker/evstack/spamoor"
+)
+
+// TestGasBurner measures gas throughput using a deterministic gasburner
+// workload. The result is tracked via BENCH_JSON_OUTPUT as seconds_per_gigagas
+// (lower is better) on the benchmark dashboard.
+func (s *SpamoorSuite) TestGasBurner() {
+	const (
+		numSpammers     = 4
+		countPerSpammer = 2500
+		totalCount      = numSpammers * countPerSpammer
+		warmupTxs       = 50
+		serviceName     = "ev-node-gasburner"
+		waitTimeout     = 5 * time.Minute
+	)
+
+	t := s.T()
+	ctx := t.Context()
+	w := newResultWriter(t, "GasBurner")
+	defer w.flush()
+
+	e := s.setupEnv(config{
+		serviceName: serviceName,
+	})
+	api := e.spamoorAPI
+
+	s.Require().NoError(deleteAllSpammers(api), "failed to delete stale spammers")
+
+	gasburnerCfg := map[string]any{
+		"gas_units_to_burn": 5_000_000,
+		"total_count":       countPerSpammer,
+		"throughput":        25,
+		"max_pending":       5000,
+		"max_wallets":       500,
+		"rebroadcast":       0,
+		"base_fee":          20,
+		"tip_fee":           5,
+		"refill_amount":     "5000000000000000000",
+		"refill_balance":    "2000000000000000000",
+		"refill_interval":   300,
+	}
+
+	for i := range numSpammers {
+		name := fmt.Sprintf("bench-gasburner-%d", i)
+		id, err := api.CreateSpammer(name, spamoor.ScenarioGasBurnerTX, gasburnerCfg, true)
+		s.Require().NoError(err, "failed to create spammer %s", name)
+		t.Cleanup(func() { _ = api.DeleteSpammer(id) })
+	}
+
+	// wait for wallet prep and contract deployment to finish before
+	// recording start block so warmup is excluded from the measurement.
+	pollSentTotal := func() (float64, error) {
+		metrics, mErr := api.GetMetrics()
+		if mErr != nil {
+			return 0, mErr
+		}
+		return sumCounter(metrics["spamoor_transactions_sent_total"]), nil
+	}
+	waitForMetricTarget(t, "spamoor_transactions_sent_total (warmup)", pollSentTotal, warmupTxs, waitTimeout)
+
+	startHeader, err := e.ethClient.HeaderByNumber(ctx, nil)
+	s.Require().NoError(err, "failed to get start block header")
+	startBlock := startHeader.Number.Uint64()
+	loadStart := time.Now()
+	t.Logf("start block: %d (after warmup)", startBlock)
+
+	// wait for all transactions to be sent
+	waitForMetricTarget(t, "spamoor_transactions_sent_total", pollSentTotal, float64(totalCount), waitTimeout)
+
+	// wait for pending txs to drain
+	drainCtx, drainCancel := context.WithTimeout(ctx, 30*time.Second)
+	defer drainCancel()
+	waitForDrain(drainCtx, t.Logf, e.ethClient, 10)
+	wallClock := time.Since(loadStart)
+
+	endHeader, err := e.ethClient.HeaderByNumber(ctx, nil)
+	s.Require().NoError(err, "failed to get end block header")
+	endBlock := endHeader.Number.Uint64()
+	t.Logf("end block: %d (range %d blocks)", endBlock, endBlock-startBlock)
+
+	// collect block-level gas/tx metrics
+	bm, err := collectBlockMetrics(ctx, e.ethClient, startBlock, endBlock)
+	s.Require().NoError(err, "failed to collect block metrics")
+
+	summary := bm.summarize()
+	s.Require().Greater(summary.SteadyState, time.Duration(0), "expected non-zero steady-state duration")
+	summary.log(t, startBlock, endBlock, bm.TotalBlockCount, bm.BlockCount, wallClock)
+
+	// derive seconds_per_gigagas from the summary's MGas/s
+	var secsPerGigagas float64
+	if summary.AchievedMGas > 0 {
+		// MGas/s -> Ggas/s = MGas/s / 1000, then invert
+		secsPerGigagas = 1000.0 / summary.AchievedMGas
+	}
+	t.Logf("seconds_per_gigagas: %.4f", secsPerGigagas)
+
+	// collect and report traces
+	traces := s.collectTraces(e, serviceName)
+
+	if overhead, ok := evNodeOverhead(traces.evNode); ok {
+		t.Logf("ev-node overhead: %.1f%%", overhead)
+		w.addEntry(entry{Name: "GasBurner - ev-node overhead", Unit: "%", Value: overhead})
+	}
+
+	w.addEntries(summary.entries("GasBurner"))
+	w.addSpans(traces.allSpans())
+	w.addEntry(entry{
+		Name:  fmt.Sprintf("%s - seconds_per_gigagas", w.label),
+		Unit:  "s/Ggas",
+		Value: secsPerGigagas,
+	})
+}

test/e2e/benchmark/helpers.go

@@ -267,6 +267,8 @@ type blockMetricsSummary struct {
 	AvgTx float64
 	// BlocksPerSec is non-empty blocks / steady-state seconds.
 	BlocksPerSec float64
+	// AvgBlockInterval is the mean time between all consecutive blocks.
+	AvgBlockInterval time.Duration
 	// NonEmptyRatio is (non-empty blocks / total blocks) * 100.
 	NonEmptyRatio float64
 }
@@ -287,6 +289,15 @@ func (m *blockMetrics) summarize() *blockMetricsSummary {
 		achievedTPS = float64(m.TotalTxCount) / ss.Seconds()
 	}
 
+	var avgBlockInterval time.Duration
+	if len(m.BlockIntervals) > 0 {
+		var total time.Duration
+		for _, d := range m.BlockIntervals {
+			total += d
+		}
+		avgBlockInterval = total / time.Duration(len(m.BlockIntervals))
+	}
+
 	return &blockMetricsSummary{
 		SteadyState:   ss,
 		AchievedMGas:  mgasPerSec(m.TotalGasUsed, ss),
@@ -300,8 +311,9 @@ func (m *blockMetrics) summarize() *blockMetricsSummary {
 		TxP99:         txP99,
 		AvgGas:        m.avgGasPerBlock(),
 		AvgTx:         m.avgTxPerBlock(),
-		BlocksPerSec:  blocksPerSec,
-		NonEmptyRatio: m.nonEmptyRatio(),
+		BlocksPerSec:     blocksPerSec,
+		AvgBlockInterval: avgBlockInterval,
+		NonEmptyRatio:    m.nonEmptyRatio(),
 	}
 }
 
@@ -312,8 +324,8 @@ func (m *blockMetrics) summarize() *blockMetricsSummary {
 func (s *blockMetricsSummary) log(t testing.TB, startBlock, endBlock uint64, totalBlocks, nonEmptyBlocks int, wallClock time.Duration) {
 	t.Logf("block range: %d-%d (%d total, %d non-empty, %.1f%% non-empty)",
 		startBlock, endBlock, totalBlocks, nonEmptyBlocks, s.NonEmptyRatio)
-	t.Logf("block intervals: p50=%s, p99=%s, max=%s",
-		s.IntervalP50.Round(time.Millisecond), s.IntervalP99.Round(time.Millisecond), s.IntervalMax.Round(time.Millisecond))
+	t.Logf("block intervals: avg=%s, p50=%s, p99=%s, max=%s",
+		s.AvgBlockInterval.Round(time.Millisecond), s.IntervalP50.Round(time.Millisecond), s.IntervalP99.Round(time.Millisecond), s.IntervalMax.Round(time.Millisecond))
 	t.Logf("gas/block (non-empty): avg=%.0f, p50=%.0f, p99=%.0f", s.AvgGas, s.GasP50, s.GasP99)
 	t.Logf("tx/block (non-empty): avg=%.1f, p50=%.0f, p99=%.0f", s.AvgTx, s.TxP50, s.TxP99)
 	t.Logf("throughput: %.2f MGas/s, %.1f TPS over %s steady-state (%s wall clock)",
@@ -332,6 +344,7 @@ func (s *blockMetricsSummary) entries(prefix string) []entry {
 		{Name: prefix + " - avg tx/block", Unit: "count", Value: s.AvgTx},
 		{Name: prefix + " - blocks/s", Unit: "blocks/s", Value: s.BlocksPerSec},
 		{Name: prefix + " - non-empty block ratio", Unit: "%", Value: s.NonEmptyRatio},
+		{Name: prefix + " - avg block interval", Unit: "ms", Value: float64(s.AvgBlockInterval.Milliseconds())},
 		{Name: prefix + " - block interval p50", Unit: "ms", Value: float64(s.IntervalP50.Milliseconds())},
 		{Name: prefix + " - block interval p99", Unit: "ms", Value: float64(s.IntervalP99.Milliseconds())},
 		{Name: prefix + " - gas/block p50", Unit: "gas", Value: s.GasP50},
@@ -368,6 +381,22 @@ func evNodeOverhead(spans []e2e.TraceSpan) (float64, bool) {
 	return (produceAvg - executeAvg) / produceAvg * 100, true
 }
 
+// waitForMetricTarget polls a metric getter function every 2s until the
+// returned value >= target, or fails the test on timeout.
+func waitForMetricTarget(t testing.TB, name string, poll func() (float64, error), target float64, timeout time.Duration) {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		v, err := poll()
+		if err == nil && v >= target {
+			t.Logf("metric %s reached %.0f (target %.0f)", name, v, target)
+			return
+		}
+		time.Sleep(2 * time.Second)
+	}
+	t.Fatalf("metric %s did not reach target %.0f within %v", name, target, timeout)
+}
+
 // collectBlockMetrics iterates all headers in [startBlock, endBlock] to collect
 // gas and transaction metrics. Empty blocks are skipped for gas/tx aggregation
 // but included in block interval tracking.