Skip to content

Commit 2f05cb9

Browse files
julienrbrtjulienrbrt
committed
ai test
1 parent bef0bef commit 2f05cb9

File tree

1 file changed

+305
-0
lines changed

1 file changed

+305
-0
lines changed

‎test/e2e/evm_force_inclusion_e2e_test.go‎

Lines changed: 305 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"net/http"
1111
"os"
1212
"path/filepath"
13+
"syscall"
1314
"testing"
1415
"time"
1516

@@ -560,3 +561,307 @@ func TestEvmSyncerMaliciousSequencerForceInclusionE2E(t *testing.T) {
560561
require.False(t, evm.CheckTxIncluded(seqClient, txForce.Hash()),
561562
"Malicious sequencer should NOT have included the forced inclusion transaction")
562563
}
564+
565+
// setDAStartHeightInGenesis modifies the genesis file to set da_start_height.
566+
// This is needed because the based sequencer requires non-zero DAStartHeight,
567+
// and catch-up detection via CalculateEpochNumber also depends on it.
568+
func setDAStartHeightInGenesis(t *testing.T, homeDir string, height uint64) {
569+
t.Helper()
570+
genesisPath := filepath.Join(homeDir, "config", "genesis.json")
571+
data, err := os.ReadFile(genesisPath)
572+
require.NoError(t, err)
573+
574+
var genesis map[string]interface{}
575+
err = json.Unmarshal(data, &genesis)
576+
require.NoError(t, err)
577+
578+
genesis["da_start_height"] = height
579+
580+
newData, err := json.MarshalIndent(genesis, "", " ")
581+
require.NoError(t, err)
582+
583+
err = os.WriteFile(genesisPath, newData, 0644)
584+
require.NoError(t, err)
585+
}
586+
587+
// TestEvmSequencerCatchUpBasedSequencerE2E tests that when a sequencer restarts after
588+
// extended downtime (multiple DA epochs), it correctly enters catch-up mode, replays
589+
// missed forced inclusion transactions from DA (matching what a based sequencer would
590+
// produce), and then resumes normal operation.
591+
//
592+
// Test Scenario:
593+
// A sequencer goes down. During downtime, forced inclusion txs accumulate on DA.
594+
// A based sequencer keeps the chain alive using those DA txs. When the original
595+
// sequencer restarts, it must catch up by replaying the same DA epochs — producing
596+
// identical forced-inclusion-only blocks — before resuming normal mempool-based operation.
597+
//
598+
// Architecture:
599+
// - 2 Reth instances (via setupCommonEVMTest with needsFullNode=true)
600+
// - Reth1 -> Sequencer (phases 1 and 4)
601+
// - Reth2 -> Based sequencer (phase 3, reuses full node Reth/port slots)
602+
//
603+
// Phases:
604+
// 0. Setup: DA, 2 Reth containers, init sequencer with force inclusion epoch=2 and da_start_height=1
605+
// 1. Normal sequencer operation: submit normal txs
606+
// 2. Sequencer downtime: stop sequencer, submit forced inclusion txs directly to DA
607+
// 3. Start based sequencer: verify it includes forced inclusion txs
608+
// 4. Restart original sequencer: it catches up via DA replay
609+
// 5. Verification: forced txs on both nodes, sequencer resumes normal operation
610+
func TestEvmSequencerCatchUpBasedSequencerE2E(t *testing.T) {
611+
sut := NewSystemUnderTest(t)
612+
workDir := t.TempDir()
613+
sequencerHome := filepath.Join(workDir, "sequencer")
614+
basedSeqHome := filepath.Join(workDir, "based-sequencer")
615+
616+
// ===== PHASE 0: Setup =====
617+
t.Log("Phase 0: Setup")
618+
619+
jwtSecret, fullNodeJwtSecret, genesisHash, endpoints := setupCommonEVMTest(t, sut, true)
620+
621+
// Create passphrase and JWT secret files for sequencer
622+
passphraseFile := createPassphraseFile(t, sequencerHome)
623+
jwtSecretFile := createJWTSecretFile(t, sequencerHome, jwtSecret)
624+
625+
// Initialize sequencer node
626+
output, err := sut.RunCmd(evmSingleBinaryPath,
627+
"init",
628+
"--evnode.node.aggregator=true",
629+
"--evnode.signer.passphrase_file", passphraseFile,
630+
"--home", sequencerHome,
631+
)
632+
require.NoError(t, err, "failed to init sequencer", output)
633+
634+
// Modify genesis: enable force inclusion with epoch=2, set da_start_height=1
635+
enableForceInclusionInGenesis(t, sequencerHome, 2)
636+
setDAStartHeightInGenesis(t, sequencerHome, 1)
637+
638+
// Start sequencer with forced inclusion namespace
639+
seqProcess := sut.ExecCmd(evmSingleBinaryPath,
640+
"start",
641+
"--evm.jwt-secret-file", jwtSecretFile,
642+
"--evm.genesis-hash", genesisHash,
643+
"--evnode.node.block_time", DefaultBlockTime,
644+
"--evnode.node.aggregator=true",
645+
"--evnode.signer.passphrase_file", passphraseFile,
646+
"--home", sequencerHome,
647+
"--evnode.da.block_time", DefaultDABlockTime,
648+
"--evnode.da.address", endpoints.GetDAAddress(),
649+
"--evnode.da.namespace", DefaultDANamespace,
650+
"--evnode.da.forced_inclusion_namespace", "forced-inc",
651+
"--evnode.rpc.address", endpoints.GetRollkitRPCListen(),
652+
"--evnode.p2p.listen_address", endpoints.GetRollkitP2PAddress(),
653+
"--evm.engine-url", endpoints.GetSequencerEngineURL(),
654+
"--evm.eth-url", endpoints.GetSequencerEthURL(),
655+
)
656+
sut.AwaitNodeUp(t, endpoints.GetRollkitRPCAddress(), NodeStartupTimeout)
657+
t.Log("Sequencer is up with force inclusion enabled")
658+
659+
// ===== PHASE 1: Normal Sequencer Operation =====
660+
t.Log("Phase 1: Normal Sequencer Operation")
661+
662+
seqClient, err := ethclient.Dial(endpoints.GetSequencerEthURL())
663+
require.NoError(t, err)
664+
defer seqClient.Close()
665+
666+
ctx := context.Background()
667+
var nonce uint64 = 0
668+
669+
// Submit 2 normal transactions
670+
var normalTxHashes []common.Hash
671+
for i := 0; i < 2; i++ {
672+
tx := evm.GetRandomTransaction(t, TestPrivateKey, TestToAddress, DefaultChainID, DefaultGasLimit, &nonce)
673+
err = seqClient.SendTransaction(ctx, tx)
674+
require.NoError(t, err)
675+
normalTxHashes = append(normalTxHashes, tx.Hash())
676+
t.Logf("Submitted normal tx %d: %s (nonce=%d)", i+1, tx.Hash().Hex(), tx.Nonce())
677+
}
678+
679+
// Wait for normal txs to be included
680+
for i, txHash := range normalTxHashes {
681+
require.Eventually(t, func() bool {
682+
return evm.CheckTxIncluded(seqClient, txHash)
683+
}, 15*time.Second, 500*time.Millisecond, "Normal tx %d not included", i+1)
684+
t.Logf("Normal tx %d included", i+1)
685+
}
686+
687+
// Record sequencer height
688+
seqHeader, err := seqClient.HeaderByNumber(ctx, nil)
689+
require.NoError(t, err)
690+
preDowntimeHeight := seqHeader.Number.Uint64()
691+
t.Logf("Sequencer height before downtime: %d", preDowntimeHeight)
692+
693+
// ===== PHASE 2: Sequencer Downtime + Submit Forced Inclusion Txs to DA =====
694+
t.Log("Phase 2: Sequencer Downtime + Submit Forced Inclusion Txs to DA")
695+
696+
// Stop sequencer process (SIGTERM to just this process, not all evm processes)
697+
err = seqProcess.Signal(syscall.SIGTERM)
698+
require.NoError(t, err, "failed to stop sequencer process")
699+
time.Sleep(1 * time.Second) // Wait for process to exit
700+
701+
// Submit forced inclusion transactions directly to DA
702+
blobClient, err := blobrpc.NewClient(ctx, endpoints.GetDAAddress(), "", "")
703+
require.NoError(t, err, "Failed to create blob RPC client")
704+
defer blobClient.Close()
705+
706+
daClient := block.NewDAClient(
707+
blobClient,
708+
config.Config{
709+
DA: config.DAConfig{
710+
Namespace: DefaultDANamespace,
711+
ForcedInclusionNamespace: "forced-inc",
712+
},
713+
},
714+
zerolog.Nop(),
715+
)
716+
717+
// Create and submit 3 forced inclusion txs to DA
718+
var forcedTxHashes []common.Hash
719+
for i := 0; i < 3; i++ {
720+
txForce := evm.GetRandomTransaction(t, TestPrivateKey, TestToAddress, DefaultChainID, DefaultGasLimit, &nonce)
721+
txBytes, err := txForce.MarshalBinary()
722+
require.NoError(t, err)
723+
724+
result := daClient.Submit(ctx, [][]byte{txBytes}, -1, daClient.GetForcedInclusionNamespace(), nil)
725+
require.Equal(t, da.StatusSuccess, result.Code, "Failed to submit forced tx %d to DA: %s", i+1, result.Message)
726+
727+
forcedTxHashes = append(forcedTxHashes, txForce.Hash())
728+
t.Logf("Submitted forced inclusion tx %d to DA: %s (nonce=%d)", i+1, txForce.Hash().Hex(), txForce.Nonce())
729+
}
730+
731+
// Wait for DA to advance past multiple epochs (epoch=2, DA block time=1s)
732+
// Need missedEpochs > 1, so DA must be at least 2 full epochs ahead
733+
t.Log("Waiting for DA to advance past multiple epochs...")
734+
time.Sleep(6 * time.Second)
735+
736+
// ===== PHASE 3: Start Based Sequencer =====
737+
t.Log("Phase 3: Start Based Sequencer")
738+
739+
// Initialize based sequencer node
740+
output, err = sut.RunCmd(evmSingleBinaryPath,
741+
"init",
742+
"--home", basedSeqHome,
743+
)
744+
require.NoError(t, err, "failed to init based sequencer", output)
745+
746+
// Copy genesis from sequencer to based sequencer (shares chain config)
747+
MustCopyFile(t,
748+
filepath.Join(sequencerHome, "config", "genesis.json"),
749+
filepath.Join(basedSeqHome, "config", "genesis.json"),
750+
)
751+
752+
// Create JWT secret file for based sequencer using fullNodeJwtSecret
753+
basedSeqJwtSecretFile := createJWTSecretFile(t, basedSeqHome, fullNodeJwtSecret)
754+
755+
// Start based sequencer (uses full node Reth/port slots)
756+
sut.ExecCmd(evmSingleBinaryPath,
757+
"start",
758+
"--evnode.node.aggregator=true",
759+
"--evnode.node.based_sequencer=true",
760+
"--evm.jwt-secret-file", basedSeqJwtSecretFile,
761+
"--evm.genesis-hash", genesisHash,
762+
"--evnode.node.block_time", DefaultBlockTime,
763+
"--home", basedSeqHome,
764+
"--evnode.da.block_time", DefaultDABlockTime,
765+
"--evnode.da.address", endpoints.GetDAAddress(),
766+
"--evnode.da.namespace", DefaultDANamespace,
767+
"--evnode.da.forced_inclusion_namespace", "forced-inc",
768+
"--evnode.rpc.address", endpoints.GetFullNodeRPCListen(),
769+
"--evnode.p2p.listen_address", endpoints.GetFullNodeP2PAddress(),
770+
"--evm.engine-url", endpoints.GetFullNodeEngineURL(),
771+
"--evm.eth-url", endpoints.GetFullNodeEthURL(),
772+
)
773+
774+
// Based sequencer may take longer to be ready (processing DA from start height)
775+
sut.AwaitNodeLive(t, endpoints.GetFullNodeRPCAddress(), NodeStartupTimeout)
776+
t.Log("Based sequencer is live")
777+
778+
// Connect ethclient to based sequencer
779+
basedSeqClient, err := ethclient.Dial(endpoints.GetFullNodeEthURL())
780+
require.NoError(t, err)
781+
defer basedSeqClient.Close()
782+
783+
// Verify based sequencer includes forced inclusion txs
784+
t.Log("Waiting for based sequencer to include forced inclusion txs...")
785+
for i, txHash := range forcedTxHashes {
786+
require.Eventually(t, func() bool {
787+
return evm.CheckTxIncluded(basedSeqClient, txHash)
788+
}, 60*time.Second, 1*time.Second,
789+
"Forced inclusion tx %d (%s) not included in based sequencer", i+1, txHash.Hex())
790+
t.Logf("Based sequencer included forced tx %d: %s", i+1, txHash.Hex())
791+
}
792+
t.Log("All forced inclusion txs verified on based sequencer")
793+
794+
// ===== PHASE 4: Restart Original Sequencer (Catch-Up) =====
795+
t.Log("Phase 4: Restart Original Sequencer (Catch-Up)")
796+
797+
// Restart the sequencer using existing home directory (no init needed)
798+
sut.ExecCmd(evmSingleBinaryPath,
799+
"start",
800+
"--evm.jwt-secret-file", jwtSecretFile,
801+
"--evm.genesis-hash", genesisHash,
802+
"--evnode.node.block_time", DefaultBlockTime,
803+
"--evnode.node.aggregator=true",
804+
"--evnode.signer.passphrase_file", passphraseFile,
805+
"--home", sequencerHome,
806+
"--evnode.da.block_time", DefaultDABlockTime,
807+
"--evnode.da.address", endpoints.GetDAAddress(),
808+
"--evnode.da.namespace", DefaultDANamespace,
809+
"--evnode.da.forced_inclusion_namespace", "forced-inc",
810+
"--evnode.rpc.address", endpoints.GetRollkitRPCListen(),
811+
"--evnode.p2p.listen_address", endpoints.GetRollkitP2PAddress(),
812+
"--evm.engine-url", endpoints.GetSequencerEngineURL(),
813+
"--evm.eth-url", endpoints.GetSequencerEthURL(),
814+
)
815+
sut.AwaitNodeUp(t, endpoints.GetRollkitRPCAddress(), NodeStartupTimeout)
816+
t.Log("Sequencer restarted successfully")
817+
818+
// Reconnect ethclient to sequencer
819+
seqClient.Close()
820+
seqClient, err = ethclient.Dial(endpoints.GetSequencerEthURL())
821+
require.NoError(t, err)
822+
823+
// ===== PHASE 5: Verification =====
824+
t.Log("Phase 5: Verification")
825+
826+
// 5a. Verify sequencer includes forced inclusion txs after catch-up
827+
t.Log("Verifying sequencer includes forced inclusion txs after catch-up...")
828+
for i, txHash := range forcedTxHashes {
829+
require.Eventually(t, func() bool {
830+
return evm.CheckTxIncluded(seqClient, txHash)
831+
}, 30*time.Second, 1*time.Second,
832+
"Forced inclusion tx %d (%s) should be included after catch-up", i+1, txHash.Hex())
833+
t.Logf("Sequencer caught up with forced tx %d: %s", i+1, txHash.Hex())
834+
}
835+
t.Log("All forced inclusion txs verified on sequencer after catch-up")
836+
837+
// 5b. Verify sequencer resumes normal operation
838+
t.Log("Verifying sequencer resumes normal mempool-based operation...")
839+
txNormal := evm.GetRandomTransaction(t, TestPrivateKey, TestToAddress, DefaultChainID, DefaultGasLimit, &nonce)
840+
err = seqClient.SendTransaction(ctx, txNormal)
841+
require.NoError(t, err)
842+
t.Logf("Submitted post-catchup normal tx: %s (nonce=%d)", txNormal.Hash().Hex(), txNormal.Nonce())
843+
844+
require.Eventually(t, func() bool {
845+
return evm.CheckTxIncluded(seqClient, txNormal.Hash())
846+
}, 15*time.Second, 500*time.Millisecond,
847+
"Normal tx after catch-up should be included")
848+
t.Log("Post-catchup normal tx included - sequencer resumed normal operation")
849+
850+
// 5c. Verify both nodes have the forced inclusion txs
851+
t.Log("Verifying both nodes have all forced inclusion txs...")
852+
for i, txHash := range forcedTxHashes {
853+
seqIncluded := evm.CheckTxIncluded(seqClient, txHash)
854+
basedIncluded := evm.CheckTxIncluded(basedSeqClient, txHash)
855+
require.True(t, seqIncluded, "Forced tx %d should be on sequencer", i+1)
856+
require.True(t, basedIncluded, "Forced tx %d should be on based sequencer", i+1)
857+
t.Logf("Forced tx %d verified on both nodes: %s", i+1, txHash.Hex())
858+
}
859+
860+
t.Log("Test PASSED: Sequencer catch-up with based sequencer verified successfully")
861+
t.Logf(" - Sequencer processed %d normal txs before downtime", len(normalTxHashes))
862+
t.Logf(" - %d forced inclusion txs submitted to DA during downtime", len(forcedTxHashes))
863+
t.Logf(" - Based sequencer included all forced txs from DA")
864+
t.Logf(" - Sequencer caught up and replayed all forced txs after restart")
865+
t.Logf(" - Sequencer resumed normal mempool-based operation")
866+
t.Logf(" - Both nodes have identical forced inclusion tx set")
867+
}

0 commit comments

Comments
 (0)