test to reproduce out of order nonce

app/abci.go

@@ -2,10 +2,13 @@ package app
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
+	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
 	"github.com/sei-protocol/sei-chain/utils/metrics"
+	evmtypes "github.com/sei-protocol/sei-chain/x/evm/types"
 	abci "github.com/tendermint/tendermint/abci/types"
 	"go.opentelemetry.io/otel/attribute"
 )
@@ -38,6 +41,26 @@ func (app *App) CheckTx(ctx context.Context, req *abci.RequestCheckTx) (*abci.Re
 	return app.BaseApp.CheckTx(ctx, req)
 }
 
+func (app *App) CheckTxWrapped(ctx context.Context, req *abci.RequestCheckTx) (*abci.ResponseCheckTxV2, any, error) {
+	_, span := app.GetBaseApp().TracingInfo.Start("CheckTxWrapped")
+	defer span.End()
+	tx, err := app.txDecoder(req.Tx)
+	if err != nil {
+		res := sdkerrors.ResponseCheckTx(err, 0, 0, false)
+		return &abci.ResponseCheckTxV2{ResponseCheckTx: &res}, nil, err
+	}
+	res, err := app.CheckTxDecoded(tx, req)
+	if err != nil {
+		return res, nil, err
+	}
+	if tx != nil && len(tx.GetMsgs()) > 0 {
+		if evmMsg, ok := tx.GetMsgs()[0].(*evmtypes.MsgEVMTransaction); ok {
+			return res, evmMsg, nil
+		}
+	}
+	return res, nil, nil
+}
+
 func (app *App) DeliverTx(ctx sdk.Context, req abci.RequestDeliverTx, tx sdk.Tx, checksum [32]byte) abci.ResponseDeliverTx {
 	defer metrics.MeasureDeliverTxDuration(time.Now())
 	// ensure we carry the initial context from tracer here
@@ -80,3 +103,12 @@ func (app *App) LoadLatest(ctx context.Context, req *abci.RequestLoadLatest) (*a
 	app.mounter()
 	return app.BaseApp.LoadLatest(ctx, req)
 }
+
+func (app *App) CheckNonce(ctx context.Context, req any, index int) (bool, error) {
+	sdkCtx := app.GetCheckCtx()
+	msg, ok := req.(*evmtypes.MsgEVMTransaction)
+	if !ok {
+		return false, fmt.Errorf("invalid request type: %T", req)
+	}
+	return app.EvmKeeper.CheckNonce(sdkCtx, msg, index)
+}

sei-cosmos/baseapp/abci.go

@@ -208,7 +208,15 @@ func (app *BaseApp) EndBlock(ctx sdk.Context, req abci.RequestEndBlock) (res abc
 // the ResponseCheckTx will contain relevant gas execution context.
 func (app *BaseApp) CheckTx(ctx context.Context, req *abci.RequestCheckTx) (*abci.ResponseCheckTxV2, error) {
 	defer telemetry.MeasureSince(time.Now(), "abci", "check_tx")
+	tx, err := app.txDecoder(req.Tx)
+	if err != nil {
+		res := sdkerrors.ResponseCheckTx(err, 0, 0, app.trace)
+		return &abci.ResponseCheckTxV2{ResponseCheckTx: &res}, err
+	}
+	return app.CheckTxDecoded(tx, req)
+}
 
+func (app *BaseApp) CheckTxDecoded(tx sdk.Tx, req *abci.RequestCheckTx) (*abci.ResponseCheckTxV2, error) {
 	var mode runTxMode
 
 	switch {
@@ -223,11 +231,6 @@ func (app *BaseApp) CheckTx(ctx context.Context, req *abci.RequestCheckTx) (*abc
 	}
 
 	sdkCtx := app.getContextForTx(mode, req.Tx)
-	tx, err := app.txDecoder(req.Tx)
-	if err != nil {
-		res := sdkerrors.ResponseCheckTx(err, 0, 0, app.trace)
-		return &abci.ResponseCheckTxV2{ResponseCheckTx: &res}, err
-	}
 	gInfo, result, _, priority, pendingTxChecker, expireTxHandler, txCtx, err := app.runTx(sdkCtx, mode, tx, sha256.Sum256(req.Tx))
 	if err != nil {
 		res := sdkerrors.ResponseCheckTx(err, gInfo.GasWanted, gInfo.GasUsed, app.trace)
@@ -1258,3 +1261,15 @@ func (app *BaseApp) GetTxPriorityHint(_ context.Context, req *abci.RequestGetTxP
 		Priority: priority,
 	}, nil
 }
+
+func (app *BaseApp) CheckTxWrapped(ctx context.Context, req *abci.RequestCheckTx) (*abci.ResponseCheckTxV2, any, error) {
+	res, err := app.CheckTx(ctx, req)
+	if err != nil {
+		return res, nil, err
+	}
+	return res, nil, nil
+}
+
+func (app *BaseApp) CheckNonce(ctx context.Context, req any, index int) (bool, error) {
+	return true, nil
+}

sei-tendermint/abci/client/grpc_client.go

@@ -190,3 +190,12 @@ func (cli *grpcClient) LoadLatest(ctx context.Context, params *types.RequestLoad
 func (cli *grpcClient) GetTxPriorityHint(ctx context.Context, req *types.RequestGetTxPriorityHint) (*types.ResponseGetTxPriorityHint, error) {
 	return cli.client.GetTxPriorityHint(ctx, types.ToRequestGetTxPriorityHint(req).GetGetTxPriorityHint(), grpc.WaitForReady(true))
 }
+
+func (cli *grpcClient) CheckNonce(ctx context.Context, req any, index int) (bool, error) {
+	return false, errors.New("not implemented")
+}
+
+func (cli *grpcClient) CheckTxWrapped(ctx context.Context, req *types.RequestCheckTx) (*types.ResponseCheckTxV2, any, error) {
+	res, err := cli.CheckTx(ctx, req)
+	return res, nil, err
+}

sei-tendermint/abci/client/socket_client.go

@@ -377,6 +377,15 @@ func (cli *socketClient) GetTxPriorityHint(ctx context.Context, req *types.Reque
 	return res.GetGetTxPriorityHint(), nil
 }
 
+func (cli *socketClient) CheckNonce(ctx context.Context, req any, index int) (bool, error) {
+	return false, errors.New("not implemented")
+}
+
+func (cli *socketClient) CheckTxWrapped(ctx context.Context, req *types.RequestCheckTx) (*types.ResponseCheckTxV2, any, error) {
+	res, err := cli.CheckTx(ctx, req)
+	return res, nil, err
+}
+
 //----------------------------------------
 
 func resMatchesReq(req *types.Request, res *types.Response) (ok bool) {

sei-tendermint/abci/example/kvstore/kvstore.go

@@ -209,6 +209,10 @@ func (*Application) CheckTx(_ context.Context, req *types.RequestCheckTx) (*type
 	return &types.ResponseCheckTxV2{ResponseCheckTx: &types.ResponseCheckTx{Code: code.CodeTypeOK, GasWanted: 1}}, nil
 }
 
+func (*Application) CheckTxWrapped(_ context.Context, req *types.RequestCheckTx) (*types.ResponseCheckTxV2, any, error) {
+	return &types.ResponseCheckTxV2{ResponseCheckTx: &types.ResponseCheckTx{Code: code.CodeTypeOK, GasWanted: 1}}, nil, nil
+}
+
 func (app *Application) Commit(_ context.Context) (*types.ResponseCommit, error) {
 	app.mu.Lock()
 	defer app.mu.Unlock()

sei-tendermint/abci/types/application.go

@@ -12,7 +12,9 @@ type Application interface {
 	Query(context.Context, *RequestQuery) (*ResponseQuery, error) // Query for state
 
 	// Mempool Connection
-	CheckTx(context.Context, *RequestCheckTx) (*ResponseCheckTxV2, error) // Validate a tx for the mempool
+	CheckTx(context.Context, *RequestCheckTx) (*ResponseCheckTxV2, error)             // Validate a tx for the mempool; deprecated
+	CheckTxWrapped(context.Context, *RequestCheckTx) (*ResponseCheckTxV2, any, error) // Validate a tx for the mempool
+	CheckNonce(context.Context, any, int) (bool, error)
 
 	// Consensus Connection
 	InitChain(context.Context, *RequestInitChain) (*ResponseInitChain, error) // Initialize blockchain w validators/other info from TendermintCore
@@ -56,6 +58,14 @@ func (BaseApplication) CheckTx(_ context.Context, req *RequestCheckTx) (*Respons
 	return &ResponseCheckTxV2{ResponseCheckTx: &ResponseCheckTx{Code: CodeTypeOK}}, nil
 }
 
+func (BaseApplication) CheckTxWrapped(_ context.Context, req *RequestCheckTx) (*ResponseCheckTxV2, any, error) {
+	return &ResponseCheckTxV2{ResponseCheckTx: &ResponseCheckTx{Code: CodeTypeOK}}, nil, nil
+}
+
+func (BaseApplication) CheckNonce(_ context.Context, req any, index int) (bool, error) {
+	return true, nil
+}
+
 func (BaseApplication) Commit(_ context.Context) (*ResponseCommit, error) {
 	return &ResponseCommit{}, nil
 }

sei-tendermint/config/config.go

@@ -862,6 +862,10 @@ type MempoolConfig struct {
 	//
 	// See DropUtilisationThreshold and DropPriorityThreshold.
 	DropPriorityReservoirSize int `mapstructure:"drop-priority-reservoir-size"`
+
+	// Do a JIT check of nonce before proposing a block to make sure no transaction
+	// with invalid nonce is included in the block.
+	CheckNonceBeforePropose bool `mapstructure:"check-nonce-before-propose"`
 }
 
 // DefaultMempoolConfig returns a default configuration for the Tendermint mempool.
@@ -888,6 +892,7 @@ func DefaultMempoolConfig() *MempoolConfig {
 		DropPriorityThreshold:        0.1,
 		DropUtilisationThreshold:     1.0,
 		DropPriorityReservoirSize:    10_240,
+		CheckNonceBeforePropose:      false,
 	}
 }
 

sei-tendermint/config/toml.go

@@ -457,6 +457,8 @@ drop-utilisation-threshold = {{ .Mempool.DropUtilisationThreshold }}
 # See DropUtilisationThreshold and DropPriorityThreshold.
 drop-priority-reservoir-size = {{ .Mempool.DropPriorityReservoirSize }}
 
+check-nonce-before-propose = {{ .Mempool.CheckNonceBeforePropose }}
+
 #######################################################
 ###         State Sync Configuration Options        ###
 #######################################################

sei-tendermint/internal/consensus/mempool_test.go

@@ -236,7 +236,7 @@ func TestMempoolRmBadTx(t *testing.T) {
 
 		// check for the tx
 		for {
-			txs := assertMempool(t, cs.txNotifier).ReapMaxBytesMaxGas(int64(len(txBytes)), -1, -1)
+			txs := assertMempool(t, cs.txNotifier).ReapMaxBytesMaxGas(ctx, int64(len(txBytes)), -1, -1)
 			if len(txs) == 0 {
 				emptyMempoolCh <- struct{}{}
 				return
@@ -328,6 +328,15 @@ func (app *CounterApplication) CheckTx(_ context.Context, req *abci.RequestCheck
 	return &abci.ResponseCheckTxV2{ResponseCheckTx: &abci.ResponseCheckTx{Code: code.CodeTypeOK}}, nil
 }
 
+func (app *CounterApplication) CheckTxWrapped(ctx context.Context, req *abci.RequestCheckTx) (*abci.ResponseCheckTxV2, any, error) {
+	res, err := app.CheckTx(ctx, req)
+	return res, nil, err
+}
+
+func (app *CounterApplication) CheckNonce(ctx context.Context, req any, index int) (bool, error) {
+	return true, nil
+}
+
 func txAsUint64(tx []byte) uint64 {
 	tx8 := make([]byte, 8)
 	copy(tx8[len(tx8)-len(tx):], tx)

sei-tendermint/internal/consensus/replay_stubs.go

@@ -37,9 +37,11 @@ func (emptyMempool) Size() int                 { return 0 }
 func (emptyMempool) CheckTx(context.Context, types.Tx, func(*abci.ResponseCheckTx), mempool.TxInfo) error {
 	return nil
 }
-func (emptyMempool) RemoveTxByKey(txKey types.TxKey) error      { return nil }
-func (emptyMempool) ReapMaxBytesMaxGas(_, _, _ int64) types.Txs { return types.Txs{} }
-func (emptyMempool) ReapMaxTxs(n int) types.Txs                 { return types.Txs{} }
+func (emptyMempool) RemoveTxByKey(txKey types.TxKey) error { return nil }
+func (emptyMempool) ReapMaxBytesMaxGas(_ context.Context, _, _, _ int64) types.Txs {
+	return types.Txs{}
+}
+func (emptyMempool) ReapMaxTxs(n int) types.Txs { return types.Txs{} }
 func (emptyMempool) Update(
 	_ context.Context,
 	_ int64,

sei-tendermint/internal/mempool/mempool.go

@@ -363,7 +363,7 @@ func (txmp *TxMempool) CheckTx(
 		txmp.duplicateTxsCache.Increment(txHash)
 	}
 
-	res, err := txmp.proxyAppConn.CheckTx(ctx, &abci.RequestCheckTx{Tx: tx})
+	res, evmMsg, err := txmp.proxyAppConn.CheckTxWrapped(ctx, &abci.RequestCheckTx{Tx: tx})
 	txmp.totalCheckTxCount.Add(1)
 	if err != nil {
 		txmp.metrics.NumberOfFailedCheckTxs.Add(1)
@@ -402,6 +402,7 @@ func (txmp *TxMempool) CheckTx(
 		isEVM:         res.IsEVM,
 		removeHandler: removeHandler,
 		estimatedGas:  res.GasEstimated,
+		evmMessage:    evmMsg,
 	}
 
 	if err == nil {
@@ -533,7 +534,7 @@ func (txmp *TxMempool) Flush() {
 // NOTE:
 //   - Transactions returned are not removed from the mempool transaction
 //     store or indexes.
-func (txmp *TxMempool) ReapMaxBytesMaxGas(maxBytes, maxGasWanted, maxGasEstimated int64) types.Txs {
+func (txmp *TxMempool) ReapMaxBytesMaxGas(ctx context.Context, maxBytes, maxGasWanted, maxGasEstimated int64) types.Txs {
 	txmp.mtx.Lock()
 	defer txmp.mtx.Unlock()
 
@@ -549,6 +550,7 @@ func (txmp *TxMempool) ReapMaxBytesMaxGas(maxBytes, maxGasWanted, maxGasEstimate
 		// do not reap anything if threshold is not met
 		return txs
 	}
+	checkNonceCount := 0
 	txmp.priorityIndex.ForEachTx(func(wtx *WrappedTx) bool {
 		size := types.ComputeProtoSizeForTxs([]types.Tx{wtx.tx})
 
@@ -557,6 +559,18 @@ func (txmp *TxMempool) ReapMaxBytesMaxGas(maxBytes, maxGasWanted, maxGasEstimate
 			return false
 		}
 
+		if wtx.evmMessage != nil && txmp.config.CheckNonceBeforePropose {
+			nonceValid, err := txmp.proxyAppConn.CheckNonce(ctx, wtx.evmMessage, checkNonceCount)
+			checkNonceCount++
+			if err != nil {
+				txmp.logger.Error("error checking nonce", "error", err)
+				return false
+			}
+			if !nonceValid {
+				return false
+			}
+		}
+
 		// if the tx doesn't have a gas estimate, fallback to gas wanted
 		var txGasEstimate int64
 		if wtx.estimatedGas >= MinGasEVMTx && wtx.estimatedGas <= wtx.gasWanted {