diff --git a/Cargo.lock b/Cargo.lock index 3b82182..4a739a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2192,6 +2192,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "thirdweb-core", "thiserror 2.0.12", "tokio", "tracing", diff --git a/aa-core/src/signer.rs b/aa-core/src/signer.rs index 968ccd4..cc15b30 100644 --- a/aa-core/src/signer.rs +++ b/aa-core/src/signer.rs @@ -161,7 +161,7 @@ impl SmartAccountSigner { }, message, format, - self.credentials.clone(), + &self.credentials, ) .await } @@ -189,7 +189,7 @@ impl SmartAccountSigner { from: self.options.signer_address, }, typed_data, - self.credentials.clone(), + &self.credentials, ) .await; } @@ -212,7 +212,7 @@ impl SmartAccountSigner { from: self.options.signer_address, }, typed_data, - self.credentials.clone(), + &self.credentials, ) .await } @@ -242,7 +242,7 @@ impl SmartAccountSigner { from: self.options.signer_address, }, &typed_data, - self.credentials.clone(), + &self.credentials, ) .await } diff --git a/core/src/execution_options/mod.rs b/core/src/execution_options/mod.rs index a738b5e..48fd2d9 100644 --- a/core/src/execution_options/mod.rs +++ b/core/src/execution_options/mod.rs @@ -89,7 +89,8 @@ pub struct WebhookOptions { pub struct SendTransactionRequest { pub execution_options: ExecutionOptions, pub params: Vec, - pub webhook_options: Option>, + #[serde(default)] + pub webhook_options: Vec, } /// # QueuedTransaction diff --git a/core/src/signer.rs b/core/src/signer.rs index 5004a92..3b75db3 100644 --- a/core/src/signer.rs +++ b/core/src/signer.rs @@ -169,7 +169,7 @@ pub trait AccountSigner { options: Self::SigningOptions, message: &str, format: MessageFormat, - credentials: SigningCredential, + credentials: &SigningCredential, ) -> impl std::future::Future> + Send; /// Sign typed data @@ -177,15 +177,15 @@ pub trait AccountSigner { &self, options: Self::SigningOptions, typed_data: &TypedData, - credentials: SigningCredential, + credentials: &SigningCredential, ) -> impl std::future::Future> + Send; /// Sign a transaction fn sign_transaction( &self, options: Self::SigningOptions, - transaction: TypedTransaction, - credentials: SigningCredential, + transaction: &TypedTransaction, + credentials: &SigningCredential, ) -> impl std::future::Future> + Send; /// Sign EIP-7702 authorization @@ -195,7 +195,7 @@ pub trait AccountSigner { chain_id: u64, address: Address, nonce: alloy::primitives::U256, - credentials: SigningCredential, + credentials: &SigningCredential, ) -> impl std::future::Future> + Send; } @@ -224,7 +224,7 @@ impl AccountSigner for EoaSigner { options: EoaSigningOptions, message: &str, format: MessageFormat, - credentials: SigningCredential, + credentials: &SigningCredential, ) -> Result { match credentials { SigningCredential::Vault(auth_method) => { @@ -260,7 +260,7 @@ impl AccountSigner for EoaSigner { .sign_message( auth_token, thirdweb_auth, - message.to_string(), + message, options.from, options.chain_id, Some(iaw_format), @@ -280,7 +280,7 @@ impl AccountSigner for EoaSigner { &self, options: EoaSigningOptions, typed_data: &TypedData, - credentials: SigningCredential, + credentials: &SigningCredential, ) -> Result { match &credentials { SigningCredential::Vault(auth_method) => { @@ -301,12 +301,7 @@ impl AccountSigner for EoaSigner { } => { let iaw_result = self .iaw_client - .sign_typed_data( - auth_token.clone(), - thirdweb_auth.clone(), - typed_data.clone(), - options.from, - ) + .sign_typed_data(auth_token, thirdweb_auth, typed_data, options.from) .await .map_err(|e| { tracing::error!("Error signing typed data with EOA (IAW): {:?}", e); @@ -321,14 +316,14 @@ impl AccountSigner for EoaSigner { async fn sign_transaction( &self, options: EoaSigningOptions, - transaction: TypedTransaction, - credentials: SigningCredential, + transaction: &TypedTransaction, + credentials: &SigningCredential, ) -> Result { match credentials { SigningCredential::Vault(auth_method) => { let vault_result = self .vault_client - .sign_transaction(auth_method.clone(), transaction, options.from) + .sign_transaction(auth_method.clone(), transaction.clone(), options.from) .await .map_err(|e| { tracing::error!("Error signing transaction with EOA (Vault): {:?}", e); @@ -343,7 +338,7 @@ impl AccountSigner for EoaSigner { } => { let iaw_result = self .iaw_client - .sign_transaction(auth_token.clone(), thirdweb_auth.clone(), transaction) + .sign_transaction(auth_token, thirdweb_auth, &transaction) .await .map_err(|e| { tracing::error!("Error signing transaction with EOA (IAW): {:?}", e); @@ -361,7 +356,7 @@ impl AccountSigner for EoaSigner { chain_id: u64, address: Address, nonce: U256, - credentials: SigningCredential, + credentials: &SigningCredential, ) -> Result { // Create the Authorization struct that both clients expect let authorization = Authorization { @@ -373,7 +368,7 @@ impl AccountSigner for EoaSigner { SigningCredential::Vault(auth_method) => { let vault_result = self .vault_client - .sign_authorization(auth_method, options.from, authorization) + .sign_authorization(auth_method.clone(), options.from, authorization) .await .map_err(|e| { tracing::error!("Error signing authorization with EOA (Vault): {:?}", e); @@ -389,7 +384,7 @@ impl AccountSigner for EoaSigner { } => { let iaw_result = self .iaw_client - .sign_authorization(auth_token, thirdweb_auth, options.from, authorization) + .sign_authorization(auth_token, thirdweb_auth, options.from, &authorization) .await .map_err(|e| { tracing::error!("Error signing authorization with EOA (IAW): {:?}", e); diff --git a/core/src/transaction.rs b/core/src/transaction.rs index 422f76c..21b7bc3 100644 --- a/core/src/transaction.rs +++ b/core/src/transaction.rs @@ -23,7 +23,7 @@ pub struct InnerTransaction { /// Gas limit for the transaction /// If not provided, engine will estimate the gas limit #[schema(value_type = Option)] - #[serde(default, rename = "gasLimit")] + #[serde(default, rename = "gasLimit", skip_serializing_if = "Option::is_none")] pub gas_limit: Option, /// Transaction type-specific data for different EIP standards @@ -33,7 +33,7 @@ pub struct InnerTransaction { /// Depending on the execution mode chosen, these might be ignored: /// /// - For ERC4337 execution, all gas fee related fields are ignored. Sending signed authorizations is also not supported. - #[serde(flatten)] + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] pub transaction_type_data: Option, } @@ -58,14 +58,17 @@ pub struct Transaction7702Data { /// List of signed authorizations for contract delegation /// Each authorization allows the EOA to temporarily delegate to a smart contract #[schema(value_type = Option>)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub authorization_list: Option>, /// Maximum fee per gas willing to pay (in wei) /// This is the total fee cap including base fee and priority fee + #[serde(default, skip_serializing_if = "Option::is_none")] pub max_fee_per_gas: Option, /// Maximum priority fee per gas willing to pay (in wei) /// This is the tip paid to validators for transaction inclusion + #[serde(default, skip_serializing_if = "Option::is_none")] pub max_priority_fee_per_gas: Option, } @@ -77,10 +80,12 @@ pub struct Transaction7702Data { pub struct Transaction1559Data { /// Maximum fee per gas willing to pay (in wei) /// This is the total fee cap including base fee and priority fee + #[serde(default, skip_serializing_if = "Option::is_none")] pub max_fee_per_gas: Option, /// Maximum priority fee per gas willing to pay (in wei) /// This is the tip paid to validators for transaction inclusion + #[serde(default, skip_serializing_if = "Option::is_none")] pub max_priority_fee_per_gas: Option, } @@ -92,5 +97,6 @@ pub struct Transaction1559Data { pub struct TransactionLegacyData { /// Gas price willing to pay (in wei) /// This is the total price per unit of gas for legacy transactions + #[serde(default, skip_serializing_if = "Option::is_none")] pub gas_price: Option, } diff --git a/executors/Cargo.toml b/executors/Cargo.toml index 2148d9e..283ef56 100644 --- a/executors/Cargo.toml +++ b/executors/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] hex = "0.4.3" alloy = { version = "1.0.8", features = ["serde"] } +thirdweb-core = { version = "0.1.0", path = "../thirdweb-core" } hmac = "0.12.1" reqwest = "0.12.15" serde = "1.0.219" diff --git a/executors/src/eip7702_executor/confirm.rs b/executors/src/eip7702_executor/confirm.rs index f1e995c..27f2efc 100644 --- a/executors/src/eip7702_executor/confirm.rs +++ b/executors/src/eip7702_executor/confirm.rs @@ -32,11 +32,12 @@ pub struct Eip7702ConfirmationJobData { pub bundler_transaction_id: String, pub eoa_address: Address, pub rpc_credentials: RpcCredentials, - pub webhook_options: Option>, + #[serde(default)] + pub webhook_options: Vec, } impl HasWebhookOptions for Eip7702ConfirmationJobData { - fn webhook_options(&self) -> Option> { + fn webhook_options(&self) -> Vec { self.webhook_options.clone() } } diff --git a/executors/src/eip7702_executor/send.rs b/executors/src/eip7702_executor/send.rs index bebf6d2..a9d4f0d 100644 --- a/executors/src/eip7702_executor/send.rs +++ b/executors/src/eip7702_executor/send.rs @@ -46,13 +46,14 @@ pub struct Eip7702SendJobData { pub transactions: Vec, pub eoa_address: Address, pub signing_credential: SigningCredential, - pub webhook_options: Option>, + #[serde(default)] + pub webhook_options: Vec, pub rpc_credentials: RpcCredentials, pub nonce: Option, } impl HasWebhookOptions for Eip7702SendJobData { - fn webhook_options(&self) -> Option> { + fn webhook_options(&self) -> Vec { self.webhook_options.clone() } } @@ -243,7 +244,7 @@ where .sign_typed_data( signing_options.clone(), &typed_data, - job_data.signing_credential.clone(), + &job_data.signing_credential, ) .await .map_err(|e| Eip7702SendError::SigningError { @@ -272,7 +273,7 @@ where job_data.chain_id, MINIMAL_ACCOUNT_IMPLEMENTATION_ADDRESS, nonce, - job_data.signing_credential.clone(), + &job_data.signing_credential, ) .await .map_err(|e| Eip7702SendError::SigningError { diff --git a/executors/src/eoa/events.rs b/executors/src/eoa/events.rs new file mode 100644 index 0000000..a8f152b --- /dev/null +++ b/executors/src/eoa/events.rs @@ -0,0 +1,158 @@ +use std::fmt::Display; + +use serde::{Deserialize, Serialize}; +use twmq::job::RequeuePosition; + +use crate::{ + eoa::{ + store::{ConfirmedTransaction, SubmittedTransactionDehydrated}, + worker::error::EoaExecutorWorkerError, + }, + webhook::envelope::{ + BareWebhookNotificationEnvelope, SerializableFailData, SerializableNackData, + SerializableSuccessData, StageEvent, + }, +}; + +pub struct EoaExecutorEvent { + pub transaction_id: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE", tag = "errorCode")] +pub enum EoaConfirmationError { + #[error( + "Previously submitted attempt for transaction replaced at nonce with different transaction" + )] + #[serde(rename_all = "camelCase")] + TransactionReplaced { nonce: u64, hash: String }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EoaSendAttemptNackData { + pub nonce: u64, + pub error: EoaExecutorWorkerError, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EoaExecutorConfirmedTransaction { + pub receipt: alloy::rpc::types::TransactionReceipt, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum EoaExecutorStage { + Send, + Confirmation, +} + +impl Display for EoaExecutorStage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EoaExecutorStage::Send => write!(f, "send"), + EoaExecutorStage::Confirmation => write!(f, "confirmation"), + } + } +} + +const EXECUTOR_NAME: &str = "eoa"; + +impl EoaExecutorEvent { + pub fn send_attempt_success_envelope( + &self, + submitted_transaction: SubmittedTransactionDehydrated, + ) -> BareWebhookNotificationEnvelope> + { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::Send.to_string(), + event_type: StageEvent::Success, + payload: SerializableSuccessData { + result: submitted_transaction.clone(), + }, + } + } + + pub fn send_attempt_nack_envelope( + &self, + nonce: u64, + error: EoaExecutorWorkerError, + attempt_number: u32, + ) -> BareWebhookNotificationEnvelope> { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::Send.to_string(), + event_type: StageEvent::Nack, + payload: SerializableNackData { + error: EoaSendAttemptNackData { + nonce, + error: error.clone(), + }, + delay_ms: None, + position: RequeuePosition::Last, + attempt_number, + max_attempts: None, + next_retry_at: None, + }, + } + } + + pub fn transaction_replaced_envelope( + &self, + replaced_transaction: SubmittedTransactionDehydrated, + ) -> BareWebhookNotificationEnvelope> { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::Confirmation.to_string(), + event_type: StageEvent::Nack, + payload: SerializableNackData { + error: EoaConfirmationError::TransactionReplaced { + nonce: replaced_transaction.nonce, + hash: replaced_transaction.hash, + }, + delay_ms: None, + position: RequeuePosition::Last, + attempt_number: 0, + max_attempts: None, + next_retry_at: None, + }, + } + } + + pub fn transaction_confirmed_envelope( + &self, + confirmed_transaction: ConfirmedTransaction, + ) -> BareWebhookNotificationEnvelope> + { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::Confirmation.to_string(), + event_type: StageEvent::Success, + payload: SerializableSuccessData { + result: EoaExecutorConfirmedTransaction { + receipt: confirmed_transaction.receipt, + }, + }, + } + } + + pub fn transaction_failed_envelope( + &self, + error: EoaExecutorWorkerError, + final_attempt_number: u32, + ) -> BareWebhookNotificationEnvelope> { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::Send.to_string(), + event_type: StageEvent::Failure, + payload: SerializableFailData { + error: error.clone(), + final_attempt_number, + }, + } + } +} diff --git a/executors/src/eoa/mod.rs b/executors/src/eoa/mod.rs index c7186f7..d0ab4e6 100644 --- a/executors/src/eoa/mod.rs +++ b/executors/src/eoa/mod.rs @@ -1,6 +1,8 @@ pub mod error_classifier; +pub mod events; pub mod store; pub mod worker; + pub use error_classifier::{EoaErrorMapper, EoaExecutionError, RecoveryStrategy}; pub use store::{EoaExecutorStore, EoaTransactionRequest}; -pub use worker::{EoaExecutorWorker, EoaExecutorWorkerJobData}; +pub use worker::{EoaExecutorJobHandler, EoaExecutorWorkerJobData}; diff --git a/executors/src/eoa/store.rs b/executors/src/eoa/store.rs deleted file mode 100644 index d377300..0000000 --- a/executors/src/eoa/store.rs +++ /dev/null @@ -1,2156 +0,0 @@ -use alloy::consensus::{Signed, Transaction, TypedTransaction}; -use alloy::network::AnyTransactionReceipt; -use alloy::primitives::{Address, B256, Bytes, U256}; -use chrono; -use engine_core::chain::RpcCredentials; -use engine_core::credentials::SigningCredential; -use engine_core::execution_options::WebhookOptions; -use engine_core::transaction::TransactionTypeData; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::future::Future; -use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; - -pub trait SafeRedisTransaction: Send + Sync { - fn name(&self) -> &str; - fn operation(&self, pipeline: &mut Pipeline); - fn validation( - &self, - conn: &mut ConnectionManager, - ) -> impl Future> + Send; - fn watch_keys(&self) -> Vec; -} - -struct MovePendingToBorrowedWithRecycledNonce { - recycled_key: String, - pending_key: String, - transaction_id: String, - borrowed_key: String, - nonce: u64, - prepared_tx_json: String, -} - -impl SafeRedisTransaction for MovePendingToBorrowedWithRecycledNonce { - fn name(&self) -> &str { - "pending->borrowed with recycled nonce" - } - - fn operation(&self, pipeline: &mut Pipeline) { - // Remove nonce from recycled set (we know it exists) - pipeline.zrem(&self.recycled_key, self.nonce); - // Remove transaction from pending (we know it exists) - pipeline.lrem(&self.pending_key, 0, &self.transaction_id); - // Store borrowed transaction - pipeline.hset( - &self.borrowed_key, - self.nonce.to_string(), - &self.prepared_tx_json, - ); - } - - fn watch_keys(&self) -> Vec { - vec![self.recycled_key.clone(), self.pending_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Check if nonce exists in recycled set - let nonce_score: Option = conn.zscore(&self.recycled_key, self.nonce).await?; - if nonce_score.is_none() { - return Err(TransactionStoreError::NonceNotInRecycledSet { nonce: self.nonce }); - } - - // Check if transaction exists in pending - let pending_transactions: Vec = conn.lrange(&self.pending_key, 0, -1).await?; - if !pending_transactions.contains(&self.transaction_id) { - return Err(TransactionStoreError::TransactionNotInPendingQueue { - transaction_id: self.transaction_id.clone(), - }); - } - - Ok(()) - } -} - -struct MovePendingToBorrowedWithNewNonce { - optimistic_key: String, - pending_key: String, - nonce: u64, - prepared_tx_json: String, - transaction_id: String, - borrowed_key: String, - eoa: Address, - chain_id: u64, -} - -impl SafeRedisTransaction for MovePendingToBorrowedWithNewNonce { - fn name(&self) -> &str { - "pending->borrowed with new nonce" - } - - fn operation(&self, pipeline: &mut Pipeline) { - // Increment optimistic nonce - pipeline.incr(&self.optimistic_key, 1); - // Remove transaction from pending - pipeline.lrem(&self.pending_key, 0, &self.transaction_id); - // Store borrowed transaction - pipeline.hset( - &self.borrowed_key, - self.nonce.to_string(), - &self.prepared_tx_json, - ); - } - - fn watch_keys(&self) -> Vec { - vec![self.optimistic_key.clone(), self.pending_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Check current optimistic nonce - let current_optimistic: Option = conn.get(&self.optimistic_key).await?; - let current_nonce = match current_optimistic { - Some(nonce) => nonce, - None => { - return Err(TransactionStoreError::NonceSyncRequired { - eoa: self.eoa, - chain_id: self.chain_id, - }); - } - }; - - if current_nonce != self.nonce { - return Err(TransactionStoreError::OptimisticNonceChanged { - expected: self.nonce, - actual: current_nonce, - }); - } - - // Check if transaction exists in pending - let pending_transactions: Vec = conn.lrange(&self.pending_key, 0, -1).await?; - if !pending_transactions.contains(&self.transaction_id) { - return Err(TransactionStoreError::TransactionNotInPendingQueue { - transaction_id: self.transaction_id.clone(), - }); - } - - Ok(()) - } -} - -struct MoveBorrowedToSubmitted { - nonce: u64, - hash: String, - transaction_id: String, - borrowed_key: String, - submitted_key: String, - hash_to_id_key: String, -} - -impl SafeRedisTransaction for MoveBorrowedToSubmitted { - fn name(&self) -> &str { - "borrowed->submitted" - } - - fn operation(&self, pipeline: &mut Pipeline) { - // Remove from borrowed (we know it exists) - pipeline.hdel(&self.borrowed_key, self.nonce.to_string()); - - // Add to submitted with hash:id format - let hash_id_value = format!("{}:{}", self.hash, self.transaction_id); - pipeline.zadd(&self.submitted_key, &hash_id_value, self.nonce); - - // Still maintain hash-to-ID mapping for backward compatibility and external lookups - pipeline.set(&self.hash_to_id_key, &self.transaction_id); - } - - fn watch_keys(&self) -> Vec { - vec![self.borrowed_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Validate that borrowed transaction actually exists - let borrowed_tx: Option = conn - .hget(&self.borrowed_key, self.nonce.to_string()) - .await?; - if borrowed_tx.is_none() { - return Err(TransactionStoreError::TransactionNotInBorrowedState { - transaction_id: self.transaction_id.clone(), - nonce: self.nonce, - }); - } - Ok(()) - } -} - -struct MoveBorrowedToRecycled { - nonce: u64, - transaction_id: String, - borrowed_key: String, - recycled_key: String, - pending_key: String, -} - -impl SafeRedisTransaction for MoveBorrowedToRecycled { - fn name(&self) -> &str { - "borrowed->recycled" - } - - fn operation(&self, pipeline: &mut Pipeline) { - // Remove from borrowed (we know it exists) - pipeline.hdel(&self.borrowed_key, self.nonce.to_string()); - - // Add nonce to recycled set (with timestamp as score) - pipeline.zadd(&self.recycled_key, self.nonce, self.nonce); - - // Add transaction back to pending - pipeline.lpush(&self.pending_key, &self.transaction_id); - } - - fn watch_keys(&self) -> Vec { - vec![self.borrowed_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Validate that borrowed transaction actually exists - let borrowed_tx: Option = conn - .hget(&self.borrowed_key, self.nonce.to_string()) - .await?; - if borrowed_tx.is_none() { - return Err(TransactionStoreError::TransactionNotInBorrowedState { - transaction_id: self.transaction_id.clone(), - nonce: self.nonce, - }); - } - Ok(()) - } -} - -/// The actual user request data -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct EoaTransactionRequest { - pub transaction_id: String, - pub chain_id: u64, - - pub from: Address, - pub to: Option
, - pub value: U256, - pub data: Bytes, - - #[serde(alias = "gas")] - pub gas_limit: Option, - - pub webhook_options: Option>, - - pub signing_credential: SigningCredential, - pub rpc_credentials: RpcCredentials, - - #[serde(flatten)] - pub transaction_type_data: Option, -} - -/// Active attempt for a transaction (full alloy transaction + metadata) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TransactionAttempt { - pub transaction_id: String, - pub details: Signed, - pub sent_at: u64, // Unix timestamp in milliseconds - pub attempt_number: u32, -} - -/// Transaction data for a transaction_id -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TransactionData { - pub transaction_id: String, - pub user_request: EoaTransactionRequest, - pub receipt: Option, - pub attempts: Vec, -} - -pub struct BorrowedTransaction { - pub transaction_id: String, - pub data: Signed, - pub borrowed_at: chrono::DateTime, -} - -/// Transaction store focused on transaction_id operations and nonce indexing -pub struct EoaExecutorStore { - pub redis: ConnectionManager, - pub namespace: Option, -} - -impl EoaExecutorStore { - pub fn new(redis: ConnectionManager, namespace: Option) -> Self { - Self { redis, namespace } - } - - /// Name of the key for the transaction data - /// - /// Transaction data is stored as a Redis HSET with the following fields: - /// - "user_request": JSON string containing EoaTransactionRequest - /// - "receipt": JSON string containing AnyTransactionReceipt (optional) - /// - "status": String status ("confirmed", "failed", etc.) - /// - "completed_at": String Unix timestamp (optional) - fn transaction_data_key_name(&self, transaction_id: &str) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:tx_data:{transaction_id}"), - None => format!("eoa_executor:_tx_data:{transaction_id}"), - } - } - - /// Name of the list for transaction attempts - /// - /// Attempts are stored as a separate Redis LIST where each element is a JSON blob - /// of a TransactionAttempt. This allows efficient append operations. - fn transaction_attempts_list_name(&self, transaction_id: &str) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:tx_attempts:{transaction_id}"), - None => format!("eoa_executor:tx_attempts:{transaction_id}"), - } - } - - /// Name of the list for pending transactions - fn pending_transactions_list_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:pending_txs:{chain_id}:{eoa}"), - None => format!("eoa_executor:pending_txs:{chain_id}:{eoa}"), - } - } - - /// Name of the zset for submitted transactions. nonce -> hash:id - /// Same transaction might appear multiple times in the zset with different nonces/gas prices (and thus different hashes) - fn submitted_transactions_zset_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:submitted_txs:{chain_id}:{eoa}"), - None => format!("eoa_executor:submitted_txs:{chain_id}:{eoa}"), - } - } - - /// Name of the key that maps transaction hash to transaction id - fn transaction_hash_to_id_key_name(&self, hash: &str) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:tx_hash_to_id:{hash}"), - None => format!("eoa_executor:tx_hash_to_id:{hash}"), - } - } - - /// Name of the hashmap that maps `transaction_id` -> `BorrowedTransactionData` - /// - /// This is used for crash recovery. Before submitting a transaction, we atomically move from pending to this borrowed hashmap. - /// - /// On worker recovery, if any borrowed transactions are found, we rebroadcast them and move back to pending or submitted - /// - /// If there's no crash, happy path moves borrowed transactions back to pending or submitted - fn borrowed_transactions_hashmap_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:borrowed_txs:{chain_id}:{eoa}"), - None => format!("eoa_executor:borrowed_txs:{chain_id}:{eoa}"), - } - } - - /// Name of the set that contains recycled nonces. - /// - /// If a transaction was submitted but failed (ie, we know with certainty it didn't enter the mempool), - /// - /// we add the nonce to this set. - /// - /// These nonces are used with priority, before any other nonces. - fn recycled_nonces_set_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:recycled_nonces:{chain_id}:{eoa}"), - None => format!("eoa_executor:recycled_nonces:{chain_id}:{eoa}"), - } - } - - /// Optimistic nonce key name. - /// - /// This is used for optimistic nonce tracking. - /// - /// We store the nonce of the last successfuly sent transaction for each EOA. - /// - /// We increment this nonce for each new transaction. - /// - /// !IMPORTANT! When sending a transaction, we use this nonce as the assigned nonce, NOT the incremented nonce. - fn optimistic_transaction_count_key_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:optimistic_nonce:{chain_id}:{eoa}"), - None => format!("eoa_executor:optimistic_nonce:{chain_id}:{eoa}"), - } - } - - /// Name of the key that contains the nonce of the last fetched ONCHAIN transaction count for each EOA. - /// - /// This is a cache for the actual transaction count, which is fetched from the RPC. - /// - /// The nonce for the NEXT transaction is the ONCHAIN transaction count (NOT + 1) - /// - /// Eg: transaction count is 0, so we use nonce 0 for sending the next transaction. Once successful, transaction count will be 1. - fn last_transaction_count_key_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:last_tx_nonce:{chain_id}:{eoa}"), - None => format!("eoa_executor:last_tx_nonce:{chain_id}:{eoa}"), - } - } - - /// EOA health key name. - /// - /// EOA health stores: - /// - cached balance, the timestamp of the last balance fetch - /// - timestamp of the last successful transaction confirmation - /// - timestamp of the last 5 nonce resets - fn eoa_health_key_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:health:{chain_id}:{eoa}"), - None => format!("eoa_executor:health:{chain_id}:{eoa}"), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct EoaHealth { - pub balance: U256, - /// Update the balance threshold when we see out of funds errors - pub balance_threshold: U256, - pub balance_fetched_at: u64, - pub last_confirmation_at: u64, - pub last_nonce_movement_at: u64, // Track when nonce last moved for gas bump detection - pub nonce_resets: Vec, // Last 5 reset timestamps -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BorrowedTransactionData { - pub transaction_id: String, - pub signed_transaction: Signed, - pub hash: String, - pub borrowed_at: u64, -} - -/// Type of nonce allocation for transaction processing -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum NonceType { - /// Nonce was recycled from a previously failed transaction - Recycled(u64), - /// Nonce was incremented from the current optimistic counter - Incremented(u64), -} - -impl NonceType { - /// Get the nonce value regardless of type - pub fn nonce(&self) -> u64 { - match self { - NonceType::Recycled(nonce) => *nonce, - NonceType::Incremented(nonce) => *nonce, - } - } - - /// Check if this is a recycled nonce - pub fn is_recycled(&self) -> bool { - matches!(self, NonceType::Recycled(_)) - } - - /// Check if this is an incremented nonce - pub fn is_incremented(&self) -> bool { - matches!(self, NonceType::Incremented(_)) - } -} - -impl EoaExecutorStore { - // ========== BOILERPLATE REDUCTION PATTERN ========== - // - // This implementation uses a helper method `execute_with_watch_and_retry` to reduce - // boilerplate in atomic Redis operations. The pattern separates: - // 1. Validation phase: async closure that checks preconditions - // 2. Pipeline phase: sync closure that builds Redis commands - // - // Benefits: - // - Eliminates ~80 lines of boilerplate per method - // - Centralizes retry logic, lock checking, and error handling - // - Makes individual methods focus on business logic - // - Reduces chance of bugs in WATCH/MULTI/EXEC handling - // - // See examples in: - // - atomic_move_pending_to_borrowed_with_recycled_nonce_v2() - // - atomic_move_pending_to_borrowed_with_new_nonce() - // - move_borrowed_to_submitted() - // - move_borrowed_to_recycled() - - /// Aggressively acquire EOA lock, forcefully taking over from stalled workers - pub async fn acquire_eoa_lock_aggressively( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - ) -> Result<(), TransactionStoreError> { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // First try normal acquisition - let acquired: bool = conn.set_nx(&lock_key, worker_id).await?; - if acquired { - return Ok(()); - } - // Lock exists, forcefully take it over - tracing::warn!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, - "Forcefully taking over EOA lock from stalled worker" - ); - // Force set - no expiry, only released by explicit takeover - let _: () = conn.set(&lock_key, worker_id).await?; - Ok(()) - } - - /// Release EOA lock following the spec's finally pattern - pub async fn release_eoa_lock( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - ) -> Result<(), TransactionStoreError> { - // Use existing utility method that handles all the atomic lock checking - match self - .with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - pipeline.del(&lock_key); - }) - .await - { - Ok(()) => { - tracing::debug!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, - "Successfully released EOA lock" - ); - Ok(()) - } - Err(TransactionStoreError::LockLost { .. }) => { - // Lock was already taken over, which is fine for release - tracing::debug!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, - "Lock already released or taken over by another worker" - ); - Ok(()) - } - Err(e) => { - // Other errors shouldn't fail the worker, just log - tracing::warn!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, - error = %e, - "Failed to release EOA lock" - ); - Ok(()) - } - } - } - - /// Helper to execute atomic operations with proper retry logic and watch handling - /// - /// This helper centralizes all the boilerplate for WATCH/MULTI/EXEC operations: - /// - Retry logic with exponential backoff - /// - Lock ownership validation - /// - WATCH key management - /// - Error handling and UNWATCH cleanup - /// - /// ## Usage: - /// Implement the `SafeRedisTransaction` trait for your operation, then call this method. - /// The trait separates validation (async) from pipeline operations (sync) for clean patterns. - /// - /// ## Example: - /// ```rust - /// let safe_tx = MovePendingToBorrowedWithNewNonce { - /// nonce: expected_nonce, - /// prepared_tx_json, - /// transaction_id, - /// borrowed_key, - /// optimistic_key, - /// pending_key, - /// eoa, - /// chain_id, - /// }; - /// - /// self.execute_with_watch_and_retry(eoa, chain_id, worker_id, &safe_tx).await?; - /// ``` - /// - /// ## When to use this helper: - /// - Operations that implement `SafeRedisTransaction` trait - /// - Need atomic WATCH/MULTI/EXEC with retry logic - /// - Want centralized lock checking and error handling - /// - /// ## When NOT to use this helper: - /// - Simple operations that can use `with_lock_check` instead - /// - Operations that don't need WATCH on multiple keys - /// - Read-only operations that don't modify state - async fn execute_with_watch_and_retry( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - safe_tx: &impl SafeRedisTransaction, - ) -> Result<(), TransactionStoreError> { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let mut retry_count = 0; - - loop { - if retry_count >= MAX_RETRIES { - return Err(TransactionStoreError::InternalError { - message: format!( - "Exceeded max retries ({}) for {} on {}:{}", - MAX_RETRIES, - safe_tx.name(), - eoa, - chain_id - ), - }); - } - - // Exponential backoff after first retry - if retry_count > 0 { - let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); - tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; - tracing::debug!( - retry_count = retry_count, - delay_ms = delay_ms, - eoa = %eoa, - chain_id = chain_id, - operation = safe_tx.name(), - "Retrying atomic operation" - ); - } - - // WATCH all specified keys including lock - let mut watch_cmd = twmq::redis::cmd("WATCH"); - watch_cmd.arg(&lock_key); - for key in safe_tx.watch_keys() { - watch_cmd.arg(key); - } - let _: () = watch_cmd.query_async(&mut conn).await?; - - // Check lock ownership - let current_owner: Option = conn.get(&lock_key).await?; - if current_owner.as_deref() != Some(worker_id) { - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - - // Execute validation - match safe_tx.validation(&mut conn).await { - Ok(()) => { - // Build and execute pipeline - let mut pipeline = twmq::redis::pipe(); - pipeline.atomic(); - safe_tx.operation(&mut pipeline); - - match pipeline - .query_async::>(&mut conn) - .await - { - Ok(_) => return Ok(()), // Success - Err(_) => { - // WATCH failed, check if it was our lock - let still_own_lock: Option = conn.get(&lock_key).await?; - if still_own_lock.as_deref() != Some(worker_id) { - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - // State changed, retry - retry_count += 1; - continue; - } - } - } - Err(e) => { - // Validation failed, unwatch and return error - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(e); - } - } - } - } - - /// Example of how to refactor a complex method using the helper to reduce boilerplate - /// This shows the pattern for atomic_move_pending_to_borrowed_with_recycled_nonce - pub async fn atomic_move_pending_to_borrowed_with_recycled_nonce( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - nonce: u64, - prepared_tx: &BorrowedTransactionData, - ) -> Result<(), TransactionStoreError> { - let safe_tx = MovePendingToBorrowedWithRecycledNonce { - recycled_key: self.recycled_nonces_set_name(eoa, chain_id), - pending_key: self.pending_transactions_list_name(eoa, chain_id), - transaction_id: transaction_id.to_string(), - borrowed_key: self.borrowed_transactions_hashmap_name(eoa, chain_id), - nonce, - prepared_tx_json: serde_json::to_string(prepared_tx)?, - }; - - self.execute_with_watch_and_retry(eoa, chain_id, worker_id, &safe_tx) - .await?; - - Ok(()) - } - - /// Atomically move specific transaction from pending to borrowed with new nonce allocation - pub async fn atomic_move_pending_to_borrowed_with_new_nonce( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - expected_nonce: u64, - prepared_tx: &BorrowedTransactionData, - ) -> Result<(), TransactionStoreError> { - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let borrowed_key = self.borrowed_transactions_hashmap_name(eoa, chain_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - let prepared_tx_json = serde_json::to_string(prepared_tx)?; - let transaction_id = transaction_id.to_string(); - - self.execute_with_watch_and_retry( - eoa, - chain_id, - worker_id, - &MovePendingToBorrowedWithNewNonce { - nonce: expected_nonce, - prepared_tx_json, - transaction_id, - borrowed_key, - optimistic_key, - pending_key, - eoa, - chain_id, - }, - ) - .await - } - - /// Generic helper that handles WATCH + retry logic for atomic operations - /// The operation closure receives a mutable connection and should: - /// 1. Perform any validation (return early errors if needed) - /// 2. Build and execute the pipeline - /// 3. Return the result - pub async fn with_atomic_operation( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - watch_keys: Vec, - operation_name: &str, - operation: F, - ) -> Result - where - F: Fn(&mut ConnectionManager) -> Fut, - Fut: std::future::Future>, - { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let mut retry_count = 0; - - loop { - if retry_count >= MAX_RETRIES { - return Err(TransactionStoreError::InternalError { - message: format!( - "Exceeded max retries ({}) for {} on {}:{}", - MAX_RETRIES, operation_name, eoa, chain_id - ), - }); - } - - // Exponential backoff after first retry - if retry_count > 0 { - let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); - tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; - tracing::debug!( - retry_count = retry_count, - delay_ms = delay_ms, - eoa = %eoa, - chain_id = chain_id, - operation = operation_name, - "Retrying atomic operation" - ); - } - - // WATCH all specified keys (lock is always included) - let mut watch_cmd = twmq::redis::cmd("WATCH"); - watch_cmd.arg(&lock_key); - for key in &watch_keys { - watch_cmd.arg(key); - } - let _: () = watch_cmd.query_async(&mut conn).await?; - - // Check if we still own the lock - let current_owner: Option = conn.get(&lock_key).await?; - match current_owner { - Some(owner) if owner == worker_id => { - // We still own it, proceed - } - _ => { - // Lost ownership - immediately fail - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - } - - // Execute operation (includes validation and pipeline execution) - match operation(&mut conn).await { - Ok(result) => return Ok(result), - Err(TransactionStoreError::LockLost { .. }) => { - // Lock was lost during operation, propagate immediately - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - Err(TransactionStoreError::WatchFailed) => { - // WATCH failed, check if it was our lock - let still_own_lock: Option = conn.get(&lock_key).await?; - if still_own_lock.as_deref() != Some(worker_id) { - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - // Our lock is fine, retry - retry_count += 1; - continue; - } - Err(other_error) => { - // Other errors propagate immediately (validation failures, etc.) - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(other_error); - } - } - } - } - - /// Wrapper that executes operations with lock validation using WATCH/MULTI/EXEC - pub async fn with_lock_check( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - operation: F, - ) -> Result - where - F: Fn(&mut Pipeline) -> R, - T: From, - { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let mut retry_count = 0; - - loop { - if retry_count >= MAX_RETRIES { - return Err(TransactionStoreError::InternalError { - message: format!( - "Exceeded max retries ({}) for lock check on {}:{}", - MAX_RETRIES, eoa, chain_id - ), - }); - } - - // Exponential backoff after first retry - if retry_count > 0 { - let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); - tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; - tracing::debug!( - retry_count = retry_count, - delay_ms = delay_ms, - eoa = %eoa, - chain_id = chain_id, - "Retrying lock check operation" - ); - } - - // WATCH the EOA lock - let _: () = twmq::redis::cmd("WATCH") - .arg(&lock_key) - .query_async(&mut conn) - .await?; - - // Check if we still own the lock - let current_owner: Option = conn.get(&lock_key).await?; - match current_owner { - Some(owner) if owner == worker_id => { - // We still own it, proceed - } - _ => { - // Lost ownership - immediately fail - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - } - - // Build pipeline with operation - let mut pipeline = twmq::redis::pipe(); - pipeline.atomic(); - let result = operation(&mut pipeline); - - // Execute with WATCH protection - match pipeline - .query_async::>(&mut conn) - .await - { - Ok(_) => return Ok(T::from(result)), - Err(_) => { - // WATCH failed, check if it was our lock or someone else's - let still_own_lock: Option = conn.get(&lock_key).await?; - if still_own_lock.as_deref() != Some(worker_id) { - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - // Our lock is fine, someone else's WATCH failed - retry - retry_count += 1; - continue; - } - } - } - } - - // ========== ATOMIC OPERATIONS ========== - - /// Peek all borrowed transactions without removing them - pub async fn peek_borrowed_transactions( - &self, - eoa: Address, - chain_id: u64, - ) -> Result, TransactionStoreError> { - let borrowed_key = self.borrowed_transactions_hashmap_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let borrowed_map: HashMap = conn.hgetall(&borrowed_key).await?; - let mut result = Vec::new(); - - for (_nonce_str, transaction_json) in borrowed_map { - let borrowed_data: BorrowedTransactionData = serde_json::from_str(&transaction_json)?; - result.push(borrowed_data); - } - - Ok(result) - } - - /// Atomically move borrowed transaction to submitted state - /// Returns error if transaction not found in borrowed state - pub async fn move_borrowed_to_submitted( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - nonce: u64, - hash: &str, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - let borrowed_key = self.borrowed_transactions_hashmap_name(eoa, chain_id); - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); - let hash = hash.to_string(); - let transaction_id = transaction_id.to_string(); - - self.execute_with_watch_and_retry( - eoa, - chain_id, - worker_id, - &MoveBorrowedToSubmitted { - nonce, - hash: hash.to_string(), - transaction_id, - borrowed_key, - submitted_key, - hash_to_id_key, - }, - ) - .await - } - - /// Atomically move borrowed transaction back to recycled nonces and pending queue - /// Returns error if transaction not found in borrowed state - pub async fn move_borrowed_to_recycled( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - nonce: u64, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - let borrowed_key = self.borrowed_transactions_hashmap_name(eoa, chain_id); - let recycled_key = self.recycled_nonces_set_name(eoa, chain_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - let transaction_id = transaction_id.to_string(); - - self.execute_with_watch_and_retry( - eoa, - chain_id, - worker_id, - &MoveBorrowedToRecycled { - nonce, - transaction_id, - borrowed_key, - recycled_key, - pending_key, - }, - ) - .await - } - - /// Get all hashes below a certain nonce from submitted transactions - /// Returns (nonce, hash, transaction_id) tuples - pub async fn get_hashes_below_nonce( - &self, - eoa: Address, - chain_id: u64, - below_nonce: u64, - ) -> Result, TransactionStoreError> { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // Get all entries with nonce < below_nonce - let results: Vec<(String, u64)> = conn - .zrangebyscore_withscores(&submitted_key, 0, below_nonce - 1) - .await?; - - let mut parsed_results = Vec::new(); - for (hash_id_value, nonce) in results { - // Parse hash:id format - if let Some((hash, transaction_id)) = hash_id_value.split_once(':') { - parsed_results.push((nonce, hash.to_string(), transaction_id.to_string())); - } else { - // Fallback for old format (just hash) - look up transaction ID - if let Some(transaction_id) = - self.get_transaction_id_for_hash(&hash_id_value).await? - { - parsed_results.push((nonce, hash_id_value, transaction_id)); - } - } - } - - Ok(parsed_results) - } - - /// Get all transaction IDs for a specific nonce - pub async fn get_transaction_ids_for_nonce( - &self, - eoa: Address, - chain_id: u64, - nonce: u64, - ) -> Result, TransactionStoreError> { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // Get all members with the exact nonce - let members: Vec = conn - .zrangebyscore(&submitted_key, nonce, nonce) - .await - .map_err(|e| TransactionStoreError::RedisError { - message: format!("Failed to get transaction IDs for nonce {}: {}", nonce, e), - })?; - - let mut transaction_ids = Vec::new(); - for value in members { - // Parse the value as hash:id format, with fallback to old format - if let Some((_, transaction_id)) = value.split_once(':') { - // New format: hash:id - transaction_ids.push(transaction_id.to_string()); - } else { - // Old format: just hash - look up transaction ID - if let Some(transaction_id) = self.get_transaction_id_for_hash(&value).await? { - transaction_ids.push(transaction_id); - } - } - } - - Ok(transaction_ids) - } - - /// Remove all hashes for a transaction and requeue it - /// Returns error if no hashes found for this transaction in submitted state - /// NOTE: This method keeps the original boilerplate pattern because it needs to pass - /// complex data (transaction_hashes) from validation to pipeline phase. - /// The helper pattern works best for simple validation that doesn't need to pass data. - pub async fn fail_and_requeue_transaction( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let mut retry_count = 0; - - loop { - if retry_count >= MAX_RETRIES { - return Err(TransactionStoreError::InternalError { - message: format!( - "Exceeded max retries ({}) for fail and requeue transaction {}:{} tx:{}", - MAX_RETRIES, eoa, chain_id, transaction_id - ), - }); - } - - if retry_count > 0 { - let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); - tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; - } - - // WATCH lock and submitted state - let _: () = twmq::redis::cmd("WATCH") - .arg(&lock_key) - .arg(&submitted_key) - .query_async(&mut conn) - .await?; - - // Check lock ownership - let current_owner: Option = conn.get(&lock_key).await?; - if current_owner.as_deref() != Some(worker_id) { - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - - // Find all hashes for this transaction that actually exist in submitted - let all_hash_id_values: Vec = conn.zrange(&submitted_key, 0, -1).await?; - let mut transaction_hashes = Vec::new(); - - for hash_id_value in all_hash_id_values { - // Parse hash:id format - if let Some((hash, tx_id)) = hash_id_value.split_once(':') { - if tx_id == transaction_id { - transaction_hashes.push(hash.to_string()); - } - } else { - // Fallback for old format (just hash) - look up transaction ID - if let Some(tx_id) = self.get_transaction_id_for_hash(&hash_id_value).await? { - if tx_id == transaction_id { - transaction_hashes.push(hash_id_value); - } - } - } - } - - if transaction_hashes.is_empty() { - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::TransactionNotInSubmittedState { - transaction_id: transaction_id.to_string(), - }); - } - - // Transaction has hashes in submitted, proceed with atomic removal and requeue - let mut pipeline = twmq::redis::pipe(); - pipeline.atomic(); - - // Remove all hash:id values for this transaction (we know they exist) - for hash in &transaction_hashes { - // Remove the hash:id value from the zset - let hash_id_value = format!("{}:{}", hash, transaction_id); - pipeline.zrem(&submitted_key, &hash_id_value); - - // Also remove the separate hash-to-ID mapping for backward compatibility - let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); - pipeline.del(&hash_to_id_key); - } - - // Add back to pending - pipeline.lpush(&pending_key, transaction_id); - - match pipeline - .query_async::>(&mut conn) - .await - { - Ok(_) => return Ok(()), // Success - Err(_) => { - // WATCH failed, check if it was our lock - let still_own_lock: Option = conn.get(&lock_key).await?; - if still_own_lock.as_deref() != Some(worker_id) { - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - // Submitted state changed, retry - retry_count += 1; - continue; - } - } - } - } - - /// Check EOA health (balance, etc.) - pub async fn check_eoa_health( - &self, - eoa: Address, - chain_id: u64, - ) -> Result, TransactionStoreError> { - let health_key = self.eoa_health_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let health_json: Option = conn.get(&health_key).await?; - if let Some(json) = health_json { - let health: EoaHealth = serde_json::from_str(&json)?; - Ok(Some(health)) - } else { - Ok(None) - } - } - - /// Update EOA health data - pub async fn update_health_data( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - health: &EoaHealth, - ) -> Result<(), TransactionStoreError> { - let health_json = serde_json::to_string(health)?; - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let health_key = self.eoa_health_key_name(eoa, chain_id); - pipeline.set(&health_key, &health_json); - }) - .await - } - - /// Update cached transaction count - pub async fn update_cached_transaction_count( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_count: u64, - ) -> Result<(), TransactionStoreError> { - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let tx_count_key = self.last_transaction_count_key_name(eoa, chain_id); - pipeline.set(&tx_count_key, transaction_count); - }) - .await - } - - /// Peek recycled nonces without removing them - pub async fn peek_recycled_nonces( - &self, - eoa: Address, - chain_id: u64, - ) -> Result, TransactionStoreError> { - let recycled_key = self.recycled_nonces_set_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let nonces: Vec = conn.zrange(&recycled_key, 0, -1).await?; - Ok(nonces) - } - - /// Peek at pending transactions without removing them (safe for planning) - pub async fn peek_pending_transactions( - &self, - eoa: Address, - chain_id: u64, - limit: u64, - ) -> Result, TransactionStoreError> { - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // Use LRANGE to peek without removing - let transaction_ids: Vec = - conn.lrange(&pending_key, 0, (limit as isize) - 1).await?; - Ok(transaction_ids) - } - - /// Get inflight budget (how many new transactions can be sent) - pub async fn get_inflight_budget( - &self, - eoa: Address, - chain_id: u64, - max_inflight: u64, - ) -> Result { - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let last_tx_count_key = self.last_transaction_count_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // Read both values atomically to avoid race conditions - let (optimistic_nonce, last_tx_count): (Option, Option) = twmq::redis::pipe() - .get(&optimistic_key) - .get(&last_tx_count_key) - .query_async(&mut conn) - .await?; - - let optimistic = match optimistic_nonce { - Some(nonce) => nonce, - None => return Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - }; - let last_count = match last_tx_count { - Some(count) => count, - None => return Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - }; - - let current_inflight = optimistic.saturating_sub(last_count); - let available_budget = max_inflight.saturating_sub(current_inflight); - - Ok(available_budget) - } - - /// Get current optimistic nonce (without incrementing) - pub async fn get_optimistic_nonce( - &self, - eoa: Address, - chain_id: u64, - ) -> Result { - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let current: Option = conn.get(&optimistic_key).await?; - match current { - Some(nonce) => Ok(nonce), - None => Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - } - } - - /// Lock key name for EOA processing - fn eoa_lock_key_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:lock:{chain_id}:{eoa}"), - None => format!("eoa_executor:lock:{chain_id}:{eoa}"), - } - } - - /// Get transaction ID for a given hash - pub async fn get_transaction_id_for_hash( - &self, - hash: &str, - ) -> Result, TransactionStoreError> { - let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); - let mut conn = self.redis.clone(); - - let transaction_id: Option = conn.get(&hash_to_id_key).await?; - Ok(transaction_id) - } - - /// Get transaction data by transaction ID - pub async fn get_transaction_data( - &self, - transaction_id: &str, - ) -> Result, TransactionStoreError> { - let tx_data_key = self.transaction_data_key_name(transaction_id); - let mut conn = self.redis.clone(); - - // Get the hash data (the transaction data is stored as a hash) - let hash_data: HashMap = conn.hgetall(&tx_data_key).await?; - - if hash_data.is_empty() { - return Ok(None); - } - - // Extract user_request from the hash data - let user_request_json = hash_data.get("user_request").ok_or_else(|| { - TransactionStoreError::TransactionNotFound { - transaction_id: transaction_id.to_string(), - } - })?; - - let user_request: EoaTransactionRequest = serde_json::from_str(user_request_json)?; - - // Extract receipt if present - let receipt = hash_data - .get("receipt") - .and_then(|receipt_str| serde_json::from_str(receipt_str).ok()); - - // Extract attempts from separate list - let attempts_key = self.transaction_attempts_list_name(transaction_id); - let attempts_json_list: Vec = conn.lrange(&attempts_key, 0, -1).await?; - let mut attempts = Vec::new(); - for attempt_json in attempts_json_list { - if let Ok(attempt) = serde_json::from_str::(&attempt_json) { - attempts.push(attempt); - } - } - - Ok(Some(TransactionData { - transaction_id: transaction_id.to_string(), - user_request, - receipt, - attempts, - })) - } - - /// Mark transaction as successful and remove from submitted - pub async fn succeed_transaction( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - hash: &str, - receipt: &str, - ) -> Result<(), TransactionStoreError> { - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); - let tx_data_key = self.transaction_data_key_name(transaction_id); - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - // Remove this hash:id from submitted - let hash_id_value = format!("{}:{}", hash, transaction_id); - pipeline.zrem(&submitted_key, &hash_id_value); - - // Remove hash mapping - pipeline.del(&hash_to_id_key); - - // Update transaction data with success - pipeline.hset(&tx_data_key, "completed_at", now); - pipeline.hset(&tx_data_key, "receipt", receipt); - pipeline.hset(&tx_data_key, "status", "confirmed"); - }) - .await - } - - /// Add a gas bump attempt (new hash) to submitted transactions - pub async fn add_gas_bump_attempt( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - signed_transaction: Signed, - ) -> Result<(), TransactionStoreError> { - let new_hash = signed_transaction.hash().to_string(); - let nonce = signed_transaction.nonce(); - - // Create new attempt - let new_attempt = TransactionAttempt { - transaction_id: transaction_id.to_string(), - details: signed_transaction, - sent_at: chrono::Utc::now().timestamp_millis().max(0) as u64, - attempt_number: 0, // Will be set correctly when reading all attempts - }; - - // Serialize the new attempt - let attempt_json = serde_json::to_string(&new_attempt)?; - - // Get key names - let attempts_list_key = self.transaction_attempts_list_name(transaction_id); - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let hash_to_id_key = self.transaction_hash_to_id_key_name(&new_hash); - let hash_id_value = format!("{}:{}", new_hash, transaction_id); - - // Now perform the atomic update - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - // Add new hash:id to submitted (keeping old ones) - pipeline.zadd(&submitted_key, &hash_id_value, nonce); - - // Still maintain separate hash-to-ID mapping for backward compatibility - pipeline.set(&hash_to_id_key, transaction_id); - - // Simply push the new attempt to the attempts list - pipeline.lpush(&attempts_list_key, &attempt_json); - }) - .await - } - - /// Efficiently batch fail and requeue multiple transactions - /// This avoids hash-to-ID lookups since we already have both pieces of information - pub async fn batch_fail_and_requeue_transactions( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - failures: Vec, - ) -> Result<(), TransactionStoreError> { - if failures.is_empty() { - return Ok(()); - } - - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - - // Remove all hash:id values from submitted - for failure in &failures { - let hash_id_value = format!("{}:{}", failure.hash, failure.transaction_id); - pipeline.zrem(&submitted_key, &hash_id_value); - - // Remove separate hash-to-ID mapping - let hash_to_id_key = self.transaction_hash_to_id_key_name(&failure.hash); - pipeline.del(&hash_to_id_key); - } - - // Add unique transaction IDs back to pending (avoid duplicates) - let mut unique_tx_ids = std::collections::HashSet::new(); - for failure in &failures { - unique_tx_ids.insert(&failure.transaction_id); - } - - for transaction_id in unique_tx_ids { - pipeline.lpush(&pending_key, transaction_id); - } - }) - .await - } - - /// Efficiently batch succeed multiple transactions - /// This avoids hash-to-ID lookups since we already have both pieces of information - pub async fn batch_succeed_transactions( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - successes: Vec, - ) -> Result<(), TransactionStoreError> { - if successes.is_empty() { - return Ok(()); - } - - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - for success in &successes { - // Remove hash:id from submitted - let hash_id_value = format!("{}:{}", success.hash, success.transaction_id); - pipeline.zrem(&submitted_key, &hash_id_value); - - // Remove separate hash-to-ID mapping - let hash_to_id_key = self.transaction_hash_to_id_key_name(&success.hash); - pipeline.del(&hash_to_id_key); - - // Update transaction data with success (following existing Redis hash pattern) - let tx_data_key = self.transaction_data_key_name(&success.transaction_id); - pipeline.hset(&tx_data_key, "completed_at", now); - pipeline.hset(&tx_data_key, "receipt", &success.receipt_data); - pipeline.hset(&tx_data_key, "status", "confirmed"); - } - }) - .await - } - - // ========== SEND FLOW ========== - - /// Get cached transaction count - pub async fn get_cached_transaction_count( - &self, - eoa: Address, - chain_id: u64, - ) -> Result { - let tx_count_key = self.last_transaction_count_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let count: Option = conn.get(&tx_count_key).await?; - match count { - Some(count) => Ok(count), - None => Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - } - } - - /// Peek next available nonce (recycled or new) - pub async fn peek_next_available_nonce( - &self, - eoa: Address, - chain_id: u64, - ) -> Result { - // Check recycled nonces first - let recycled = self.peek_recycled_nonces(eoa, chain_id).await?; - if !recycled.is_empty() { - return Ok(NonceType::Recycled(recycled[0])); - } - - // Get next optimistic nonce - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let current_optimistic: Option = conn.get(&optimistic_key).await?; - - match current_optimistic { - Some(nonce) => Ok(NonceType::Incremented(nonce)), - None => Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - } - } - - /// Synchronize nonces with the chain - /// - /// Part of standard nonce management flow, called in the confirm stage when chain nonce advances, and we need to update our cached nonce - pub async fn synchronize_nonces_with_chain( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - current_chain_tx_count: u64, - ) -> Result<(), TransactionStoreError> { - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - // First, read current health data - let current_health = self.check_eoa_health(eoa, chain_id).await?; - - // Prepare health update if health data exists - let health_update = if let Some(mut health) = current_health { - health.last_nonce_movement_at = now; - health.last_confirmation_at = now; - Some(serde_json::to_string(&health)?) - } else { - None - }; - - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let tx_count_key = self.last_transaction_count_key_name(eoa, chain_id); - - // Update cached transaction count - pipeline.set(&tx_count_key, current_chain_tx_count); - - // Update health data only if it exists - if let Some(ref health_json) = health_update { - let health_key = self.eoa_health_key_name(eoa, chain_id); - pipeline.set(&health_key, health_json); - } - }) - .await - } - - /// Reset nonces to specified value - /// - /// This is called when we have too many recycled nonces and detect something wrong - /// We want to start fresh, with the chain nonce as the new optimistic nonce - pub async fn reset_nonces( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - current_chain_tx_count: u64, - ) -> Result<(), TransactionStoreError> { - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - let current_health = self.check_eoa_health(eoa, chain_id).await?; - - // Prepare health update if health data exists - let health_update = if let Some(mut health) = current_health { - health.nonce_resets.push(now); - Some(serde_json::to_string(&health)?) - } else { - None - }; - - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let cached_nonce_key = self.last_transaction_count_key_name(eoa, chain_id); - let recycled_key = self.recycled_nonces_set_name(eoa, chain_id); - - // Update health data only if it exists - if let Some(ref health_json) = health_update { - let health_key = self.eoa_health_key_name(eoa, chain_id); - pipeline.set(&health_key, health_json); - } - - // Reset the optimistic nonce - pipeline.set(&optimistic_key, current_chain_tx_count); - - // Reset the cached nonce - pipeline.set(&cached_nonce_key, current_chain_tx_count); - - // Reset the recycled nonces - pipeline.del(recycled_key); - }) - .await - } - - /// Add a transaction to the pending queue and store its data - /// This is called when a new transaction request comes in for an EOA - pub async fn add_transaction( - &self, - transaction_request: EoaTransactionRequest, - ) -> Result<(), TransactionStoreError> { - let transaction_id = &transaction_request.transaction_id; - let eoa = transaction_request.from; - let chain_id = transaction_request.chain_id; - - let tx_data_key = self.transaction_data_key_name(transaction_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - - // Store transaction data as JSON in the user_request field of the hash - let user_request_json = serde_json::to_string(&transaction_request)?; - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - let mut conn = self.redis.clone(); - - // Use a pipeline to atomically store data and add to pending queue - let mut pipeline = twmq::redis::pipe(); - - // Store transaction data - pipeline.hset(&tx_data_key, "user_request", &user_request_json); - pipeline.hset(&tx_data_key, "status", "pending"); - pipeline.hset(&tx_data_key, "created_at", now); - - // Add to pending queue - pipeline.lpush(&pending_key, transaction_id); - - pipeline.query_async::<()>(&mut conn).await?; - - Ok(()) - } -} - -// Additional error types -#[derive(Debug, thiserror::Error, Serialize, Deserialize, Clone)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE", tag = "errorCode")] -pub enum TransactionStoreError { - #[error("Redis error: {message}")] - RedisError { message: String }, - - #[error("Serialization error: {message}")] - DeserError { message: String, text: String }, - - #[error("Transaction not found: {transaction_id}")] - TransactionNotFound { transaction_id: String }, - - #[error("Lost EOA lock: {eoa}:{chain_id} worker: {worker_id}")] - LockLost { - eoa: Address, - chain_id: u64, - worker_id: String, - }, - - #[error("Internal error - worker should quit: {message}")] - InternalError { message: String }, - - #[error("Transaction {transaction_id} not in borrowed state for nonce {nonce}")] - TransactionNotInBorrowedState { transaction_id: String, nonce: u64 }, - - #[error("Hash {hash} not found in submitted transactions")] - HashNotInSubmittedState { hash: String }, - - #[error("Transaction {transaction_id} has no hashes in submitted state")] - TransactionNotInSubmittedState { transaction_id: String }, - - #[error("Nonce {nonce} not available in recycled set")] - NonceNotInRecycledSet { nonce: u64 }, - - #[error("Transaction {transaction_id} not found in pending queue")] - TransactionNotInPendingQueue { transaction_id: String }, - - #[error("Optimistic nonce changed: expected {expected}, found {actual}")] - OptimisticNonceChanged { expected: u64, actual: u64 }, - - #[error("WATCH failed - state changed during operation")] - WatchFailed, - - #[error( - "Nonce synchronization required for {eoa}:{chain_id} - no cached transaction count available" - )] - NonceSyncRequired { eoa: Address, chain_id: u64 }, -} - -impl From for TransactionStoreError { - fn from(error: twmq::redis::RedisError) -> Self { - TransactionStoreError::RedisError { - message: error.to_string(), - } - } -} - -impl From for TransactionStoreError { - fn from(error: serde_json::Error) -> Self { - TransactionStoreError::DeserError { - message: error.to_string(), - text: error.to_string(), - } - } -} - -const MAX_RETRIES: u32 = 10; -const RETRY_BASE_DELAY_MS: u64 = 10; - -/// Scoped transaction store for a specific EOA, chain, and worker -/// -/// This wrapper eliminates the need to repeatedly pass EOA, chain_id, and worker_id -/// to every method call. It provides the same interface as TransactionStore but with -/// these parameters already bound. -/// -/// ## Usage: -/// ```rust -/// let scoped = ScopedTransactionStore::build(store, eoa, chain_id, worker_id).await?; -/// -/// // Much cleaner method calls: -/// scoped.peek_pending_transactions(limit).await?; -/// scoped.move_borrowed_to_submitted(nonce, hash, tx_id, attempt).await?; -/// ``` -pub struct ScopedEoaExecutorStore<'a> { - store: &'a EoaExecutorStore, - eoa: Address, - chain_id: u64, - worker_id: String, -} - -impl<'a> ScopedEoaExecutorStore<'a> { - /// Build a scoped transaction store for a specific EOA, chain, and worker - /// - /// This acquires the lock for the given EOA/chain. - /// If the lock is not acquired, returns a LockLost error. - #[tracing::instrument(skip_all, fields(eoa = %eoa, chain_id = chain_id, worker_id = %worker_id))] - pub async fn build( - store: &'a EoaExecutorStore, - eoa: Address, - chain_id: u64, - worker_id: String, - ) -> Result { - // 1. ACQUIRE LOCK AGGRESSIVELY - tracing::info!("Acquiring EOA lock aggressively"); - store - .acquire_eoa_lock_aggressively(eoa, chain_id, &worker_id) - .await - .map_err(|e| { - tracing::error!("Failed to acquire EOA lock: {}", e); - TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.clone(), - } - })?; - - Ok(Self { - store, - eoa, - chain_id, - worker_id, - }) - } - - /// Create a scoped store without lock validation (for read-only operations) - pub fn new_unchecked( - store: &'a EoaExecutorStore, - eoa: Address, - chain_id: u64, - worker_id: String, - ) -> Self { - Self { - store, - eoa, - chain_id, - worker_id, - } - } - - // ========== ATOMIC OPERATIONS ========== - - /// Atomically move specific transaction from pending to borrowed with recycled nonce allocation - pub async fn atomic_move_pending_to_borrowed_with_recycled_nonce( - &self, - transaction_id: &str, - nonce: u64, - prepared_tx: &BorrowedTransactionData, - ) -> Result<(), TransactionStoreError> { - self.store - .atomic_move_pending_to_borrowed_with_recycled_nonce( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_id, - nonce, - prepared_tx, - ) - .await - } - - /// Atomically move specific transaction from pending to borrowed with new nonce allocation - pub async fn atomic_move_pending_to_borrowed_with_new_nonce( - &self, - transaction_id: &str, - expected_nonce: u64, - prepared_tx: &BorrowedTransactionData, - ) -> Result<(), TransactionStoreError> { - self.store - .atomic_move_pending_to_borrowed_with_new_nonce( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_id, - expected_nonce, - prepared_tx, - ) - .await - } - - /// Peek all borrowed transactions without removing them - pub async fn peek_borrowed_transactions( - &self, - ) -> Result, TransactionStoreError> { - self.store - .peek_borrowed_transactions(self.eoa, self.chain_id) - .await - } - - /// Atomically move borrowed transaction to submitted state - pub async fn move_borrowed_to_submitted( - &self, - nonce: u64, - hash: &str, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - self.store - .move_borrowed_to_submitted( - self.eoa, - self.chain_id, - &self.worker_id, - nonce, - hash, - transaction_id, - ) - .await - } - - /// Atomically move borrowed transaction back to recycled nonces and pending queue - pub async fn move_borrowed_to_recycled( - &self, - nonce: u64, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - self.store - .move_borrowed_to_recycled( - self.eoa, - self.chain_id, - &self.worker_id, - nonce, - transaction_id, - ) - .await - } - - /// Get all hashes below a certain nonce from submitted transactions - /// Returns (nonce, hash, transaction_id) tuples - pub async fn get_hashes_below_nonce( - &self, - below_nonce: u64, - ) -> Result, TransactionStoreError> { - self.store - .get_hashes_below_nonce(self.eoa, self.chain_id, below_nonce) - .await - } - - /// Get all transaction IDs for a specific nonce - pub async fn get_transaction_ids_for_nonce( - &self, - nonce: u64, - ) -> Result, TransactionStoreError> { - self.store - .get_transaction_ids_for_nonce(self.eoa, self.chain_id, nonce) - .await - } - - /// Remove all hashes for a transaction and requeue it - pub async fn fail_and_requeue_transaction( - &self, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - self.store - .fail_and_requeue_transaction(self.eoa, self.chain_id, &self.worker_id, transaction_id) - .await - } - - /// Efficiently batch fail and requeue multiple transactions - pub async fn batch_fail_and_requeue_transactions( - &self, - failures: Vec, - ) -> Result<(), TransactionStoreError> { - self.store - .batch_fail_and_requeue_transactions(self.eoa, self.chain_id, &self.worker_id, failures) - .await - } - - /// Efficiently batch succeed multiple transactions - pub async fn batch_succeed_transactions( - &self, - successes: Vec, - ) -> Result<(), TransactionStoreError> { - self.store - .batch_succeed_transactions(self.eoa, self.chain_id, &self.worker_id, successes) - .await - } - - // ========== EOA HEALTH & NONCE MANAGEMENT ========== - - /// Check EOA health (balance, etc.) - pub async fn check_eoa_health(&self) -> Result, TransactionStoreError> { - self.store.check_eoa_health(self.eoa, self.chain_id).await - } - - /// Update EOA health data - pub async fn update_health_data( - &self, - health: &EoaHealth, - ) -> Result<(), TransactionStoreError> { - self.store - .update_health_data(self.eoa, self.chain_id, &self.worker_id, health) - .await - } - - /// Update cached transaction count - pub async fn update_cached_transaction_count( - &self, - transaction_count: u64, - ) -> Result<(), TransactionStoreError> { - self.store - .update_cached_transaction_count( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_count, - ) - .await - } - - /// Peek recycled nonces without removing them - pub async fn peek_recycled_nonces(&self) -> Result, TransactionStoreError> { - self.store - .peek_recycled_nonces(self.eoa, self.chain_id) - .await - } - - /// Peek at pending transactions without removing them - pub async fn peek_pending_transactions( - &self, - limit: u64, - ) -> Result, TransactionStoreError> { - self.store - .peek_pending_transactions(self.eoa, self.chain_id, limit) - .await - } - - /// Get inflight budget (how many new transactions can be sent) - pub async fn get_inflight_budget( - &self, - max_inflight: u64, - ) -> Result { - self.store - .get_inflight_budget(self.eoa, self.chain_id, max_inflight) - .await - } - - /// Get current optimistic nonce (without incrementing) - pub async fn get_optimistic_nonce(&self) -> Result { - self.store - .get_optimistic_nonce(self.eoa, self.chain_id) - .await - } - - /// Mark transaction as successful and remove from submitted - pub async fn succeed_transaction( - &self, - transaction_id: &str, - hash: &str, - receipt: &str, - ) -> Result<(), TransactionStoreError> { - self.store - .succeed_transaction( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_id, - hash, - receipt, - ) - .await - } - - /// Add a gas bump attempt (new hash) to submitted transactions - pub async fn add_gas_bump_attempt( - &self, - transaction_id: &str, - signed_transaction: Signed, - ) -> Result<(), TransactionStoreError> { - self.store - .add_gas_bump_attempt( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_id, - signed_transaction, - ) - .await - } - - pub async fn synchronize_nonces_with_chain( - &self, - nonce: u64, - ) -> Result<(), TransactionStoreError> { - self.store - .synchronize_nonces_with_chain(self.eoa, self.chain_id, &self.worker_id, nonce) - .await - } - - pub async fn reset_nonces(&self, nonce: u64) -> Result<(), TransactionStoreError> { - self.store - .reset_nonces(self.eoa, self.chain_id, &self.worker_id, nonce) - .await - } - - // ========== READ-ONLY OPERATIONS ========== - - /// Get cached transaction count - pub async fn get_cached_transaction_count(&self) -> Result { - self.store - .get_cached_transaction_count(self.eoa, self.chain_id) - .await - } - - /// Peek next available nonce (recycled or new) - pub async fn peek_next_available_nonce(&self) -> Result { - self.store - .peek_next_available_nonce(self.eoa, self.chain_id) - .await - } - - // ========== ACCESSORS ========== - - /// Get the EOA address this store is scoped to - pub fn eoa(&self) -> Address { - self.eoa - } - - /// Get the chain ID this store is scoped to - pub fn chain_id(&self) -> u64 { - self.chain_id - } - - /// Get the worker ID this store is scoped to - pub fn worker_id(&self) -> &str { - &self.worker_id - } - - /// Get a reference to the underlying transaction store - pub fn inner(&self) -> &EoaExecutorStore { - self.store - } - - /// Get transaction data by transaction ID - pub async fn get_transaction_data( - &self, - transaction_id: &str, - ) -> Result, TransactionStoreError> { - self.store.get_transaction_data(transaction_id).await - } -} diff --git a/executors/src/eoa/store/atomic.rs b/executors/src/eoa/store/atomic.rs new file mode 100644 index 0000000..50a74fc --- /dev/null +++ b/executors/src/eoa/store/atomic.rs @@ -0,0 +1,622 @@ +use std::sync::Arc; + +use alloy::{ + consensus::{Signed, TypedTransaction}, + primitives::Address, +}; +use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; + +use crate::{ + eoa::{ + EoaExecutorStore, + events::EoaExecutorEvent, + store::{ + BorrowedTransactionData, ConfirmedTransaction, EoaHealth, PendingTransaction, + SubmittedTransactionDehydrated, TransactionAttempt, TransactionStoreError, + borrowed::{BorrowedProcessingReport, ProcessBorrowedTransactions, SubmissionResult}, + pending::{ + MovePendingToBorrowedWithIncrementedNonces, MovePendingToBorrowedWithRecycledNonces, + }, + submitted::{ + CleanAndGetRecycledNonces, CleanSubmittedTransactions, CleanupReport, + SubmittedNoopTransaction, SubmittedTransaction, + }, + }, + worker::error::EoaExecutorWorkerError, + }, + webhook::{WebhookJobHandler, queue_webhook_envelopes}, +}; + +const MAX_RETRIES: u32 = 10; +const RETRY_BASE_DELAY_MS: u64 = 10; + +pub trait SafeRedisTransaction: Send + Sync { + type ValidationData; + type OperationResult; + + fn name(&self) -> &str; + fn operation( + &self, + pipeline: &mut Pipeline, + validation_data: Self::ValidationData, + ) -> Self::OperationResult; + fn validation( + &self, + conn: &mut ConnectionManager, + store: &EoaExecutorStore, + ) -> impl Future> + Send; + fn watch_keys(&self) -> Vec; +} + +/// Atomic transaction store that owns the base store and provides atomic operations +/// +/// This store is created by calling `acquire_lock()` on the base store and provides +/// access to both atomic (lock-protected) and non-atomic operations. +/// +/// ## Usage: +/// ```rust +/// let base_store = EoaExecutorStore::new(redis, namespace, ); +/// let atomic_store = base_store.acquire_lock(worker_id).await?; +/// +/// // Atomic operations: +/// atomic_store.move_borrowed_to_submitted(nonce, hash, tx_id).await?; +/// +/// // Non-atomic operations via deref: +/// atomic_store.peek_pending_transactions(limit).await?; +/// ``` +pub struct AtomicEoaExecutorStore { + pub store: EoaExecutorStore, + pub worker_id: String, +} + +impl std::ops::Deref for AtomicEoaExecutorStore { + type Target = EoaExecutorStore; + + fn deref(&self) -> &Self::Target { + &self.store + } +} + +impl AtomicEoaExecutorStore { + /// Get the EOA address this store is scoped to + pub fn eoa(&self) -> Address { + self.store.eoa + } + + /// Get the chain ID this store is scoped to + pub fn chain_id(&self) -> u64 { + self.store.chain_id + } + + /// Get the worker ID this store is scoped to + pub fn worker_id(&self) -> &str { + &self.worker_id + } + + /// Release EOA lock following the spec's finally pattern + pub async fn release_eoa_lock(self) -> Result { + // Use existing utility method that handles all the atomic lock checking + match self + .with_lock_check(|pipeline| { + let lock_key = self.eoa_lock_key_name(); + pipeline.del(&lock_key); + }) + .await + { + Ok(()) => { + tracing::debug!( + eoa = %self.eoa(), + chain_id = %self.chain_id(), + worker_id = %self.worker_id(), + "Successfully released EOA lock" + ); + Ok(self.store) + } + Err(TransactionStoreError::LockLost { .. }) => { + // Lock was already taken over, which is fine for release + tracing::debug!( + eoa = %self.eoa(), + chain_id = %self.chain_id(), + worker_id = %self.worker_id(), + "Lock already released or taken over by another worker" + ); + Ok(self.store) + } + Err(e) => { + // Other errors shouldn't fail the worker, just log + tracing::warn!( + eoa = %self.eoa(), + chain_id = %self.chain_id(), + worker_id = %self.worker_id(), + error = %e, + "Failed to release EOA lock" + ); + Ok(self.store) + } + } + } + + /// Atomically move multiple pending transactions to borrowed state using incremented nonces + /// + /// The transactions must have sequential nonces starting from the current optimistic count. + /// This operation validates nonce ordering and atomically moves all transactions. + pub async fn atomic_move_pending_to_borrowed_with_incremented_nonces( + &self, + transactions: &[BorrowedTransactionData], + ) -> Result { + self.execute_with_watch_and_retry(&MovePendingToBorrowedWithIncrementedNonces { + transactions, + keys: &self.keys, + eoa: self.eoa, + chain_id: self.chain_id, + }) + .await + } + + /// Atomically move multiple pending transactions to borrowed state using recycled nonces + /// + /// All nonces must exist in the recycled nonces set. This operation validates nonce + /// availability and atomically moves all transactions. + pub async fn atomic_move_pending_to_borrowed_with_recycled_nonces( + &self, + transactions: &[BorrowedTransactionData], + ) -> Result { + self.execute_with_watch_and_retry(&MovePendingToBorrowedWithRecycledNonces { + transactions, + keys: &self.keys, + }) + .await + } + + /// Wrapper that executes operations with lock validation using WATCH/MULTI/EXEC + pub async fn with_lock_check(&self, operation: F) -> Result + where + F: Fn(&mut Pipeline) -> R, + T: From, + { + let lock_key = self.eoa_lock_key_name(); + let mut conn = self.redis.clone(); + let mut retry_count = 0; + + loop { + if retry_count >= MAX_RETRIES { + return Err(TransactionStoreError::InternalError { + message: format!( + "Exceeded max retries ({}) for lock check on {}:{}", + MAX_RETRIES, + self.eoa(), + self.chain_id() + ), + }); + } + + // Exponential backoff after first retry + if retry_count > 0 { + let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); + tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; + tracing::debug!( + retry_count = retry_count, + delay_ms = delay_ms, + eoa = %self.eoa(), + chain_id = self.chain_id(), + "Retrying lock check operation" + ); + } + + // WATCH the EOA lock + let _: () = twmq::redis::cmd("WATCH") + .arg(&lock_key) + .query_async(&mut conn) + .await?; + + // Check if we still own the lock + let current_owner: Option = conn.get(&lock_key).await?; + match current_owner { + Some(owner) if owner == self.worker_id() => { + // We still own it, proceed + } + _ => { + // Lost ownership - immediately fail + let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; + return Err(self.eoa_lock_lost_error()); + } + } + + // Build pipeline with operation + let mut pipeline = twmq::redis::pipe(); + pipeline.atomic(); + let result = operation(&mut pipeline); + + // Execute with WATCH protection + match pipeline + .query_async::>(&mut conn) + .await + { + Ok(_) => return Ok(T::from(result)), + Err(_) => { + // WATCH failed, check if it was our lock or someone else's + let still_own_lock: Option = conn.get(&lock_key).await?; + if still_own_lock.as_deref() != Some(self.worker_id()) { + return Err(self.eoa_lock_lost_error()); + } + // Our lock is fine, someone else's WATCH failed - retry + retry_count += 1; + continue; + } + } + } + } + + /// Helper to execute atomic operations with proper retry logic and watch handling + /// + /// This helper centralizes all the boilerplate for WATCH/MULTI/EXEC operations: + /// - Retry logic with exponential backoff + /// - Lock ownership validation + /// - WATCH key management + /// - Error handling and UNWATCH cleanup + /// + /// ## Usage: + /// Implement the `SafeRedisTransaction` trait for your operation, then call this method. + /// The trait separates validation (async) from pipeline operations (sync) for clean patterns. + /// + /// ## Example: + /// ```rust + /// let safe_tx = MovePendingToBorrowedWithNewNonce { + /// nonce: expected_nonce, + /// prepared_tx_json, + /// transaction_id, + /// borrowed_key, + /// optimistic_key, + /// pending_key, + /// eoa, + /// chain_id, + /// }; + /// + /// self.execute_with_watch_and_retry(, worker_id, &safe_tx).await?; + /// ``` + /// + /// ## When to use this helper: + /// - Operations that implement `SafeRedisTransaction` trait + /// - Need atomic WATCH/MULTI/EXEC with retry logic + /// - Want centralized lock checking and error handling + /// + /// ## When NOT to use this helper: + /// - Simple operations that can use `with_lock_check` instead + /// - Operations that don't need WATCH on multiple keys + /// - Read-only operations that don't modify state + async fn execute_with_watch_and_retry( + &self, + safe_tx: &T, + ) -> Result { + let lock_key = self.eoa_lock_key_name(); + let mut conn = self.redis.clone(); + let mut retry_count = 0; + + loop { + if retry_count >= MAX_RETRIES { + return Err(TransactionStoreError::InternalError { + message: format!( + "Exceeded max retries ({}) for {} on {}:{}", + MAX_RETRIES, + safe_tx.name(), + self.eoa, + self.chain_id + ), + }); + } + + // Exponential backoff after first retry + if retry_count > 0 { + let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); + tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; + tracing::debug!( + retry_count = retry_count, + delay_ms = delay_ms, + eoa = %self.eoa, + chain_id = self.chain_id, + operation = safe_tx.name(), + "Retrying atomic operation" + ); + } + + // WATCH all specified keys including lock + let mut watch_cmd = twmq::redis::cmd("WATCH"); + watch_cmd.arg(&lock_key); + for key in safe_tx.watch_keys() { + watch_cmd.arg(key); + } + let _: () = watch_cmd.query_async(&mut conn).await?; + + // Check lock ownership + let current_owner: Option = conn.get(&lock_key).await?; + if current_owner.as_deref() != Some(self.worker_id()) { + let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; + return Err(TransactionStoreError::LockLost { + eoa: self.eoa, + chain_id: self.chain_id, + worker_id: self.worker_id().to_string(), + }); + } + + // Execute validation + match safe_tx.validation(&mut conn, &self.store).await { + Ok(validation_data) => { + // Build and execute pipeline + let mut pipeline = twmq::redis::pipe(); + pipeline.atomic(); + let result = safe_tx.operation(&mut pipeline, validation_data); + + match pipeline + .query_async::>(&mut conn) + .await + { + Ok(_) => return Ok(result), // Success + Err(e) => { + tracing::error!("WATCH failed: {}", e); + // WATCH failed, check if it was our lock + let still_own_lock: Option = conn.get(&lock_key).await?; + if still_own_lock.as_deref() != Some(self.worker_id()) { + return Err(TransactionStoreError::LockLost { + eoa: self.eoa, + chain_id: self.chain_id, + worker_id: self.worker_id().to_string(), + }); + } + // State changed, retry + retry_count += 1; + continue; + } + } + } + Err(e) => { + // Validation failed, unwatch and return error + let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; + return Err(e); + } + } + } + } + + /// Update EOA health data + pub async fn update_health_data( + &self, + health: &EoaHealth, + ) -> Result<(), TransactionStoreError> { + let health_json = serde_json::to_string(health)?; + self.with_lock_check(|pipeline| { + let health_key = self.eoa_health_key_name(); + pipeline.set(&health_key, &health_json); + }) + .await + } + + /// Synchronize nonces with the chain + /// + /// Part of standard nonce management flow, called in the confirm stage when chain nonce advances, and we need to update our cached nonce + pub async fn update_cached_transaction_count( + &self, + current_chain_tx_count: u64, + ) -> Result<(), TransactionStoreError> { + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // First, read current health data + let current_health = self.get_eoa_health().await?; + + // Prepare health update if health data exists + let health_update = if let Some(mut health) = current_health { + health.last_nonce_movement_at = now; + health.last_confirmation_at = now; + Some(serde_json::to_string(&health)?) + } else { + None + }; + + self.with_lock_check(|pipeline| { + let tx_count_key = self.last_transaction_count_key_name(); + + // Update cached transaction count + pipeline.set(&tx_count_key, current_chain_tx_count); + + // Update health data only if it exists + if let Some(ref health_json) = health_update { + let health_key = self.eoa_health_key_name(); + pipeline.set(&health_key, health_json); + } + }) + .await + } + + /// Add a gas bump attempt (new hash) to submitted transactions + pub async fn add_gas_bump_attempt( + &self, + submitted_transaction: &SubmittedTransactionDehydrated, + signed_transaction: Signed, + ) -> Result<(), TransactionStoreError> { + let new_hash = signed_transaction.hash().to_string(); + + // Create new attempt + let new_attempt = TransactionAttempt { + transaction_id: submitted_transaction.transaction_id.clone(), + details: signed_transaction, + sent_at: chrono::Utc::now().timestamp_millis().max(0) as u64, + attempt_number: 0, // Will be set correctly when reading all attempts + }; + + // Serialize the new attempt + let attempt_json = serde_json::to_string(&new_attempt)?; + + // Get key names + let attempts_list_key = + self.transaction_attempts_list_name(&submitted_transaction.transaction_id); + let submitted_key = self.submitted_transactions_zset_name(); + + let hash_to_id_key = self.transaction_hash_to_id_key_name(&new_hash); + + let (submitted_transaction_string, nonce) = + submitted_transaction.to_redis_string_with_nonce(); + + // Now perform the atomic update + self.with_lock_check(|pipeline| { + // Add new hash:id to submitted (keeping old ones) + pipeline.zadd(&submitted_key, &submitted_transaction_string, nonce); + + // Still maintain separate hash-to-ID mapping for backward compatibility + pipeline.set(&hash_to_id_key, &submitted_transaction.transaction_id); + + // Simply push the new attempt to the attempts list + pipeline.lpush(&attempts_list_key, &attempt_json); + }) + .await + } + + /// Reset nonces to specified value + /// + /// This is called when we have too many recycled nonces and detect something wrong + /// We want to start fresh, with the chain nonce as the new optimistic nonce + pub async fn reset_nonces( + &self, + current_chain_tx_count: u64, + ) -> Result<(), TransactionStoreError> { + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + let current_health = self.get_eoa_health().await?; + + // Prepare health update if health data exists + let health_update = if let Some(mut health) = current_health { + health.nonce_resets.push(now); + Some(serde_json::to_string(&health)?) + } else { + None + }; + + self.with_lock_check(|pipeline| { + let optimistic_key = self.optimistic_transaction_count_key_name(); + let cached_nonce_key = self.last_transaction_count_key_name(); + let recycled_key = self.recycled_nonces_zset_name(); + + // Update health data only if it exists + if let Some(ref health_json) = health_update { + let health_key = self.eoa_health_key_name(); + pipeline.set(&health_key, health_json); + } + + // Reset the optimistic nonce + pipeline.set(&optimistic_key, current_chain_tx_count); + + // Reset the cached nonce + pipeline.set(&cached_nonce_key, current_chain_tx_count); + + // Reset the recycled nonces + pipeline.del(recycled_key); + }) + .await + } + + /// Fail a transaction that's in the pending state (remove from pending and fail) + /// This is used for deterministic failures during preparation that should not retry + pub async fn fail_pending_transaction( + &self, + pending_transaction: &PendingTransaction, + error: EoaExecutorWorkerError, + webhook_queue: Arc>, + ) -> Result<(), TransactionStoreError> { + self.with_lock_check(|pipeline| { + let pending_key = self.pending_transactions_zset_name(); + let tx_data_key = self.transaction_data_key_name(&pending_transaction.transaction_id); + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // Remove from pending state + pipeline.zrem(&pending_key, &pending_transaction.transaction_id); + + // Update transaction data with failure + pipeline.hset(&tx_data_key, "completed_at", now); + pipeline.hset(&tx_data_key, "failure_reason", error.to_string()); + pipeline.hset(&tx_data_key, "status", "failed"); + + let event = EoaExecutorEvent { + transaction_id: pending_transaction.transaction_id.clone(), + }; + + let fail_envelope = event.transaction_failed_envelope(error.clone(), 1); + + if !pending_transaction.user_request.webhook_options.is_empty() { + let mut tx_context = webhook_queue.transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + fail_envelope, + pending_transaction.user_request.webhook_options.clone(), + &mut tx_context, + webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for fail: {}", e); + } + } + }) + .await + } + + pub async fn clean_submitted_transactions( + &self, + confirmed_transactions: &[ConfirmedTransaction], + last_confirmed_nonce: u64, + webhook_queue: Arc>, + ) -> Result { + self.execute_with_watch_and_retry(&CleanSubmittedTransactions { + confirmed_transactions, + last_confirmed_nonce, + keys: &self.keys, + webhook_queue, + }) + .await + } + + /// Process borrowed transactions with given submission results + /// This method moves transactions from borrowed state to submitted/pending/failed states + /// based on the submission results, and queues appropriate webhook events + pub async fn process_borrowed_transactions( + &self, + results: Vec, + webhook_queue: Arc>, + ) -> Result { + self.execute_with_watch_and_retry(&ProcessBorrowedTransactions { + results, + keys: &self.keys, + webhook_queue, + }) + .await + } + + pub async fn clean_and_get_recycled_nonces(&self) -> Result, TransactionStoreError> { + self.execute_with_watch_and_retry(&CleanAndGetRecycledNonces { keys: &self.keys }) + .await + } + + pub async fn process_noop_transactions( + &self, + noop_transactions: &[SubmittedNoopTransaction], + ) -> Result<(), TransactionStoreError> { + self.with_lock_check(|pipeline| { + let recycled_key = self.recycled_nonces_zset_name(); + let submitted_key = self.submitted_transactions_zset_name(); + + pipeline.zrem( + &recycled_key, + noop_transactions + .iter() + .map(|tx| tx.nonce) + .collect::>(), + ); + + pipeline.zadd_multiple( + submitted_key, + &noop_transactions + .iter() + .map(|tx| { + let (tx_string, nonce) = tx.to_redis_string_with_nonce(); + (nonce, tx_string) + }) + .collect::>(), + ); + }) + .await + } +} diff --git a/executors/src/eoa/store/borrowed.rs b/executors/src/eoa/store/borrowed.rs new file mode 100644 index 0000000..f4c5e36 --- /dev/null +++ b/executors/src/eoa/store/borrowed.rs @@ -0,0 +1,238 @@ +use std::sync::Arc; + +use twmq::Queue; +use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; + +use crate::eoa::EoaExecutorStore; +use crate::eoa::{ + events::EoaExecutorEvent, + store::{ + EoaExecutorStoreKeys, TransactionStoreError, atomic::SafeRedisTransaction, + submitted::SubmittedTransaction, + }, + worker::error::EoaExecutorWorkerError, +}; +use crate::webhook::{WebhookJobHandler, queue_webhook_envelopes}; + +#[derive(Debug, Clone)] +pub enum SubmissionResultType { + Success, + Nack(EoaExecutorWorkerError), + Fail(EoaExecutorWorkerError), +} + +/// Result of a submission attempt +#[derive(Debug, Clone)] +pub struct SubmissionResult { + pub transaction: SubmittedTransaction, + pub result: SubmissionResultType, +} + +impl SubmissionResult { + pub fn transaction_id(&self) -> &str { + &self.transaction.transaction_id + } +} + +/// Batch operation to process borrowed transactions +pub struct ProcessBorrowedTransactions<'a> { + pub results: Vec, + pub keys: &'a EoaExecutorStoreKeys, + pub webhook_queue: Arc>, +} + +#[derive(Debug, Default)] +pub struct BorrowedProcessingReport { + pub total_processed: usize, + pub moved_to_submitted: usize, + pub moved_to_pending: usize, + pub failed_transactions: usize, + pub webhook_events_queued: usize, + pub ignored_not_in_borrowed: usize, +} + +impl SafeRedisTransaction for ProcessBorrowedTransactions<'_> { + type ValidationData = Vec; + type OperationResult = BorrowedProcessingReport; + + fn name(&self) -> &str { + "process borrowed transactions" + } + + fn watch_keys(&self) -> Vec { + vec![self.keys.borrowed_transactions_hashmap_name()] + } + + async fn validation( + &self, + conn: &mut ConnectionManager, + _store: &EoaExecutorStore, + ) -> Result { + // Get all borrowed transaction IDs + let borrowed_transaction_ids: Vec = conn + .hkeys(self.keys.borrowed_transactions_hashmap_name()) + .await?; + + // Filter submission results to only include those that exist in borrowed state + let mut valid_results = Vec::new(); + for result in &self.results { + let transaction_id = result.transaction_id(); + if borrowed_transaction_ids.contains(&transaction_id.to_string()) { + valid_results.push(result.clone()); + } else { + tracing::warn!( + transaction_id = %transaction_id, + nonce = %result.transaction.nonce, + "Submission result not found in borrowed state, ignoring" + ); + } + } + + Ok(valid_results) + } + + fn operation( + &self, + pipeline: &mut Pipeline, + validation_data: Self::ValidationData, + ) -> Self::OperationResult { + let valid_results = validation_data; + let mut report = BorrowedProcessingReport::default(); + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + for result in &valid_results { + let transaction_id = result.transaction_id(); + let nonce = result.transaction.nonce; + + // Remove from borrowed hashmap + pipeline.hdel( + self.keys.borrowed_transactions_hashmap_name(), + transaction_id, + ); + + // Add attempt to attempts list (using transaction hash as attempt data) + // let attempt_json = serde_json::to_string(&result.transaction.hash).unwrap(); + // todo figure out what to do about attempts + // pipeline.lpush(&attempts_key, &attempt_json); + + match &result.result { + SubmissionResultType::Success => { + // Add to submitted zset + let (submitted_tx_redis_string, nonce) = + result.transaction.to_redis_string_with_nonce(); + pipeline.zadd( + self.keys.submitted_transactions_zset_name(), + &submitted_tx_redis_string, + nonce, + ); + + // Update hash-to-ID mapping + let hash_to_id_key = self + .keys + .transaction_hash_to_id_key_name(&result.transaction.hash); + + pipeline.set(&hash_to_id_key, transaction_id); + + // Update transaction data status + let tx_data_key = self.keys.transaction_data_key_name(transaction_id); + pipeline.hset(&tx_data_key, "status", "submitted"); + + // Queue webhook event using user_request from SubmissionResult + let event = EoaExecutorEvent { + transaction_id: transaction_id.to_string(), + }; + + let envelope = + event.send_attempt_success_envelope(result.transaction.data.clone()); + if !result.transaction.user_request.webhook_options.is_empty() { + let mut tx_context = self + .webhook_queue + .transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + envelope, + result.transaction.user_request.webhook_options.clone(), + &mut tx_context, + self.webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for success: {}", e); + } else { + report.webhook_events_queued += 1; + } + } + + report.moved_to_submitted += 1; + } + SubmissionResultType::Nack(err) => { + // Add back to pending + pipeline.zadd( + self.keys.pending_transactions_zset_name(), + transaction_id, + now, + ); + + // Update transaction data status + let tx_data_key = self.keys.transaction_data_key_name(transaction_id); + pipeline.hset(&tx_data_key, "status", "pending"); + + // Queue webhook event using user_request from SubmissionResult + let event = EoaExecutorEvent { + transaction_id: transaction_id.to_string(), + }; + let envelope = event.send_attempt_nack_envelope(nonce, err.clone(), 1); + + if !result.transaction.user_request.webhook_options.is_empty() { + let mut tx_context = self + .webhook_queue + .transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + envelope, + result.transaction.user_request.webhook_options.clone(), + &mut tx_context, + self.webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for nack: {}", e); + } else { + report.webhook_events_queued += 1; + } + } + + report.moved_to_pending += 1; + } + SubmissionResultType::Fail(err) => { + // Mark as failed + let tx_data_key = self.keys.transaction_data_key_name(transaction_id); + pipeline.hset(&tx_data_key, "status", "failed"); + pipeline.hset(&tx_data_key, "completed_at", now); + pipeline.hset(&tx_data_key, "failure_reason", err.to_string()); + + // Queue webhook event using user_request from SubmissionResult + let event = EoaExecutorEvent { + transaction_id: transaction_id.to_string(), + }; + let envelope = event.transaction_failed_envelope(err.clone(), 1); + if !result.transaction.user_request.webhook_options.is_empty() { + let mut tx_context = self + .webhook_queue + .transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + envelope, + result.transaction.user_request.webhook_options.clone(), + &mut tx_context, + self.webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for fail: {}", e); + } else { + report.webhook_events_queued += 1; + } + } + + report.failed_transactions += 1; + } + } + } + + report.total_processed = valid_results.len(); + report.ignored_not_in_borrowed = self.results.len() - valid_results.len(); + report + } +} diff --git a/executors/src/eoa/store/error.rs b/executors/src/eoa/store/error.rs new file mode 100644 index 0000000..275541f --- /dev/null +++ b/executors/src/eoa/store/error.rs @@ -0,0 +1,22 @@ +use crate::eoa::EoaExecutorStore; +use crate::eoa::store::TransactionStoreError; +use crate::eoa::store::atomic::AtomicEoaExecutorStore; + +impl AtomicEoaExecutorStore { + pub fn eoa_lock_lost_error(&self) -> TransactionStoreError { + TransactionStoreError::LockLost { + eoa: self.eoa(), + chain_id: self.chain_id(), + worker_id: self.worker_id().to_string(), + } + } +} + +impl EoaExecutorStore { + pub fn nonce_sync_required_error(&self) -> TransactionStoreError { + TransactionStoreError::NonceSyncRequired { + eoa: self.eoa, + chain_id: self.chain_id, + } + } +} diff --git a/executors/src/eoa/store/hydrate.rs b/executors/src/eoa/store/hydrate.rs new file mode 100644 index 0000000..74f113a --- /dev/null +++ b/executors/src/eoa/store/hydrate.rs @@ -0,0 +1,157 @@ +use std::collections::{HashMap, HashSet, VecDeque}; + +use crate::eoa::{ + EoaExecutorStore, EoaTransactionRequest, + store::{ + BorrowedTransaction, BorrowedTransactionData, NO_OP_TRANSACTION_ID, + SubmittedNoopTransaction, SubmittedTransaction, SubmittedTransactionHydrated, + TransactionStoreError, submitted::SubmittedTransactionDehydrated, + }, +}; + +pub trait Dehydrated { + type Hydrated; + + fn transaction_id(&self) -> &str; + + fn hydrate(self, required_data: R) -> Self::Hydrated; +} + +#[derive(Debug, Clone)] +pub enum SubmittedTransactionHydrator { + Noop, + Real(EoaTransactionRequest), +} + +impl Dehydrated for SubmittedTransactionDehydrated { + type Hydrated = SubmittedTransactionHydrated; + + fn transaction_id(&self) -> &str { + &self.transaction_id + } + + fn hydrate(self, required_data: SubmittedTransactionHydrator) -> SubmittedTransactionHydrated { + match required_data { + SubmittedTransactionHydrator::Noop => { + SubmittedTransactionHydrated::Noop(SubmittedNoopTransaction { + nonce: self.nonce, + hash: self.hash, + }) + } + SubmittedTransactionHydrator::Real(request) => { + SubmittedTransactionHydrated::Real(SubmittedTransaction { + data: self, + user_request: request, + }) + } + } + } +} + +impl Dehydrated for BorrowedTransactionData { + type Hydrated = BorrowedTransaction; + + fn transaction_id(&self) -> &str { + &self.transaction_id + } + + fn hydrate(self, required_data: EoaTransactionRequest) -> BorrowedTransaction { + BorrowedTransaction { + data: self, + user_request: required_data, + } + } +} + +impl EoaExecutorStore { + pub async fn hydrate_all( + &self, + dehydrated: Vec, + ) -> Result, TransactionStoreError> + where + D: Dehydrated, + { + let mut pipe = twmq::redis::pipe(); + + for d in &dehydrated { + pipe.hget( + self.keys.transaction_data_key_name(d.transaction_id()), + "user_request", + ); + } + + let results: Vec = pipe.query_async(&mut self.redis.clone()).await?; + + let mut hydrated = Vec::with_capacity(dehydrated.len()); + for (d, r) in dehydrated.into_iter().zip(results.iter()) { + hydrated.push(d.hydrate(serde_json::from_str::(r)?)); + } + + Ok(hydrated) + } + + pub async fn hydrate(&self, dehydrated: D) -> Result + where + D: Dehydrated, + { + let mut pipe = twmq::redis::pipe(); + pipe.hget( + self.keys + .transaction_data_key_name(dehydrated.transaction_id()), + "user_request", + ); + let result: String = pipe.query_async(&mut self.redis.clone()).await?; + Ok(dehydrated.hydrate(serde_json::from_str::(&result)?)) + } + + pub async fn hydrate_all_submitted( + &self, + dehydrated: Vec, + ) -> Result, TransactionStoreError> + where + D: Dehydrated, + { + let mut pipe = twmq::redis::pipe(); + for d in &dehydrated { + if d.transaction_id() == NO_OP_TRANSACTION_ID { + continue; + } + + pipe.hget( + self.keys.transaction_data_key_name(d.transaction_id()), + "user_request", + ); + } + + let results: Vec = pipe.query_async(&mut self.redis.clone()).await?; + + let id_to_eoa_request = results + .into_iter() + .map(|r| { + let request = serde_json::from_str::(&r)?; + Ok((request.transaction_id.clone(), request)) + }) + .collect::, TransactionStoreError>>()?; + + let mut hydrated = Vec::with_capacity(dehydrated.len()); + + for d in dehydrated { + let id = d.transaction_id(); + if id == NO_OP_TRANSACTION_ID { + hydrated.push(d.hydrate(SubmittedTransactionHydrator::Noop)); + continue; + } + + let request = + id_to_eoa_request + .get(id) + .ok_or(TransactionStoreError::TransactionNotFound { + transaction_id: id.to_string(), + })?; + + hydrated.push(d.hydrate(SubmittedTransactionHydrator::Real(request.clone()))); + } + + Ok(hydrated) + } +} diff --git a/executors/src/eoa/store/mod.rs b/executors/src/eoa/store/mod.rs new file mode 100644 index 0000000..1bfb39b --- /dev/null +++ b/executors/src/eoa/store/mod.rs @@ -0,0 +1,801 @@ +use alloy::consensus::{Signed, Transaction, TypedTransaction}; +use alloy::network::AnyTransactionReceipt; +use alloy::primitives::{Address, Bytes, U256}; +use chrono; +use engine_core::chain::RpcCredentials; +use engine_core::credentials::SigningCredential; +use engine_core::execution_options::WebhookOptions; +use engine_core::transaction::TransactionTypeData; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::ops::Deref; +use twmq::redis::{AsyncCommands, aio::ConnectionManager}; + +mod atomic; +mod borrowed; +mod pending; +mod submitted; + +pub mod hydrate; + +pub mod error; +pub use atomic::AtomicEoaExecutorStore; +pub use borrowed::{BorrowedProcessingReport, SubmissionResult, SubmissionResultType}; +pub use submitted::{ + CleanupReport, SubmittedNoopTransaction, SubmittedTransaction, SubmittedTransactionDehydrated, + SubmittedTransactionHydrated, SubmittedTransactionStringWithNonce, +}; + +pub const NO_OP_TRANSACTION_ID: &str = "noop"; + +#[derive(Debug, Clone)] +pub struct ReplacedTransaction { + pub hash: String, + pub transaction_id: String, +} + +#[derive(Debug, Clone)] +pub struct ConfirmedTransaction { + pub hash: String, + pub transaction_id: String, + pub receipt: alloy::rpc::types::TransactionReceipt, + pub receipt_serialized: String, +} + +/// (transaction_id, queued_at) +type PendingTransactionStringWithQueuedAt = (String, u64); + +#[derive(Debug, Clone)] +pub struct PendingTransaction { + pub transaction_id: String, + pub queued_at: u64, + pub user_request: EoaTransactionRequest, +} + +/// The actual user request data +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct EoaTransactionRequest { + pub transaction_id: String, + pub chain_id: u64, + + pub from: Address, + pub to: Option
, + pub value: U256, + pub data: Bytes, + + #[serde(alias = "gas")] + pub gas_limit: Option, + + #[serde(default)] + pub webhook_options: Vec, + + pub signing_credential: SigningCredential, + pub rpc_credentials: RpcCredentials, + + #[serde(flatten)] + pub transaction_type_data: Option, +} + +/// Active attempt for a transaction (full alloy transaction + metadata) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TransactionAttempt { + pub transaction_id: String, + pub details: Signed, + pub sent_at: u64, // Unix timestamp in milliseconds + pub attempt_number: u32, +} + +/// Transaction data for a transaction_id +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TransactionData { + pub transaction_id: String, + pub user_request: EoaTransactionRequest, + pub receipt: Option, + pub attempts: Vec, + pub created_at: u64, // Unix timestamp in milliseconds +} + +/// Transaction store focused on transaction_id operations and nonce indexing +pub struct EoaExecutorStore { + pub redis: ConnectionManager, + pub keys: EoaExecutorStoreKeys, +} + +pub struct EoaExecutorStoreKeys { + pub eoa: Address, + pub chain_id: u64, + pub namespace: Option, +} + +impl EoaExecutorStoreKeys { + pub fn new(eoa: Address, chain_id: u64, namespace: Option) -> Self { + Self { + eoa, + chain_id, + namespace, + } + } + + /// Lock key name for EOA processing + pub fn eoa_lock_key_name(&self) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:lock:{}:{}", self.chain_id, self.eoa), + None => format!("eoa_executor:lock:{}:{}", self.chain_id, self.eoa), + } + } + + /// Name of the key for the transaction data + /// + /// Transaction data is stored as a Redis HSET with the following fields: + /// - "user_request": JSON string containing EoaTransactionRequest + /// - "receipt": JSON string containing AnyTransactionReceipt (optional) + /// - "status": String status ("confirmed", "failed", etc.) + /// - "completed_at": String Unix timestamp (optional) + /// - "created_at": String Unix timestamp (optional) + /// - "failure_reason": String failure reason (optional) + pub fn transaction_data_key_name(&self, transaction_id: &str) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:tx_data:{transaction_id}"), + None => format!("eoa_executor:_tx_data:{transaction_id}"), + } + } + + /// Name of the list for transaction attempts + /// + /// Attempts are stored as a separate Redis LIST where each element is a JSON blob + /// of a TransactionAttempt. This allows efficient append operations. + pub fn transaction_attempts_list_name(&self, transaction_id: &str) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:tx_attempts:{transaction_id}"), + None => format!("eoa_executor:tx_attempts:{transaction_id}"), + } + } + + /// Name of the zset for pending transactions + /// + /// zset contains the `transaction_id` scored by the queued_at timestamp (unix timestamp in milliseconds) + pub fn pending_transactions_zset_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:pending_txs:{}:{}", + self.chain_id, self.eoa + ), + None => format!("eoa_executor:pending_txs:{}:{}", self.chain_id, self.eoa), + } + } + + /// Name of the zset for submitted transactions. nonce -> hash:id + /// + /// Same transaction might appear multiple times in the zset with different nonces/gas prices (and thus different hashes) + pub fn submitted_transactions_zset_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:submitted_txs:{}:{}", + self.chain_id, self.eoa + ), + None => format!("eoa_executor:submitted_txs:{}:{}", self.chain_id, self.eoa), + } + } + + /// Name of the key that maps transaction hash to transaction id + pub fn transaction_hash_to_id_key_name(&self, hash: &str) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:tx_hash_to_id:{hash}"), + None => format!("eoa_executor:tx_hash_to_id:{hash}"), + } + } + + /// Name of the hashmap that maps `transaction_id` -> `BorrowedTransactionData` + /// + /// This is used for crash recovery. Before submitting a transaction, we atomically move from pending to this borrowed hashmap. + /// + /// On worker recovery, if any borrowed transactions are found, we rebroadcast them and move back to pending or submitted + /// + /// If there's no crash, happy path moves borrowed transactions back to pending or submitted + pub fn borrowed_transactions_hashmap_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:borrowed_txs:{}:{}", + self.chain_id, self.eoa + ), + None => format!("eoa_executor:borrowed_txs:{}:{}", self.chain_id, self.eoa), + } + } + + /// Name of the set that contains recycled nonces. + /// + /// If a transaction was submitted but failed (ie, we know with certainty it didn't enter the mempool), + /// + /// we add the nonce to this set. + /// + /// These nonces are used with priority, before any other nonces. + pub fn recycled_nonces_zset_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:recycled_nonces:{}:{}", + self.chain_id, self.eoa + ), + None => format!( + "eoa_executor:recycled_nonces:{}:{}", + self.chain_id, self.eoa + ), + } + } + + /// Optimistic nonce key name. + /// + /// This is used for optimistic nonce tracking. + /// + /// We store the nonce of the last successfuly sent transaction for each EOA. + /// + /// We increment this nonce for each new transaction. + /// + /// !IMPORTANT! When sending a transaction, we use this nonce as the assigned nonce, NOT the incremented nonce. + pub fn optimistic_transaction_count_key_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:optimistic_nonce:{}:{}", + self.chain_id, self.eoa + ), + None => format!( + "eoa_executor:optimistic_nonce:{}:{}", + self.chain_id, self.eoa + ), + } + } + + /// Name of the key that contains the nonce of the last fetched ONCHAIN transaction count for each EOA. + /// + /// This is a cache for the actual transaction count, which is fetched from the RPC. + /// + /// The nonce for the NEXT transaction is the ONCHAIN transaction count (NOT + 1) + /// + /// Eg: transaction count is 0, so we use nonce 0 for sending the next transaction. Once successful, transaction count will be 1. + pub fn last_transaction_count_key_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:last_tx_nonce:{}:{}", + self.chain_id, self.eoa + ), + None => format!("eoa_executor:last_tx_nonce:{}:{}", self.chain_id, self.eoa), + } + } + + /// EOA health key name. + /// + /// EOA health stores: + /// - cached balance, the timestamp of the last balance fetch + /// - timestamp of the last successful transaction confirmation + /// - timestamp of the last 5 nonce resets + pub fn eoa_health_key_name(&self) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:health:{}:{}", self.chain_id, self.eoa), + None => format!("eoa_executor:health:{}:{}", self.chain_id, self.eoa), + } + } +} + +impl EoaExecutorStore { + pub fn new( + redis: ConnectionManager, + namespace: Option, + eoa: Address, + chain_id: u64, + ) -> Self { + Self { + redis, + keys: EoaExecutorStoreKeys { + eoa, + chain_id, + namespace, + }, + } + } +} + +impl std::ops::Deref for EoaExecutorStore { + type Target = EoaExecutorStoreKeys; + fn deref(&self) -> &Self::Target { + &self.keys + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EoaHealth { + pub balance: U256, + /// Update the balance threshold when we see out of funds errors + pub balance_threshold: U256, + pub balance_fetched_at: u64, + pub last_confirmation_at: u64, + pub last_nonce_movement_at: u64, // Track when nonce last moved for gas bump detection + pub nonce_resets: Vec, // Last 5 reset timestamps +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BorrowedTransactionData { + pub transaction_id: String, + pub signed_transaction: Signed, + pub queued_at: u64, + pub hash: String, + pub borrowed_at: u64, +} + +#[derive(Debug, Clone)] +pub struct BorrowedTransaction { + pub data: BorrowedTransactionData, + pub user_request: EoaTransactionRequest, +} + +impl Deref for BorrowedTransaction { + type Target = BorrowedTransactionData; + fn deref(&self) -> &Self::Target { + &self.data + } +} + +impl From for SubmittedTransactionDehydrated { + fn from(data: BorrowedTransactionData) -> Self { + SubmittedTransactionDehydrated { + nonce: data.signed_transaction.nonce(), + hash: data.signed_transaction.hash().to_string(), + transaction_id: data.transaction_id.clone(), + queued_at: data.queued_at, + } + } +} + +impl From for SubmittedTransaction { + fn from(data: BorrowedTransaction) -> Self { + SubmittedTransaction { + data: data.data.into(), + user_request: data.user_request.clone(), + } + } +} + +/// Type of nonce allocation for transaction processing +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NonceType { + /// Nonce was recycled from a previously failed transaction + Recycled(u64), + /// Nonce was incremented from the current optimistic counter + Incremented(u64), +} + +impl NonceType { + /// Get the nonce value regardless of type + pub fn nonce(&self) -> u64 { + match self { + NonceType::Recycled(nonce) => *nonce, + NonceType::Incremented(nonce) => *nonce, + } + } + + /// Check if this is a recycled nonce + pub fn is_recycled(&self) -> bool { + matches!(self, NonceType::Recycled(_)) + } + + /// Check if this is an incremented nonce + pub fn is_incremented(&self) -> bool { + matches!(self, NonceType::Incremented(_)) + } +} + +impl EoaExecutorStore { + /// Aggressively acquire EOA lock, forcefully taking over from stalled workers + /// + /// Creates an AtomicEoaExecutorStore that owns the lock. + pub async fn acquire_eoa_lock_aggressively( + self, + worker_id: &str, + ) -> Result { + let lock_key = self.eoa_lock_key_name(); + let mut conn = self.redis.clone(); + + // First try normal acquisition + let acquired: bool = conn.set_nx(&lock_key, worker_id).await?; + if acquired { + return Ok(AtomicEoaExecutorStore { + store: self, + worker_id: worker_id.to_string(), + }); + } + // Lock exists, forcefully take it over + tracing::warn!( + eoa = %self.eoa, + chain_id = %self.chain_id, + worker_id = %worker_id, + "Forcefully taking over EOA lock from stalled worker" + ); + // Force set - no expiry, only released by explicit takeover + let _: () = conn.set(&lock_key, worker_id).await?; + Ok(AtomicEoaExecutorStore { + store: self, + worker_id: worker_id.to_string(), + }) + } + + /// Peek all borrowed transactions without removing them + pub async fn peek_borrowed_transactions( + &self, + ) -> Result, TransactionStoreError> { + let borrowed_key = self.borrowed_transactions_hashmap_name(); + let mut conn = self.redis.clone(); + + let borrowed_map: HashMap = conn.hgetall(&borrowed_key).await?; + let mut result = Vec::new(); + + for (_transaction_id, transaction_json) in borrowed_map { + let borrowed_data: BorrowedTransactionData = serde_json::from_str(&transaction_json)?; + result.push(borrowed_data); + } + + Ok(result) + } + + /// Get all hashes below a certain nonce from submitted transactions + /// Returns (nonce, hash, transaction_id) tuples + pub async fn get_submitted_transactions_below_chain_transaction_count( + &self, + count: u64, + ) -> Result, TransactionStoreError> { + if count == 0 { + return Ok(Vec::new()); + } + + let submitted_key = self.submitted_transactions_zset_name(); + let mut conn = self.redis.clone(); + + // Get all entries with nonce < transaction count + let results: Vec = conn + .zrangebyscore_withscores(&submitted_key, 0, count - 1) + .await?; + + let submitted_txs: Vec = + SubmittedTransactionDehydrated::from_redis_strings(&results); + + Ok(submitted_txs) + } + + /// Get all transaction IDs for a specific nonce + pub async fn get_submitted_transactions_for_nonce( + &self, + nonce: u64, + ) -> Result, TransactionStoreError> { + let submitted_key = self.submitted_transactions_zset_name(); + let mut conn = self.redis.clone(); + + let results: Vec = conn + .zrangebyscore_withscores(&submitted_key, nonce, nonce) + .await?; + + let submitted_txs: Vec = + SubmittedTransactionDehydrated::from_redis_strings(&results); + + Ok(submitted_txs) + } + + /// Check EOA health (balance, etc.) + pub async fn get_eoa_health(&self) -> Result, TransactionStoreError> { + let mut conn = self.redis.clone(); + + let health_json: Option = conn.get(self.eoa_health_key_name()).await?; + if let Some(json) = health_json { + let health: EoaHealth = serde_json::from_str(&json)?; + Ok(Some(health)) + } else { + Ok(None) + } + } + + /// Peek recycled nonces without removing them + pub async fn peek_recycled_nonces(&self) -> Result, TransactionStoreError> { + let recycled_key = self.recycled_nonces_zset_name(); + let mut conn = self.redis.clone(); + + let nonces: Vec = conn.zrange(&recycled_key, 0, -1).await?; + Ok(nonces) + } + + /// Peek at pending transactions without removing them (safe for planning) + pub async fn peek_pending_transactions( + &self, + limit: u64, + ) -> Result, TransactionStoreError> { + let pending_key = self.pending_transactions_zset_name(); + let mut conn = self.redis.clone(); + + // Use ZRANGE to peek without removing + let transaction_ids: Vec = conn + .zrange_withscores(&pending_key, 0, (limit - 1) as isize) + .await?; + + let mut pipe = twmq::redis::pipe(); + + for (transaction_id, _) in &transaction_ids { + let tx_data_key = self.transaction_data_key_name(transaction_id); + pipe.hget(&tx_data_key, "user_request"); + } + + let user_requests: Vec = pipe.query_async(&mut conn).await?; + + let user_requests: Vec = user_requests + .into_iter() + .map(|user_request_json| serde_json::from_str(&user_request_json)) + .collect::, serde_json::Error>>()?; + + let pending_transactions: Vec = transaction_ids + .into_iter() + .zip(user_requests) + .map( + |((transaction_id, queued_at), user_request)| PendingTransaction { + transaction_id, + queued_at, + user_request, + }, + ) + .collect(); + + Ok(pending_transactions) + } + + /// Get inflight budget (how many new transactions can be sent) + pub async fn get_inflight_budget( + &self, + max_inflight: u64, + ) -> Result { + let optimistic_key = self.optimistic_transaction_count_key_name(); + let last_tx_count_key = self.last_transaction_count_key_name(); + let mut conn = self.redis.clone(); + + // Read both values atomically to avoid race conditions + let (optimistic_nonce, last_tx_count): (Option, Option) = twmq::redis::pipe() + .get(&optimistic_key) + .get(&last_tx_count_key) + .query_async(&mut conn) + .await?; + + let optimistic = match optimistic_nonce { + Some(nonce) => nonce, + None => return Err(self.nonce_sync_required_error()), + }; + let last_count = match last_tx_count { + Some(count) => count, + None => return Err(self.nonce_sync_required_error()), + }; + + let current_inflight = optimistic.saturating_sub(last_count); + let available_budget = max_inflight.saturating_sub(current_inflight); + + Ok(available_budget) + } + + /// Get current optimistic nonce (without incrementing) + pub async fn get_optimistic_transaction_count(&self) -> Result { + let optimistic_key = self.optimistic_transaction_count_key_name(); + let mut conn = self.redis.clone(); + + let current: Option = conn.get(&optimistic_key).await?; + match current { + Some(nonce) => Ok(nonce), + None => Err(self.nonce_sync_required_error()), + } + } + /// Get transaction ID for a given hash + pub async fn get_transaction_id_for_hash( + &self, + hash: &str, + ) -> Result, TransactionStoreError> { + let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); + let mut conn = self.redis.clone(); + + let transaction_id: Option = conn.get(&hash_to_id_key).await?; + Ok(transaction_id) + } + + /// Get transaction data by transaction ID + pub async fn get_transaction_data( + &self, + transaction_id: &str, + ) -> Result, TransactionStoreError> { + let tx_data_key = self.transaction_data_key_name(transaction_id); + let mut conn = self.redis.clone(); + + // Get the hash data (the transaction data is stored as a hash) + let hash_data: HashMap = conn.hgetall(&tx_data_key).await?; + + if hash_data.is_empty() { + return Ok(None); + } + + // Extract user_request from the hash data + let user_request_json = hash_data.get("user_request").ok_or_else(|| { + TransactionStoreError::TransactionNotFound { + transaction_id: transaction_id.to_string(), + } + })?; + + let user_request: EoaTransactionRequest = serde_json::from_str(user_request_json)?; + + // Extract receipt if present + let receipt = hash_data + .get("receipt") + .and_then(|receipt_str| serde_json::from_str(receipt_str).ok()); + + let created_at = hash_data.get("created_at").ok_or_else(|| { + TransactionStoreError::TransactionNotFound { + transaction_id: transaction_id.to_string(), + } + })?; + + // todo: in case of non-existent created_at, we should return a default value + let created_at = + created_at + .parse::() + .map_err(|_| TransactionStoreError::TransactionNotFound { + transaction_id: transaction_id.to_string(), + })?; + + // Extract attempts from separate list + let attempts_key = self.transaction_attempts_list_name(transaction_id); + let attempts_json_list: Vec = conn.lrange(&attempts_key, 0, -1).await?; + let mut attempts = Vec::new(); + for attempt_json in attempts_json_list { + if let Ok(attempt) = serde_json::from_str::(&attempt_json) { + attempts.push(attempt); + } + } + + Ok(Some(TransactionData { + transaction_id: transaction_id.to_string(), + created_at, + user_request, + receipt, + attempts, + })) + } + + /// Get cached transaction count + pub async fn get_cached_transaction_count(&self) -> Result { + let tx_count_key = self.last_transaction_count_key_name(); + let mut conn = self.redis.clone(); + + let count: Option = conn.get(&tx_count_key).await?; + match count { + Some(count) => Ok(count), + None => Err(self.nonce_sync_required_error()), + } + } + + /// Add a transaction to the pending queue and store its data + /// This is called when a new transaction request comes in for an EOA + pub async fn add_transaction( + &self, + transaction_request: EoaTransactionRequest, + ) -> Result<(), TransactionStoreError> { + let transaction_id = &transaction_request.transaction_id; + + let tx_data_key = self.transaction_data_key_name(transaction_id); + let pending_key = self.pending_transactions_zset_name(); + + // Store transaction data as JSON in the user_request field of the hash + let user_request_json = serde_json::to_string(&transaction_request)?; + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + let mut conn = self.redis.clone(); + + // Use a pipeline to atomically store data and add to pending queue + let mut pipeline = twmq::redis::pipe(); + + // Store transaction data + pipeline.hset(&tx_data_key, "user_request", &user_request_json); + pipeline.hset(&tx_data_key, "status", "pending"); + pipeline.hset(&tx_data_key, "created_at", now); + + // Add to pending queue + pipeline.zadd(&pending_key, transaction_id, now); + + pipeline.query_async::<()>(&mut conn).await?; + + Ok(()) + } + + /// Get count of submitted transactions awaiting confirmation + pub async fn get_submitted_transactions_count(&self) -> Result { + let submitted_key = self.submitted_transactions_zset_name(); + let mut conn = self.redis.clone(); + + let count: u64 = conn.zcard(&submitted_key).await?; + Ok(count) + } + + /// Get the submitted transactions for the highest nonce value + /// + /// Internally submissions are stored in a zset by nonce -> hash:id + /// + /// This will return all hash:id pairs for the highest nonce + #[tracing::instrument(skip_all)] + pub async fn get_highest_submitted_nonce_tranasactions( + &self, + ) -> Result, TransactionStoreError> { + let submitted_key = self.submitted_transactions_zset_name(); + let mut conn = self.redis.clone(); + + let highest_nonce_txs: Vec = + conn.zrange_withscores(&submitted_key, -1, -1).await?; + + let submitted_txs: Vec = + SubmittedTransactionDehydrated::from_redis_strings(&highest_nonce_txs); + + Ok(submitted_txs) + } +} + +// Additional error types +#[derive(Debug, thiserror::Error, Serialize, Deserialize, Clone)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE", tag = "errorCode")] +pub enum TransactionStoreError { + #[error("Redis error: {message}")] + RedisError { message: String }, + + #[error("Serialization error: {message}")] + DeserError { message: String, text: String }, + + #[error("Transaction not found: {transaction_id}")] + TransactionNotFound { transaction_id: String }, + + #[error("Lost EOA lock: {eoa}:{chain_id} worker: {worker_id}")] + LockLost { + eoa: Address, + chain_id: u64, + worker_id: String, + }, + + #[error("Internal error - worker should quit: {message}")] + InternalError { message: String }, + + #[error("Transaction {transaction_id} not in borrowed state for nonce {nonce}")] + TransactionNotInBorrowedState { transaction_id: String, nonce: u64 }, + + #[error("Hash {hash} not found in submitted transactions")] + HashNotInSubmittedState { hash: String }, + + #[error("Transaction {transaction_id} has no hashes in submitted state")] + TransactionNotInSubmittedState { transaction_id: String }, + + #[error("Nonce {nonce} not available in recycled set")] + NonceNotInRecycledSet { nonce: u64 }, + + #[error("Transaction {transaction_id} not found in pending queue")] + TransactionNotInPendingQueue { transaction_id: String }, + + #[error("Optimistic nonce changed: expected {expected}, found {actual}")] + OptimisticNonceChanged { expected: u64, actual: u64 }, + + #[error("WATCH failed - state changed during operation")] + WatchFailed, + + #[error( + "Nonce synchronization required for {eoa}:{chain_id} - no cached transaction count available" + )] + NonceSyncRequired { eoa: Address, chain_id: u64 }, +} + +impl From for TransactionStoreError { + fn from(error: twmq::redis::RedisError) -> Self { + TransactionStoreError::RedisError { + message: error.to_string(), + } + } +} + +impl From for TransactionStoreError { + fn from(error: serde_json::Error) -> Self { + TransactionStoreError::DeserError { + message: error.to_string(), + text: error.to_string(), + } + } +} diff --git a/executors/src/eoa/store/pending.rs b/executors/src/eoa/store/pending.rs new file mode 100644 index 0000000..c1c5f67 --- /dev/null +++ b/executors/src/eoa/store/pending.rs @@ -0,0 +1,259 @@ +use std::collections::HashSet; + +use alloy::{consensus::Transaction, primitives::Address}; +use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; + +use crate::eoa::{ + EoaExecutorStore, + store::{ + BorrowedTransactionData, EoaExecutorStoreKeys, TransactionStoreError, + atomic::SafeRedisTransaction, + }, +}; + +/// Atomic operation to move pending transactions to borrowed state using incremented nonces +/// +/// This operation validates that: +/// 1. The nonces in the vector are sequential with no gaps +/// 2. The lowest nonce matches the current optimistic transaction count +/// 3. All transactions exist in the pending queue +/// +/// Then atomically: +/// 1. Removes transactions from pending queue +/// 2. Adds transactions to borrowed state +/// 3. Updates optimistic transaction count to highest nonce + 1 +pub struct MovePendingToBorrowedWithIncrementedNonces<'a> { + pub transactions: &'a [BorrowedTransactionData], + pub keys: &'a EoaExecutorStoreKeys, + pub eoa: Address, + pub chain_id: u64, +} + +impl SafeRedisTransaction for MovePendingToBorrowedWithIncrementedNonces<'_> { + type ValidationData = Vec; // serialized borrowed transactions + type OperationResult = usize; // number of transactions processed + + fn name(&self) -> &str { + "pending->borrowed with incremented nonces" + } + + fn watch_keys(&self) -> Vec { + vec![ + self.keys.optimistic_transaction_count_key_name(), + self.keys.borrowed_transactions_hashmap_name(), + ] + } + + async fn validation( + &self, + conn: &mut ConnectionManager, + _store: &EoaExecutorStore, + ) -> Result { + if self.transactions.is_empty() { + return Err(TransactionStoreError::InternalError { + message: "Cannot process empty transaction list".to_string(), + }); + } + + // Get current optimistic nonce + let current_optimistic: Option = conn + .get(self.keys.optimistic_transaction_count_key_name()) + .await?; + let current_nonce = current_optimistic.ok_or(TransactionStoreError::NonceSyncRequired { + eoa: self.eoa, + chain_id: self.chain_id, + })?; + + // Extract and validate nonces + let mut nonces: Vec = self + .transactions + .iter() + .map(|tx| tx.signed_transaction.nonce()) + .collect(); + nonces.sort(); + + // Check that nonces are sequential with no gaps + for (i, &nonce) in nonces.iter().enumerate() { + let expected_nonce = current_nonce + i as u64; + if nonce != expected_nonce { + return Err(TransactionStoreError::InternalError { + message: format!( + "Non-sequential nonces detected: expected {}, found {} at position {}", + expected_nonce, nonce, i + ), + }); + } + } + + // Verify all transactions exist in pending queue using batched ZSCORE calls + if !self.transactions.is_empty() { + let mut pipe = twmq::redis::pipe(); + for tx in self.transactions { + pipe.zscore( + self.keys.pending_transactions_zset_name(), + &tx.transaction_id, + ); + } + let scores: Vec> = pipe.query_async(conn).await?; + + for (tx, score) in self.transactions.iter().zip(scores.iter()) { + if score.is_none() { + return Err(TransactionStoreError::TransactionNotInPendingQueue { + transaction_id: tx.transaction_id.clone(), + }); + } + } + } + + // Pre-serialize all borrowed transaction data + let mut serialized_transactions = Vec::with_capacity(self.transactions.len()); + for tx in self.transactions { + let borrowed_json = + serde_json::to_string(tx).map_err(|e| TransactionStoreError::InternalError { + message: format!("Failed to serialize borrowed transaction: {}", e), + })?; + serialized_transactions.push(borrowed_json); + } + + Ok(serialized_transactions) + } + + fn operation( + &self, + pipeline: &mut Pipeline, + serialized_transactions: Self::ValidationData, + ) -> Self::OperationResult { + let borrowed_key = self.keys.borrowed_transactions_hashmap_name(); + let pending_key = self.keys.pending_transactions_zset_name(); + let optimistic_key = self.keys.optimistic_transaction_count_key_name(); + + for (tx, borrowed_json) in self.transactions.iter().zip(serialized_transactions.iter()) { + // Remove from pending queue + pipeline.zrem(&pending_key, &tx.transaction_id); + + // Add to borrowed state + pipeline.hset(&borrowed_key, &tx.transaction_id, borrowed_json); + } + + // Update optimistic tx count to highest nonce + 1 + if let Some(last_tx) = self.transactions.last() { + let new_optimistic_tx_count = last_tx.signed_transaction.nonce() + 1; + pipeline.set(&optimistic_key, new_optimistic_tx_count); + } + + self.transactions.len() + } +} + +/// Atomic operation to move pending transactions to borrowed state using recycled nonces +/// +/// This operation validates that: +/// 1. All nonces exist in the recycled nonces set +/// 2. All transactions exist in the pending queue +/// +/// Then atomically: +/// 1. Removes nonces from recycled set +/// 2. Removes transactions from pending queue +/// 3. Adds transactions to borrowed state +pub struct MovePendingToBorrowedWithRecycledNonces<'a> { + pub transactions: &'a [BorrowedTransactionData], + pub keys: &'a EoaExecutorStoreKeys, +} + +impl SafeRedisTransaction for MovePendingToBorrowedWithRecycledNonces<'_> { + type ValidationData = Vec; // serialized borrowed transactions + type OperationResult = usize; // number of transactions processed + + fn name(&self) -> &str { + "pending->borrowed with recycled nonces" + } + + fn watch_keys(&self) -> Vec { + vec![ + self.keys.recycled_nonces_zset_name(), + self.keys.borrowed_transactions_hashmap_name(), + ] + } + + async fn validation( + &self, + conn: &mut ConnectionManager, + _store: &EoaExecutorStore, + ) -> Result { + if self.transactions.is_empty() { + return Err(TransactionStoreError::InternalError { + message: "Cannot process empty transaction list".to_string(), + }); + } + + // Get all recycled nonces + let recycled_nonces: HashSet = conn + .zrange(self.keys.recycled_nonces_zset_name(), 0, -1) + .await?; + + // Verify all nonces are in recycled set + for tx in self.transactions { + let nonce = tx.signed_transaction.nonce(); + if !recycled_nonces.contains(&nonce) { + return Err(TransactionStoreError::NonceNotInRecycledSet { nonce }); + } + } + + // Verify all transactions exist in pending queue using batched ZSCORE calls + if !self.transactions.is_empty() { + let mut pipe = twmq::redis::pipe(); + for tx in self.transactions { + pipe.zscore( + self.keys.pending_transactions_zset_name(), + &tx.transaction_id, + ); + } + let scores: Vec> = pipe.query_async(conn).await?; + + for (tx, score) in self.transactions.iter().zip(scores.iter()) { + if score.is_none() { + return Err(TransactionStoreError::TransactionNotInPendingQueue { + transaction_id: tx.transaction_id.clone(), + }); + } + } + } + + // Pre-serialize all borrowed transaction data + let mut serialized_transactions = Vec::with_capacity(self.transactions.len()); + for tx in self.transactions { + let borrowed_json = + serde_json::to_string(tx).map_err(|e| TransactionStoreError::InternalError { + message: format!("Failed to serialize borrowed transaction: {}", e), + })?; + serialized_transactions.push(borrowed_json); + } + + Ok(serialized_transactions) + } + + fn operation( + &self, + pipeline: &mut Pipeline, + serialized_transactions: Self::ValidationData, + ) -> Self::OperationResult { + let recycled_key = self.keys.recycled_nonces_zset_name(); + let pending_key = self.keys.pending_transactions_zset_name(); + let borrowed_key = self.keys.borrowed_transactions_hashmap_name(); + + for (tx, borrowed_json) in self.transactions.iter().zip(serialized_transactions.iter()) { + let nonce = tx.signed_transaction.nonce(); + + // Remove nonce from recycled set + pipeline.zrem(&recycled_key, nonce); + + // Remove from pending queue + pipeline.zrem(&pending_key, &tx.transaction_id); + + // Add to borrowed state + pipeline.hset(&borrowed_key, &tx.transaction_id, borrowed_json); + } + + self.transactions.len() + } +} diff --git a/executors/src/eoa/store/submitted.rs b/executors/src/eoa/store/submitted.rs new file mode 100644 index 0000000..82d3d2b --- /dev/null +++ b/executors/src/eoa/store/submitted.rs @@ -0,0 +1,494 @@ +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + ops::Deref, + sync::Arc, +}; + +use serde::{Deserialize, Serialize}; +use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; + +use crate::{ + eoa::{ + EoaExecutorStore, EoaTransactionRequest, + events::EoaExecutorEvent, + store::{ + ConfirmedTransaction, EoaExecutorStoreKeys, NO_OP_TRANSACTION_ID, + TransactionStoreError, atomic::SafeRedisTransaction, + }, + }, + webhook::{WebhookJobHandler, queue_webhook_envelopes}, +}; + +#[derive(Debug, Clone)] +pub struct SubmittedTransaction { + pub data: SubmittedTransactionDehydrated, + pub user_request: EoaTransactionRequest, +} + +impl Deref for SubmittedTransaction { + type Target = SubmittedTransactionDehydrated; + fn deref(&self) -> &Self::Target { + &self.data + } +} + +#[derive(Debug, Clone)] +pub struct SubmittedNoopTransaction { + pub nonce: u64, + pub hash: String, +} + +pub type SubmittedTransactionStringWithNonce = (String, u64); + +impl SubmittedNoopTransaction { + pub fn to_redis_string_with_nonce(&self) -> SubmittedTransactionStringWithNonce { + ( + format!("{}:{}:0", self.hash, NO_OP_TRANSACTION_ID), + self.nonce, + ) + } +} + +#[derive(Debug, Clone)] +pub enum SubmittedTransactionHydrated { + Noop(SubmittedNoopTransaction), + Real(SubmittedTransaction), +} + +impl SubmittedTransactionHydrated { + pub fn hash(&self) -> &str { + match self { + SubmittedTransactionHydrated::Noop(tx) => &tx.hash, + SubmittedTransactionHydrated::Real(tx) => &tx.hash, + } + } + + pub fn nonce(&self) -> u64 { + match self { + SubmittedTransactionHydrated::Noop(tx) => tx.nonce, + SubmittedTransactionHydrated::Real(tx) => tx.nonce, + } + } + + pub fn transaction_id(&self) -> &str { + match self { + SubmittedTransactionHydrated::Noop(_) => NO_OP_TRANSACTION_ID, + SubmittedTransactionHydrated::Real(tx) => &tx.transaction_id, + } + } + + pub fn to_redis_string_with_nonce(&self) -> SubmittedTransactionStringWithNonce { + match self { + SubmittedTransactionHydrated::Noop(tx) => tx.to_redis_string_with_nonce(), + SubmittedTransactionHydrated::Real(tx) => tx.to_redis_string_with_nonce(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SubmittedTransactionDehydrated { + pub nonce: u64, + pub hash: String, + pub transaction_id: String, + pub queued_at: u64, +} + +impl SubmittedTransactionDehydrated { + pub fn from_redis_strings(redis_strings: &[SubmittedTransactionStringWithNonce]) -> Vec { + redis_strings + .iter() + .filter_map(|tx| { + let parts: Vec<&str> = tx.0.split(':').collect(); + if parts.len() == 3 { + if let Ok(queued_at) = parts[2].parse::() { + Some(SubmittedTransactionDehydrated { + hash: parts[0].to_string(), + transaction_id: parts[1].to_string(), + nonce: tx.1, + queued_at, + }) + } else { + tracing::error!("Invalid queued_at timestamp: {}", tx.0); + None + } + } else { + tracing::error!( + "Invalid transaction format, expected 3 parts separated by ':': {}", + tx.0 + ); + None + } + }) + .collect() + } + + /// Returns the string representation of the submitted transaction with the nonce + /// + /// This is used to add the transaction to the submitted state in Redis + /// + /// The format is: + /// + /// ```text + /// hash:transaction_id:queued_at + /// ``` + /// + /// The nonce is the value of the transaction in the submitted state, and is used as the score of the submitted zset + pub fn to_redis_string_with_nonce(&self) -> SubmittedTransactionStringWithNonce { + ( + format!("{}:{}:{}", self.hash, self.transaction_id, self.queued_at), + self.nonce, + ) + } +} + +pub struct CleanSubmittedTransactions<'a> { + pub last_confirmed_nonce: u64, + pub confirmed_transactions: &'a [ConfirmedTransaction], + pub keys: &'a EoaExecutorStoreKeys, + pub webhook_queue: Arc>, +} + +pub struct CleanAndGetRecycledNonces<'a> { + pub keys: &'a EoaExecutorStoreKeys, +} + +#[derive(Debug, Default)] +pub struct CleanupReport { + pub total_hashes_processed: usize, + pub unique_transaction_ids: usize, + pub noop_count: usize, + pub moved_to_success: usize, + pub moved_to_pending: usize, + + /// Any transaction ID values that have multiple nonces in the submitted state + pub cross_nonce_violations: Vec<(String, Vec)>, // (transaction_id, nonces) + + /// Any nonces that have multiple confirmations (very rare, indicates re-org) + pub per_nonce_violations: Vec<(u64, Vec)>, // (nonce, confirmed_hashes) + + /// Any nonces that have no confirmations (transactions we sent got replaced by a different one uknown to us) + pub nonces_without_receipts: Vec<(u64, Vec)>, // (nonce, hashes) +} + +/// This operation takes a list of confirmed transactions and the last confirmed nonce +/// +/// It will fetch all submitted transactions with a nonce less than or equal to the last confirmed nonce. +/// For each nonce: +/// - it will go through all the hashes for that nonce +/// - if the hash is in the confirmed transactions, it will be removed from submitted to success +/// - if the hash is not in the confirmed transactions, it will be removed from submitted to pending +/// +/// It will also deduplicate transactions by ID, so if any of the hashes for that ID are in the confirmed transactions, +/// this hash will not be moved back to pending. +/// +/// ***IMPORTANT***: This should not happen with different nonces. A transaction ID should only appear once in the submitted state. +/// Multiple submissions for the same transaction ID with different nonces can cause duplicate transactions +/// Multiple submissions for the same transaction ID with the same nonce is fine, because this indicated gas bumps. +impl SafeRedisTransaction for CleanSubmittedTransactions<'_> { + type ValidationData = Vec; + type OperationResult = CleanupReport; + + fn name(&self) -> &str { + "clean submitted transactions" + } + + fn watch_keys(&self) -> Vec { + vec![self.keys.submitted_transactions_zset_name()] + } + + async fn validation( + &self, + conn: &mut ConnectionManager, + store: &EoaExecutorStore, + ) -> Result { + let submitted_txs: Vec = conn + .zrangebyscore_withscores( + self.keys.submitted_transactions_zset_name(), + 0, + self.last_confirmed_nonce as isize, + ) + .await?; + + let submitted_txs = SubmittedTransactionDehydrated::from_redis_strings(&submitted_txs); + let hydrated = store.hydrate_all_submitted(submitted_txs).await?; + Ok(hydrated) + } + + fn operation( + &self, + pipeline: &mut Pipeline, + submitted_txs: Self::ValidationData, + ) -> Self::OperationResult { + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // Build confirmed lookups + let confirmed_hashes: HashSet<&str> = self + .confirmed_transactions + .iter() + .map(|tx| tx.hash.as_str()) + .collect(); + + let confirmed_ids: BTreeMap<&str, ConfirmedTransaction> = self + .confirmed_transactions + .iter() + .map(|tx| (tx.transaction_id.as_str(), tx.clone())) + .collect(); + + // Detect violations and get grouped data + let (_, _, mut report) = detect_violations(&submitted_txs, &confirmed_hashes); + + // Process every hash and track unique IDs + let mut processed_ids = HashSet::new(); + + let mut replaced_transactions = Vec::with_capacity(submitted_txs.len()); + + for tx in &submitted_txs { + // Clean up this hash from Redis (happens for ALL hashes) + let (submitted_tx_redis_string, _nonce) = tx.clone().to_redis_string_with_nonce(); + + pipeline.zrem( + self.keys.submitted_transactions_zset_name(), + &submitted_tx_redis_string, + ); + pipeline.del(self.keys.transaction_hash_to_id_key_name(tx.hash())); + + // Process each unique transaction_id once + if processed_ids.insert(tx.transaction_id()) { + match (tx.transaction_id(), confirmed_ids.get(tx.transaction_id())) { + // if the transaction id is noop, we don't do anything + (NO_OP_TRANSACTION_ID, _) => report.noop_count += 1, + + // in case of a valid ID, we check if it's in the confirmed transactions + // if it is confirmed, we succeed it and queue success jobs + (id, Some(confirmed_tx)) => { + let data_key_name = self.keys.transaction_data_key_name(id); + pipeline.hset(&data_key_name, "status", "confirmed"); + pipeline.hset(&data_key_name, "completed_at", now); + pipeline.hset( + &data_key_name, + "receipt", + confirmed_tx.receipt_serialized.clone(), + ); + + if let SubmittedTransactionHydrated::Real(tx) = tx { + if !tx.user_request.webhook_options.is_empty() { + let event = EoaExecutorEvent { + transaction_id: tx.transaction_id.clone(), + }; + + let success_envelope = + event.transaction_confirmed_envelope(confirmed_tx.clone()); + + let mut tx_context = self + .webhook_queue + .transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + success_envelope, + tx.user_request.webhook_options.clone(), + &mut tx_context, + self.webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for fail: {}", e); + } + } + } + + report.moved_to_success += 1; + } + + // if the ID is not in the confirmed transactions, we queue it for pending + _ => { + if let SubmittedTransactionHydrated::Real(tx) = tx { + // zadd_multiple expects (score, member) + replaced_transactions.push((tx.queued_at, tx.transaction_id.clone())); + + if !tx.user_request.webhook_options.is_empty() { + let event = EoaExecutorEvent { + transaction_id: tx.transaction_id.clone(), + }; + + let success_envelope = + event.transaction_replaced_envelope(tx.data.clone()); + + let mut tx_context = self + .webhook_queue + .transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + success_envelope, + tx.user_request.webhook_options.clone(), + &mut tx_context, + self.webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for fail: {}", e); + } + } + + report.moved_to_pending += 1; + } + } + } + } + } + + if !replaced_transactions.is_empty() { + pipeline.zadd_multiple( + self.keys.pending_transactions_zset_name(), + &replaced_transactions, + ); + } + + pipeline.set( + self.keys.last_transaction_count_key_name(), + self.last_confirmed_nonce + 1, + ); + + // Finalize report stats + report.total_hashes_processed = submitted_txs.len(); + report.unique_transaction_ids = processed_ids.len(); + + report + } +} + +fn detect_violations<'a>( + submitted_txs: &'a [SubmittedTransactionHydrated], + confirmed_hashes: &'a HashSet<&str>, +) -> ( + HashMap<&'a str, Vec>, + BTreeMap>, + CleanupReport, +) { + let mut report = CleanupReport::default(); + let mut txs_by_nonce: BTreeMap> = BTreeMap::new(); + let mut transaction_id_to_nonces: HashMap<&str, Vec> = HashMap::new(); + + let real_submitted_txs: Vec<&SubmittedTransaction> = submitted_txs + .iter() + .filter_map(|tx| match tx { + SubmittedTransactionHydrated::Real(tx) => Some(tx), + SubmittedTransactionHydrated::Noop(_) => None, + }) + .collect(); + + // Group data + for tx in real_submitted_txs { + txs_by_nonce.entry(tx.nonce).or_default().push(tx); + transaction_id_to_nonces + .entry(&tx.transaction_id) + .or_default() + .push(tx.nonce); + } + + // Check cross-nonce violations + for (transaction_id, nonces) in &transaction_id_to_nonces { + let mut unique_nonces = nonces.clone(); + unique_nonces.sort(); + unique_nonces.dedup(); + if unique_nonces.len() > 1 { + report + .cross_nonce_violations + .push((transaction_id.to_string(), unique_nonces)); + } + } + + // Check per-nonce violations + for (nonce, txs) in &txs_by_nonce { + let confirmed_hashes_for_nonce: Vec = txs + .iter() + .filter(|tx| confirmed_hashes.contains(tx.hash.as_str())) + .map(|tx| tx.hash.clone()) + .collect(); + + if confirmed_hashes_for_nonce.len() > 1 { + report + .per_nonce_violations + .push((*nonce, confirmed_hashes_for_nonce)); + } + } + + // Check nonces without receipts + for (nonce, txs) in &txs_by_nonce { + let has_confirmed = txs + .iter() + .any(|tx| confirmed_hashes.contains(tx.hash.as_str())); + if !has_confirmed { + let hashes: Vec = txs.iter().map(|tx| tx.hash.clone()).collect(); + report.nonces_without_receipts.push((*nonce, hashes)); + } + } + + (transaction_id_to_nonces, txs_by_nonce, report) +} + +impl SafeRedisTransaction for CleanAndGetRecycledNonces<'_> { + type ValidationData = (u64, Vec); + type OperationResult = Vec; + + fn name(&self) -> &str { + "clean and get recycled nonces" + } + + fn watch_keys(&self) -> Vec { + vec![ + self.keys.recycled_nonces_zset_name(), + self.keys.last_transaction_count_key_name(), + self.keys.submitted_transactions_zset_name(), + ] + } + + async fn validation( + &self, + conn: &mut ConnectionManager, + _store: &EoaExecutorStore, + ) -> Result { + // get the highest submitted nonce + let highest_submitted: Vec = conn + .zrange_withscores(self.keys.submitted_transactions_zset_name(), -1, -1) + .await?; + + let highest_submitted_nonce = highest_submitted.first().map(|tx| tx.1); + + let highest_submitted_nonce = match highest_submitted_nonce { + Some(nonce) => nonce, + None => { + let cached_tx_count: Option = conn + .get(self.keys.last_transaction_count_key_name()) + .await?; + + let Some(count) = cached_tx_count else { + return Err(TransactionStoreError::NonceSyncRequired { + eoa: self.keys.eoa, + chain_id: self.keys.chain_id, + }); + }; + count.saturating_sub(1) + } + }; + + let recycled_nonces: Vec = conn + .zrange(self.keys.recycled_nonces_zset_name(), 0, -1) + .await?; + + let recycled_nonces = recycled_nonces + .into_iter() + .filter(|nonce| *nonce < highest_submitted_nonce) + .collect(); + + Ok((highest_submitted_nonce, recycled_nonces)) + } + + fn operation( + &self, + pipeline: &mut Pipeline, + (highest_submitted_nonce, recycled_nonces): Self::ValidationData, + ) -> Self::OperationResult { + pipeline.zrembyscore( + self.keys.recycled_nonces_zset_name(), + highest_submitted_nonce, + "+inf", + ); + + recycled_nonces + } +} diff --git a/executors/src/eoa/worker.rs b/executors/src/eoa/worker.rs deleted file mode 100644 index f4939ba..0000000 --- a/executors/src/eoa/worker.rs +++ /dev/null @@ -1,1935 +0,0 @@ -use alloy::consensus::{ - SignableTransaction, Signed, Transaction, TxEip4844Variant, TxEip4844WithSidecar, - TypedTransaction, -}; -use alloy::network::{TransactionBuilder, TransactionBuilder7702}; -use alloy::primitives::{Address, B256, Bytes, U256}; -use alloy::providers::Provider; -use alloy::rpc::types::TransactionRequest as AlloyTransactionRequest; -use alloy::signers::Signature; -use alloy::transports::{RpcError, TransportErrorKind}; -use engine_core::error::EngineError; -use engine_core::signer::AccountSigner; -use engine_core::transaction::TransactionTypeData; -use engine_core::{ - chain::{Chain, ChainService, RpcCredentials}, - credentials::SigningCredential, - error::{AlloyRpcErrorToEngineError, RpcErrorKind}, - signer::{EoaSigner, EoaSigningOptions}, -}; -use hex; -use serde::{Deserialize, Serialize}; -use std::{sync::Arc, time::Duration}; -use tokio::time::sleep; -use twmq::{ - DurableExecution, FailHookData, NackHookData, SuccessHookData, UserCancellable, - error::TwmqError, - hooks::TransactionContext, - job::{BorrowedJob, JobResult, RequeuePosition, ToJobResult}, -}; - -use crate::eoa::store::{ - BorrowedTransactionData, EoaExecutorStore, EoaHealth, EoaTransactionRequest, - ScopedEoaExecutorStore, TransactionData, TransactionStoreError, -}; - -// ========== SPEC-COMPLIANT CONSTANTS ========== -const MAX_INFLIGHT_PER_EOA: u64 = 100; // Default from spec -const MAX_RECYCLED_THRESHOLD: u64 = 50; // Circuit breaker from spec -const TARGET_TRANSACTIONS_PER_EOA: u64 = 10; // Fleet management from spec -const MIN_TRANSACTIONS_PER_EOA: u64 = 1; // Fleet management from spec -const HEALTH_CHECK_INTERVAL: u64 = 300; // 5 minutes in seconds -const NONCE_STALL_TIMEOUT: u64 = 300_000; // 5 minutes in milliseconds - after this time, attempt gas bump - -// ========== JOB DATA ========== -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct EoaExecutorWorkerJobData { - pub eoa_address: Address, - pub chain_id: u64, - pub worker_id: String, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct EoaExecutorWorkerResult { - pub recovered_transactions: u32, - pub confirmed_transactions: u32, - pub failed_transactions: u32, - pub sent_transactions: u32, -} - -#[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE", tag = "errorCode")] -pub enum EoaExecutorWorkerError { - #[error("Chain service error for chainId {chain_id}: {message}")] - ChainServiceError { chain_id: u64, message: String }, - - #[error("Store error: {message}")] - StoreError { - message: String, - inner_error: TransactionStoreError, - }, - - #[error("Transaction not found: {transaction_id}")] - TransactionNotFound { transaction_id: String }, - - #[error("Transaction simulation failed: {message}")] - TransactionSimulationFailed { - message: String, - inner_error: EngineError, - }, - - #[error("Transaction build failed: {message}")] - TransactionBuildFailed { message: String }, - - #[error("RPC error: {message}")] - RpcError { - message: String, - inner_error: EngineError, - }, - - #[error("Signature parsing failed: {message}")] - SignatureParsingFailed { message: String }, - - #[error("Transaction signing failed: {message}")] - SigningError { - message: String, - inner_error: EngineError, - }, - - #[error("Work still remaining: {message}")] - WorkRemaining { message: String }, - - #[error("Internal error: {message}")] - InternalError { message: String }, - - #[error("User cancelled")] - UserCancelled, -} - -impl From for EoaExecutorWorkerError { - fn from(error: TwmqError) -> Self { - EoaExecutorWorkerError::InternalError { - message: format!("Queue error: {}", error), - } - } -} - -impl From for EoaExecutorWorkerError { - fn from(error: TransactionStoreError) -> Self { - EoaExecutorWorkerError::StoreError { - message: error.to_string(), - inner_error: error, - } - } -} - -impl UserCancellable for EoaExecutorWorkerError { - fn user_cancelled() -> Self { - EoaExecutorWorkerError::UserCancelled - } -} - -// ========== SIMPLE ERROR CLASSIFICATION ========== -#[derive(Debug)] -enum SendErrorClassification { - PossiblySent, // "nonce too low", "already known" etc - DeterministicFailure, // Invalid signature, malformed tx, insufficient funds etc -} - -#[derive(PartialEq, Eq, Debug)] -enum SendContext { - Rebroadcast, - InitialBroadcast, -} - -#[tracing::instrument(skip_all, fields(error = %error, context = ?context))] -fn classify_send_error( - error: &RpcError, - context: SendContext, -) -> SendErrorClassification { - if !error.is_error_resp() { - return SendErrorClassification::DeterministicFailure; - } - - let error_str = error.to_string().to_lowercase(); - - // Deterministic failures that didn't consume nonce (spec-compliant) - if error_str.contains("invalid signature") - || error_str.contains("malformed transaction") - || (context == SendContext::InitialBroadcast && error_str.contains("insufficient funds")) - || error_str.contains("invalid transaction format") - || error_str.contains("nonce too high") - // Should trigger nonce reset - { - return SendErrorClassification::DeterministicFailure; - } - - // Transaction possibly made it to mempool (spec-compliant) - if error_str.contains("nonce too low") - || error_str.contains("already known") - || error_str.contains("replacement transaction underpriced") - { - return SendErrorClassification::PossiblySent; - } - - // Additional common failures that didn't consume nonce - if error_str.contains("malformed") - || error_str.contains("gas limit") - || error_str.contains("intrinsic gas too low") - { - return SendErrorClassification::DeterministicFailure; - } - - tracing::warn!( - "Unknown send error: {}. PLEASE REPORT FOR ADDING CORRECT CLASSIFICATION [NOTIFY]", - error_str - ); - - // Default: assume possibly sent for safety - SendErrorClassification::PossiblySent -} - -fn should_trigger_nonce_reset(error: &RpcError) -> bool { - let error_str = error.to_string().to_lowercase(); - - // "nonce too high" should trigger nonce reset as per spec - error_str.contains("nonce too high") -} - -fn should_update_balance_threshold(error: &EngineError) -> bool { - match error { - EngineError::RpcError { kind, .. } - | EngineError::PaymasterError { kind, .. } - | EngineError::BundlerError { kind, .. } => match kind { - RpcErrorKind::ErrorResp(resp) => { - let message = resp.message.to_lowercase(); - message.contains("insufficient funds") - || message.contains("insufficient balance") - || message.contains("out of gas") - || message.contains("insufficient eth") - || message.contains("balance too low") - || message.contains("not enough funds") - || message.contains("insufficient native token") - } - _ => false, - }, - _ => false, - } -} - -fn is_retryable_rpc_error(kind: &RpcErrorKind) -> bool { - match kind { - RpcErrorKind::TransportHttpError { status, .. } if *status >= 400 && *status < 500 => false, - RpcErrorKind::UnsupportedFeature { .. } => false, - _ => true, - } -} - -// ========== PREPARED TRANSACTION ========== -#[derive(Debug, Clone)] -struct PreparedTransaction { - transaction_id: String, - signed_tx: Signed, - nonce: u64, -} - -// ========== CONFIRMATION FLOW DATA STRUCTURES ========== -#[derive(Debug, Clone)] -struct PendingTransaction { - nonce: u64, - hash: String, - transaction_id: String, -} - -#[derive(Debug, Clone)] -struct ConfirmedTransaction { - nonce: u64, - hash: String, - transaction_id: String, - receipt: alloy::rpc::types::TransactionReceipt, -} - -#[derive(Debug, Clone)] -struct FailedTransaction { - hash: String, - transaction_id: String, -} - -// ========== STORE BATCH OPERATION TYPES ========== -#[derive(Debug, Clone)] -pub struct TransactionSuccess { - pub hash: String, - pub transaction_id: String, - pub receipt_data: String, -} - -#[derive(Debug, Clone)] -pub struct TransactionFailure { - pub hash: String, - pub transaction_id: String, -} - -// ========== MAIN WORKER ========== -/// EOA Executor Worker -/// -/// ## Core Workflow: -/// 1. **Acquire Lock Aggressively** - Takes over stalled workers using force acquisition. This is a lock over EOA:CHAIN -/// 2. **Crash Recovery** - Rebroadcasts borrowed transactions, handles deterministic failures -/// 3. **Confirmation Flow** - Fetches receipts, confirms transactions, handles nonce sync, requeues replaced transactions -/// 4. **Send Flow** - Processes recycled nonces first, then new transactions with in-flight budget control -/// 5. **Lock Release** - Explicit release in finally pattern as per spec -/// -/// ## Key Features: -/// - **Atomic Operations**: All state transitions use Redis WATCH/MULTI/EXEC for durability -/// - **Borrowed State**: Mid-send crash recovery with atomic pending->borrowed->submitted transitions -/// - **Nonce Management**: Optimistic nonce tracking with recycled nonce priority -/// - **Error Classification**: Spec-compliant deterministic vs. possibly-sent error handling -/// - **Circuit Breakers**: Automatic recycled nonce nuking when threshold exceeded -/// - **Health Monitoring**: Balance checking with configurable thresholds -pub struct EoaExecutorWorker -where - CS: ChainService + Send + Sync + 'static, -{ - pub chain_service: Arc, - pub store: Arc, - pub eoa_signer: Arc, - pub max_inflight: u64, // Note: Spec uses MAX_INFLIGHT_PER_EOA constant - pub max_recycled_nonces: u64, // Note: Spec uses MAX_RECYCLED_THRESHOLD constant -} - -impl DurableExecution for EoaExecutorWorker -where - CS: ChainService + Send + Sync + 'static, -{ - type Output = EoaExecutorWorkerResult; - type ErrorData = EoaExecutorWorkerError; - type JobData = EoaExecutorWorkerJobData; - - #[tracing::instrument(skip_all, fields(eoa = %job.job.data.eoa_address, chain_id = job.job.data.chain_id))] - async fn process( - &self, - job: &BorrowedJob, - ) -> JobResult { - let data = &job.job.data; - - // 1. GET CHAIN - let chain = self - .chain_service - .get_chain(data.chain_id) - .map_err(|e| EoaExecutorWorkerError::ChainServiceError { - chain_id: data.chain_id, - message: format!("Failed to get chain: {}", e), - }) - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; - - // 2. CREATE SCOPED STORE (acquires lock) - let scoped = ScopedEoaExecutorStore::build( - &self.store, - data.eoa_address, - data.chain_id, - data.worker_id.clone(), - ) - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; - - // initiate health data if doesn't exist - self.get_eoa_health(&scoped, &chain) - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; - - // Execute main workflow with proper error handling - self.execute_main_workflow(&scoped, &chain).await - } - - async fn on_success( - &self, - job: &BorrowedJob, - _success_data: SuccessHookData<'_, Self::Output>, - _tx: &mut TransactionContext<'_>, - ) { - // Release EOA lock on success - self.release_eoa_lock( - job.job.data.eoa_address, - job.job.data.chain_id, - &job.job.data.worker_id, - ) - .await; - } - - async fn on_nack( - &self, - job: &BorrowedJob, - _nack_data: NackHookData<'_, Self::ErrorData>, - _tx: &mut TransactionContext<'_>, - ) { - // Release EOA lock on nack - self.release_eoa_lock( - job.job.data.eoa_address, - job.job.data.chain_id, - &job.job.data.worker_id, - ) - .await; - } - - async fn on_fail( - &self, - job: &BorrowedJob, - _fail_data: FailHookData<'_, Self::ErrorData>, - _tx: &mut TransactionContext<'_>, - ) { - // Release EOA lock on fail - self.release_eoa_lock( - job.job.data.eoa_address, - job.job.data.chain_id, - &job.job.data.worker_id, - ) - .await; - } -} - -impl EoaExecutorWorker -where - CS: ChainService + Send + Sync + 'static, -{ - /// Execute the main EOA worker workflow - async fn execute_main_workflow( - &self, - scoped: &ScopedEoaExecutorStore<'_>, - chain: &impl Chain, - ) -> JobResult { - // 1. CRASH RECOVERY - let recovered = self - .recover_borrowed_state(scoped, chain) - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; - - // 2. CONFIRM FLOW - let (confirmed, failed) = self - .confirm_flow(scoped, chain) - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; - - // 3. SEND FLOW - let sent = self - .send_flow(scoped, chain) - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; - - // 4. CHECK FOR REMAINING WORK - let pending_count = scoped - .peek_pending_transactions(1000) - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)? - .len(); - let borrowed_count = scoped - .peek_borrowed_transactions() - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)? - .len(); - let recycled_count = scoped - .peek_recycled_nonces() - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)? - .len(); - - // NACK here is a yield, when you think of the queue as a distributed EOA scheduler - if pending_count > 0 || borrowed_count > 0 || recycled_count > 0 { - return Err(EoaExecutorWorkerError::WorkRemaining { - message: format!( - "Work remaining: {} pending, {} borrowed, {} recycled", - pending_count, borrowed_count, recycled_count - ), - }) - .map_err_nack(Some(Duration::from_secs(2)), RequeuePosition::Last); - } - - // Only succeed if no work remains - Ok(EoaExecutorWorkerResult { - recovered_transactions: recovered, - confirmed_transactions: confirmed, - failed_transactions: failed, - sent_transactions: sent, - }) - } - - /// Release EOA lock following the spec's finally pattern - async fn release_eoa_lock(&self, eoa: Address, chain_id: u64, worker_id: &str) { - if let Err(e) = self.store.release_eoa_lock(eoa, chain_id, worker_id).await { - tracing::error!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, - error = %e, - "Failed to release EOA lock" - ); - } - } - - // ========== CRASH RECOVERY ========== - #[tracing::instrument(skip_all)] - async fn recover_borrowed_state( - &self, - scoped: &ScopedEoaExecutorStore<'_>, - chain: &impl Chain, - ) -> Result { - let mut borrowed_transactions = scoped.peek_borrowed_transactions().await?; - - if borrowed_transactions.is_empty() { - return Ok(0); - } - - tracing::warn!( - "Recovering {} borrowed transactions. This indicates a worker crash or system issue", - borrowed_transactions.len() - ); - - // Sort borrowed transactions by nonce to ensure proper ordering - borrowed_transactions.sort_by_key(|tx| tx.signed_transaction.nonce()); - - // Rebroadcast all transactions in parallel - let rebroadcast_futures: Vec<_> = borrowed_transactions - .iter() - .map(|borrowed| { - let tx_envelope = borrowed.signed_transaction.clone().into(); - let nonce = borrowed.signed_transaction.nonce(); - let transaction_id = borrowed.transaction_id.clone(); - - tracing::info!( - transaction_id = %transaction_id, - nonce = nonce, - "Recovering borrowed transaction" - ); - - async move { - let send_result = chain.provider().send_tx_envelope(tx_envelope).await; - (borrowed, send_result) - } - }) - .collect(); - - let rebroadcast_results = futures::future::join_all(rebroadcast_futures).await; - - // Process results sequentially for Redis state changes - let mut recovered_count = 0; - for (borrowed, send_result) in rebroadcast_results { - let nonce = borrowed.signed_transaction.nonce(); - - match send_result { - Ok(_) => { - // Transaction was sent successfully - scoped - .move_borrowed_to_submitted( - nonce, - &format!("{:?}", borrowed.hash), - &borrowed.transaction_id, - ) - .await?; - tracing::info!(transaction_id = %borrowed.transaction_id, nonce = nonce, "Moved recovered transaction to submitted"); - } - Err(e) => { - match classify_send_error(&e, SendContext::Rebroadcast) { - SendErrorClassification::PossiblySent => { - // Transaction possibly sent, move to submitted - scoped - .move_borrowed_to_submitted( - nonce, - &format!("{:?}", borrowed.hash), - &borrowed.transaction_id, - ) - .await?; - tracing::info!(transaction_id = %borrowed.transaction_id, nonce = nonce, "Moved recovered transaction to submitted (possibly sent)"); - } - SendErrorClassification::DeterministicFailure => { - // Transaction is broken, recycle nonce and requeue - scoped - .move_borrowed_to_recycled(nonce, &borrowed.transaction_id) - .await?; - tracing::warn!(transaction_id = %borrowed.transaction_id, nonce = nonce, error = %e, "Recycled failed transaction"); - - if should_update_balance_threshold(&e.to_engine_error(chain)) { - self.update_balance_threshold(scoped, chain).await?; - } - - // Check if this should trigger nonce reset - if should_trigger_nonce_reset(&e) { - tracing::warn!( - eoa = %scoped.eoa(), - chain_id = %scoped.chain_id(), - "Nonce too high error detected, may need nonce synchronization" - ); - // The next confirm_flow will fetch fresh nonce and auto-sync - } - } - } - } - } - - recovered_count += 1; - } - - Ok(recovered_count) - } - - // ========== CONFIRM FLOW ========== - #[tracing::instrument(skip_all)] - async fn confirm_flow( - &self, - scoped: &ScopedEoaExecutorStore<'_>, - chain: &impl Chain, - ) -> Result<(u32, u32), EoaExecutorWorkerError> { - // Get fresh on-chain transaction count - let current_chain_nonce = chain - .provider() - .get_transaction_count(scoped.eoa()) - .await - .map_err(|e| { - let engine_error = e.to_engine_error(chain); - EoaExecutorWorkerError::RpcError { - message: format!("Failed to get transaction count: {}", engine_error), - inner_error: engine_error, - } - })?; - - let cached_nonce = match scoped.get_cached_transaction_count().await { - Err(e) => match e { - TransactionStoreError::NonceSyncRequired { .. } => { - scoped.reset_nonces(current_chain_nonce).await?; - current_chain_nonce - } - _ => return Err(e.into()), - }, - Ok(cached_nonce) => cached_nonce, - }; - - // no nonce progress - if current_chain_nonce == cached_nonce { - let current_health = self.get_eoa_health(scoped, chain).await?; - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - // No nonce progress - check if we should attempt gas bumping for stalled nonce - let time_since_movement = now.saturating_sub(current_health.last_nonce_movement_at); - - if time_since_movement > NONCE_STALL_TIMEOUT { - tracing::info!( - time_since_movement = time_since_movement, - stall_timeout = NONCE_STALL_TIMEOUT, - current_chain_nonce = current_chain_nonce, - "Nonce has been stalled, attempting gas bump" - ); - - // Attempt gas bump for the next expected nonce - if let Err(e) = self - .attempt_gas_bump_for_stalled_nonce(scoped, chain, current_chain_nonce) - .await - { - tracing::warn!( - error = %e, - "Failed to attempt gas bump for stalled nonce" - ); - } - } - - tracing::debug!("No nonce progress, skipping confirm flow"); - return Ok((0, 0)); - } - - tracing::info!( - current_chain_nonce = current_chain_nonce, - cached_nonce = cached_nonce, - "Processing confirmations" - ); - - // Get all pending transactions below the current chain nonce - let pending_txs = self - .get_pending_transactions_below_nonce(scoped, current_chain_nonce) - .await?; - - if pending_txs.is_empty() { - tracing::debug!("No pending transactions to confirm"); - return Ok((0, 0)); - } - - // Fetch receipts and categorize transactions - let (confirmed_txs, failed_txs) = self - .fetch_and_categorize_transactions(chain, pending_txs) - .await; - - // Process confirmed transactions - let confirmed_count = if !confirmed_txs.is_empty() { - let successes: Vec = confirmed_txs - .into_iter() - .map(|tx| { - let receipt_data = match serde_json::to_string(&tx.receipt) { - Ok(receipt_json) => receipt_json, - Err(e) => { - tracing::warn!( - transaction_id = %tx.transaction_id, - hash = %tx.hash, - error = %e, - "Failed to serialize receipt as JSON, using debug format" - ); - format!("{:?}", tx.receipt) - } - }; - - tracing::info!( - transaction_id = %tx.transaction_id, - nonce = tx.nonce, - hash = %tx.hash, - "Transaction confirmed" - ); - - TransactionSuccess { - hash: tx.hash, - transaction_id: tx.transaction_id, - receipt_data, - } - }) - .collect(); - - let count = successes.len() as u32; - scoped.batch_succeed_transactions(successes).await?; - count - } else { - 0 - }; - - // Process failed transactions - let failed_count = if !failed_txs.is_empty() { - let failures: Vec = failed_txs - .into_iter() - .map(|tx| { - tracing::warn!( - transaction_id = %tx.transaction_id, - hash = %tx.hash, - "Transaction failed, requeued" - ); - TransactionFailure { - hash: tx.hash, - transaction_id: tx.transaction_id, - } - }) - .collect(); - - let count = failures.len() as u32; - scoped.batch_fail_and_requeue_transactions(failures).await?; - count - } else { - 0 - }; - - // Update cached transaction count - scoped - .update_cached_transaction_count(current_chain_nonce) - .await?; - - // Synchronize nonces to ensure consistency - if let Err(e) = self - .store - .synchronize_nonces_with_chain( - scoped.eoa(), - scoped.chain_id(), - scoped.worker_id(), - current_chain_nonce, - ) - .await - { - tracing::warn!(error = %e, "Failed to synchronize nonces with chain"); - } - - Ok((confirmed_count, failed_count)) - } - - // ========== SEND FLOW ========== - #[tracing::instrument(skip_all)] - async fn send_flow( - &self, - scoped: &ScopedEoaExecutorStore<'_>, - chain: &impl Chain, - ) -> Result { - // 1. Get EOA health (initializes if needed) and check if we should update balance - let mut health = self.get_eoa_health(scoped, chain).await?; - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - // Update balance if it's stale - if now - health.balance_fetched_at > HEALTH_CHECK_INTERVAL { - let balance = chain - .provider() - .get_balance(scoped.eoa()) - .await - .map_err(|e| { - let engine_error = e.to_engine_error(chain); - EoaExecutorWorkerError::RpcError { - message: format!("Failed to get balance: {}", engine_error), - inner_error: engine_error, - } - })?; - - health.balance = balance; - health.balance_fetched_at = now; - scoped.update_health_data(&health).await?; - } - - if health.balance <= health.balance_threshold { - tracing::warn!( - "EOA has insufficient balance (<= {} wei), skipping send flow", - health.balance_threshold - ); - return Ok(0); - } - - let mut total_sent = 0; - - // 2. Process recycled nonces first - total_sent += self.process_recycled_nonces(scoped, chain).await?; - - // 3. Only proceed to new nonces if we successfully used all recycled nonces - let remaining_recycled = scoped.peek_recycled_nonces().await?.len(); - if remaining_recycled == 0 { - let inflight_budget = scoped.get_inflight_budget(self.max_inflight).await?; - if inflight_budget > 0 { - total_sent += self - .process_new_transactions(scoped, chain, inflight_budget) - .await?; - } - } else { - tracing::warn!( - "Still have {} recycled nonces, not sending new transactions", - remaining_recycled - ); - } - - Ok(total_sent) - } - - async fn process_recycled_nonces( - &self, - scoped: &ScopedEoaExecutorStore<'_>, - chain: &impl Chain, - ) -> Result { - let recycled_nonces = scoped.peek_recycled_nonces().await?; - - if recycled_nonces.is_empty() { - return Ok(0); - } - - // Get pending transactions (one per recycled nonce) - let pending_txs = scoped - .peek_pending_transactions(recycled_nonces.len() as u64) - .await?; - - // 1. SEQUENTIAL REDIS: Collect nonce-transaction pairs - let mut nonce_tx_pairs = Vec::new(); - for (i, nonce) in recycled_nonces.into_iter().enumerate() { - if let Some(tx_id) = pending_txs.get(i) { - // Get transaction data - if let Some(tx_data) = scoped.get_transaction_data(tx_id).await? { - nonce_tx_pairs.push((nonce, tx_id.clone(), tx_data)); - } else { - tracing::warn!("Transaction data not found for {}", tx_id); - continue; - } - } else { - // No pending transactions - skip recycled nonces without pending transactions - tracing::debug!("No pending transaction for recycled nonce {}", nonce); - continue; - } - } - - if nonce_tx_pairs.is_empty() { - return Ok(0); - } - - // 2. PARALLEL BUILD/SIGN: Build and sign all transactions in parallel - let build_futures: Vec<_> = nonce_tx_pairs - .iter() - .map(|(nonce, transaction_id, tx_data)| async move { - let prepared = self - .build_and_sign_transaction(tx_data, *nonce, chain) - .await; - (*nonce, transaction_id, prepared) - }) - .collect(); - - let build_results = futures::future::join_all(build_futures).await; - - // 3. SEQUENTIAL REDIS: Move successfully built transactions to borrowed state - let mut prepared_txs = Vec::new(); - let mut balance_threshold_update_needed = false; - - for (nonce, transaction_id, build_result) in build_results { - match build_result { - Ok(signed_tx) => { - let borrowed_data = BorrowedTransactionData { - transaction_id: transaction_id.clone(), - signed_transaction: signed_tx.clone(), - hash: signed_tx.hash().to_string(), - borrowed_at: chrono::Utc::now().timestamp_millis().max(0) as u64, - }; - - // Try to atomically move from pending to borrowed with recycled nonce - match scoped - .atomic_move_pending_to_borrowed_with_recycled_nonce( - transaction_id, - nonce, - &borrowed_data, - ) - .await - { - Ok(()) => { - let prepared = PreparedTransaction { - transaction_id: transaction_id.clone(), - signed_tx, - nonce, - }; - prepared_txs.push(prepared); - } - Err(TransactionStoreError::NonceNotInRecycledSet { .. }) => { - tracing::debug!("Nonce {} was consumed by another worker", nonce); - continue; - } - Err(TransactionStoreError::TransactionNotInPendingQueue { .. }) => { - tracing::debug!("Transaction {} already processed", transaction_id); - continue; - } - Err(e) => { - tracing::error!("Failed to move {} to borrowed: {}", transaction_id, e); - continue; - } - } - } - Err(e) => { - // Accumulate balance threshold issues instead of updating immediately - if let EoaExecutorWorkerError::TransactionSimulationFailed { - inner_error, .. - } = &e - { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; - } - } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; - } - } - - tracing::warn!("Failed to build transaction {}: {}", transaction_id, e); - continue; - } - } - } - - // Update balance threshold once if any build failures were due to balance issues - if balance_threshold_update_needed { - if let Err(e) = self.update_balance_threshold(scoped, chain).await { - tracing::error!( - "Failed to update balance threshold after parallel build failures: {}", - e - ); - } - } - - if prepared_txs.is_empty() { - return Ok(0); - } - - // 4. PARALLEL SEND: Send all transactions in parallel - let send_futures: Vec<_> = prepared_txs - .iter() - .map(|prepared| async move { - let result = chain - .provider() - .send_tx_envelope(prepared.signed_tx.clone().into()) - .await; - (prepared, result) - }) - .collect(); - - let send_results = futures::future::join_all(send_futures).await; - - // 5. SEQUENTIAL REDIS: Process results and update states - let mut sent_count = 0; - for (prepared, send_result) in send_results { - match send_result { - Ok(_) => { - // Transaction sent successfully - match scoped - .move_borrowed_to_submitted( - prepared.nonce, - &format!("{:?}", prepared.signed_tx.hash()), - &prepared.transaction_id, - ) - .await - { - Ok(()) => { - sent_count += 1; - tracing::info!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - hash = ?prepared.signed_tx.hash(), - "Successfully sent recycled transaction" - ); - } - Err(e) => { - tracing::error!( - "Failed to move {} to submitted: {}", - prepared.transaction_id, - e - ); - } - } - } - Err(e) => { - match classify_send_error(&e, SendContext::InitialBroadcast) { - SendErrorClassification::PossiblySent => { - // Move to submitted state - match scoped - .move_borrowed_to_submitted( - prepared.nonce, - &format!("{:?}", prepared.signed_tx.hash()), - &prepared.transaction_id, - ) - .await - { - Ok(()) => { - sent_count += 1; - tracing::info!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - "Recycled transaction possibly sent" - ); - } - Err(e) => { - tracing::error!( - "Failed to move {} to submitted: {}", - prepared.transaction_id, - e - ); - } - } - } - SendErrorClassification::DeterministicFailure => { - // Recycle nonce and requeue transaction - match scoped - .move_borrowed_to_recycled(prepared.nonce, &prepared.transaction_id) - .await - { - Ok(()) => { - tracing::warn!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - error = %e, - "Recycled transaction failed, re-recycled nonce" - ); - - if should_update_balance_threshold(&e.to_engine_error(chain)) { - if let Err(e) = - self.update_balance_threshold(scoped, chain).await - { - tracing::error!( - "Failed to update balance threshold: {}", - e - ); - } - } - - if should_trigger_nonce_reset(&e) { - tracing::warn!( - nonce = prepared.nonce, - "Nonce too high error detected, may need nonce synchronization" - ); - } - } - Err(e) => { - tracing::error!( - "Failed to move {} back to recycled: {}", - prepared.transaction_id, - e - ); - } - } - } - } - } - } - } - - Ok(sent_count) - } - - async fn process_new_transactions( - &self, - scoped: &ScopedEoaExecutorStore<'_>, - chain: &impl Chain, - budget: u64, - ) -> Result { - if budget == 0 { - return Ok(0); - } - - // 1. SEQUENTIAL REDIS: Get pending transactions - let pending_txs = scoped.peek_pending_transactions(budget).await?; - if pending_txs.is_empty() { - return Ok(0); - } - - let optimistic_nonce = scoped.get_optimistic_nonce().await?; - - // 2. PARALLEL BUILD/SIGN: Build and sign all transactions in parallel - let build_tasks: Vec<_> = pending_txs - .iter() - .enumerate() - .map(|(i, tx_id)| { - let expected_nonce = optimistic_nonce + i as u64; - self.build_and_sign_single_transaction(scoped, tx_id, expected_nonce, chain) - }) - .collect(); - - let prepared_results = futures::future::join_all(build_tasks).await; - - // 3. SEQUENTIAL REDIS: Move successful transactions to borrowed state (maintain nonce order) - let mut prepared_txs = Vec::new(); - let mut balance_threshold_update_needed = false; - - for (i, result) in prepared_results.into_iter().enumerate() { - match result { - Ok(prepared) => { - let borrowed_data = BorrowedTransactionData { - transaction_id: prepared.transaction_id.clone(), - signed_transaction: prepared.signed_tx.clone(), - hash: prepared.signed_tx.hash().to_string(), - borrowed_at: chrono::Utc::now().timestamp_millis().max(0) as u64, - }; - - match scoped - .atomic_move_pending_to_borrowed_with_new_nonce( - &prepared.transaction_id, - prepared.nonce, - &borrowed_data, - ) - .await - { - Ok(()) => prepared_txs.push(prepared), - Err(TransactionStoreError::OptimisticNonceChanged { .. }) => { - tracing::debug!( - "Nonce changed for transaction {}, skipping", - prepared.transaction_id - ); - break; // Stop processing if nonce changed - } - Err(TransactionStoreError::TransactionNotInPendingQueue { .. }) => { - tracing::debug!( - "Transaction {} already processed, skipping", - prepared.transaction_id - ); - continue; - } - Err(e) => { - tracing::error!( - "Failed to move transaction {} to borrowed: {}", - prepared.transaction_id, - e - ); - continue; - } - } - } - Err(e) => { - // Accumulate balance threshold issues instead of updating immediately - if let EoaExecutorWorkerError::TransactionSimulationFailed { - inner_error, .. - } = &e - { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; - } - } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; - } - } - - tracing::warn!("Failed to build transaction {}: {}", pending_txs[i], e); - // Individual transaction failure doesn't stop the worker - continue; - } - } - } - - // Update balance threshold once if any build failures were due to balance issues - if balance_threshold_update_needed { - if let Err(e) = self.update_balance_threshold(scoped, chain).await { - tracing::error!( - "Failed to update balance threshold after parallel build failures: {}", - e - ); - } - } - - if prepared_txs.is_empty() { - return Ok(0); - } - - // 4. PARALLEL SEND (but ordered): Send all transactions in parallel but in nonce order - let send_futures: Vec<_> = prepared_txs - .iter() - .enumerate() - .map(|(i, prepared)| async move { - // Add delay for ordering (except first transaction) - if i > 0 { - sleep(Duration::from_millis(50)).await; // 50ms delay between consecutive nonces - } - - let result = chain - .provider() - .send_tx_envelope(prepared.signed_tx.clone().into()) - .await; - (prepared, result) - }) - .collect(); - - let send_results = futures::future::join_all(send_futures).await; - - // 5. SEQUENTIAL REDIS: Process results and update states - let mut sent_count = 0; - for (prepared, send_result) in send_results { - match send_result { - Ok(_) => { - // Transaction sent successfully - match scoped - .move_borrowed_to_submitted( - prepared.nonce, - &format!("{:?}", prepared.signed_tx.hash()), - &prepared.transaction_id, - ) - .await - { - Ok(()) => { - sent_count += 1; - tracing::info!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - hash = ?prepared.signed_tx.hash(), - "Successfully sent new transaction" - ); - } - Err(e) => { - tracing::error!( - "Failed to move {} to submitted: {}", - prepared.transaction_id, - e - ); - } - } - } - Err(e) => { - match classify_send_error(&e, SendContext::InitialBroadcast) { - SendErrorClassification::PossiblySent => { - // Move to submitted state - match scoped - .move_borrowed_to_submitted( - prepared.nonce, - &format!("{:?}", prepared.signed_tx.hash()), - &prepared.transaction_id, - ) - .await - { - Ok(()) => { - sent_count += 1; - tracing::info!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - "New transaction possibly sent" - ); - } - Err(e) => { - tracing::error!( - "Failed to move {} to submitted: {}", - prepared.transaction_id, - e - ); - } - } - } - SendErrorClassification::DeterministicFailure => { - // Recycle nonce and requeue transaction - match scoped - .move_borrowed_to_recycled(prepared.nonce, &prepared.transaction_id) - .await - { - Ok(()) => { - tracing::warn!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - error = %e, - "New transaction failed, recycled nonce" - ); - - if should_update_balance_threshold(&e.to_engine_error(chain)) { - if let Err(e) = - self.update_balance_threshold(scoped, chain).await - { - tracing::error!( - "Failed to update balance threshold: {}", - e - ); - } - } - - if should_trigger_nonce_reset(&e) { - tracing::warn!( - nonce = prepared.nonce, - "Nonce too high error detected, may need nonce synchronization" - ); - } - } - Err(e) => { - tracing::error!( - "Failed to move {} to recycled: {}", - prepared.transaction_id, - e - ); - } - } - } - } - } - } - } - - Ok(sent_count) - } - - // ========== TRANSACTION BUILDING & SENDING ========== - async fn build_and_sign_single_transaction( - &self, - scoped: &ScopedEoaExecutorStore<'_>, - transaction_id: &str, - nonce: u64, - chain: &impl Chain, - ) -> Result { - // Get transaction data - let tx_data = scoped - .get_transaction_data(transaction_id) - .await? - .ok_or_else(|| EoaExecutorWorkerError::TransactionNotFound { - transaction_id: transaction_id.to_string(), - })?; - - // Build and sign transaction - let signed_tx = self - .build_and_sign_transaction(&tx_data, nonce, chain) - .await?; - - Ok(PreparedTransaction { - transaction_id: transaction_id.to_string(), - signed_tx, - nonce, - }) - } - - async fn send_noop_transaction( - &self, - scoped: &ScopedEoaExecutorStore<'_>, - chain: &impl Chain, - nonce: u64, - ) -> Result { - // Create a minimal transaction to consume the recycled nonce - // Send 0 ETH to self with minimal gas - let eoa = scoped.eoa(); - - // Build no-op transaction (send 0 to self) - let mut tx_request = AlloyTransactionRequest::default() - .with_from(eoa) - .with_to(eoa) // Send to self - .with_value(U256::ZERO) // Send 0 value - .with_input(Bytes::new()) // No data - .with_chain_id(scoped.chain_id()) - .with_nonce(nonce) - .with_gas_limit(21000); // Minimal gas for basic transfer - - // Estimate gas to ensure the transaction is valid - match chain.provider().estimate_gas(tx_request.clone()).await { - Ok(gas_limit) => { - tx_request = tx_request.with_gas_limit(gas_limit); - } - Err(e) => { - tracing::warn!( - nonce = nonce, - error = %e, - "Failed to estimate gas for no-op transaction" - ); - return Ok(false); - } - } - - // Build typed transaction - let typed_tx = match tx_request.build_typed_tx() { - Ok(tx) => tx, - Err(e) => { - tracing::warn!( - nonce = nonce, - error = ?e, - "Failed to build typed transaction for no-op" - ); - return Ok(false); - } - }; - - // Get signing credential from health or use default approach - // For no-op transactions, we need to find a valid signing credential - // This is a limitation of the current design - no-op transactions - // need access to signing credentials which are transaction-specific - tracing::warn!( - nonce = nonce, - "No-op transaction requires signing credential access - recycled nonce will remain unconsumed" - ); - Ok(false) - } - - // ========== GAS BUMP METHODS ========== - - /// Attempt to gas bump a stalled transaction for the next expected nonce - async fn attempt_gas_bump_for_stalled_nonce( - &self, - scoped: &ScopedEoaExecutorStore<'_>, - chain: &impl Chain, - expected_nonce: u64, - ) -> Result { - tracing::info!( - nonce = expected_nonce, - "Attempting gas bump for stalled nonce" - ); - - // Get all transaction IDs for this nonce - let transaction_ids = scoped.get_transaction_ids_for_nonce(expected_nonce).await?; - - if transaction_ids.is_empty() { - tracing::debug!( - nonce = expected_nonce, - "No transactions found for stalled nonce" - ); - return Ok(false); - } - - // Load transaction data for all IDs and find the newest one - let mut newest_transaction: Option<(String, TransactionData)> = None; - let mut newest_submitted_at = 0u64; - - for transaction_id in transaction_ids { - if let Some(tx_data) = scoped.get_transaction_data(&transaction_id).await? { - // Find the most recent attempt for this transaction - if let Some(latest_attempt) = tx_data.attempts.last() { - let submitted_at = latest_attempt.sent_at; - if submitted_at > newest_submitted_at { - newest_submitted_at = submitted_at; - newest_transaction = Some((transaction_id, tx_data)); - } - } - } - } - - if let Some((transaction_id, tx_data)) = newest_transaction { - tracing::info!( - transaction_id = %transaction_id, - nonce = expected_nonce, - "Found newest transaction for gas bump" - ); - - // Get the latest attempt to extract gas values from - // Build typed transaction -> manually bump -> sign - let typed_tx = match self - .build_typed_transaction(&tx_data, expected_nonce, chain) - .await - { - Ok(tx) => tx, - Err(e) => { - // Check if this is a balance threshold issue during simulation - if let EoaExecutorWorkerError::TransactionSimulationFailed { - inner_error, .. - } = &e - { - if should_update_balance_threshold(inner_error) { - if let Err(e) = self.update_balance_threshold(scoped, chain).await { - tracing::error!("Failed to update balance threshold: {}", e); - } - } - } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { - if should_update_balance_threshold(inner_error) { - if let Err(e) = self.update_balance_threshold(scoped, chain).await { - tracing::error!("Failed to update balance threshold: {}", e); - } - } - } - - tracing::warn!( - transaction_id = %transaction_id, - nonce = expected_nonce, - error = %e, - "Failed to build typed transaction for gas bump" - ); - return Ok(false); - } - }; - let bumped_typed_tx = self.apply_gas_bump_to_typed_transaction(typed_tx, 120); // 20% increase - let bumped_tx = match self.sign_transaction(bumped_typed_tx, &tx_data).await { - Ok(tx) => tx, - Err(e) => { - tracing::warn!( - transaction_id = %transaction_id, - nonce = expected_nonce, - error = %e, - "Failed to sign transaction for gas bump" - ); - return Ok(false); - } - }; - - // Record the gas bump attempt - scoped - .add_gas_bump_attempt(&transaction_id, bumped_tx.clone()) - .await?; - - // Send the bumped transaction - let tx_envelope = bumped_tx.into(); - match chain.provider().send_tx_envelope(tx_envelope).await { - Ok(_) => { - tracing::info!( - transaction_id = %transaction_id, - nonce = expected_nonce, - "Successfully sent gas bumped transaction" - ); - return Ok(true); - } - Err(e) => { - tracing::warn!( - transaction_id = %transaction_id, - nonce = expected_nonce, - error = %e, - "Failed to send gas bumped transaction" - ); - // Don't fail the worker, just log the error - return Ok(false); - } - } - } - - Ok(false) - } - - // ========== HEALTH ACCESSOR ========== - - /// Get EOA health, initializing it if it doesn't exist - /// This method ensures the health data is always available for the worker - async fn get_eoa_health( - &self, - scoped: &ScopedEoaExecutorStore<'_>, - chain: &impl Chain, - ) -> Result { - let store_health = scoped.check_eoa_health().await?; - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - match store_health { - Some(health) => Ok(health), - None => { - // Initialize with fresh data from chain - let balance = chain - .provider() - .get_balance(scoped.eoa()) - .await - .map_err(|e| { - let engine_error = e.to_engine_error(chain); - EoaExecutorWorkerError::RpcError { - message: format!( - "Failed to get balance during initialization: {}", - engine_error - ), - inner_error: engine_error, - } - })?; - - let health = EoaHealth { - balance, - balance_threshold: U256::ZERO, - balance_fetched_at: now, - last_confirmation_at: now, - last_nonce_movement_at: now, - nonce_resets: Vec::new(), - }; - - // Save to store - scoped.update_health_data(&health).await?; - Ok(health) - } - } - } - - #[tracing::instrument(skip_all, fields(eoa = %scoped.eoa(), chain_id = %chain.chain_id()))] - async fn update_balance_threshold( - &self, - scoped: &ScopedEoaExecutorStore<'_>, - chain: &impl Chain, - ) -> Result<(), EoaExecutorWorkerError> { - let mut health = self.get_eoa_health(scoped, chain).await?; - - tracing::info!("Updating balance threshold"); - let balance_threshold = chain - .provider() - .get_balance(scoped.eoa()) - .await - .map_err(|e| { - let engine_error = e.to_engine_error(chain); - EoaExecutorWorkerError::RpcError { - message: format!("Failed to get balance: {}", engine_error), - inner_error: engine_error, - } - })?; - - health.balance_threshold = balance_threshold; - scoped.update_health_data(&health).await?; - Ok(()) - } - - // ========== CONFIRMATION FLOW HELPERS ========== - - /// Get pending transactions below the given nonce - async fn get_pending_transactions_below_nonce( - &self, - scoped: &ScopedEoaExecutorStore<'_>, - nonce: u64, - ) -> Result, EoaExecutorWorkerError> { - let pending_hashes = scoped.get_hashes_below_nonce(nonce).await?; - - let pending_txs = pending_hashes - .into_iter() - .map(|(nonce, hash, transaction_id)| PendingTransaction { - nonce, - hash, - transaction_id, - }) - .collect(); - - Ok(pending_txs) - } - - /// Fetch receipts for all pending transactions and categorize them - async fn fetch_and_categorize_transactions( - &self, - chain: &impl Chain, - pending_txs: Vec, - ) -> (Vec, Vec) { - // Fetch all receipts in parallel - let receipt_futures: Vec<_> = pending_txs - .iter() - .filter_map(|tx| match tx.hash.parse::() { - Ok(hash_bytes) => Some(async move { - let receipt = chain.provider().get_transaction_receipt(hash_bytes).await; - (tx, receipt) - }), - Err(_) => { - tracing::warn!("Invalid hash format: {}, skipping", tx.hash); - None - } - }) - .collect(); - - let receipt_results = futures::future::join_all(receipt_futures).await; - - // Categorize transactions - let mut confirmed_txs = Vec::new(); - let mut failed_txs = Vec::new(); - - for (tx, receipt_result) in receipt_results { - match receipt_result { - Ok(Some(receipt)) => { - confirmed_txs.push(ConfirmedTransaction { - nonce: tx.nonce, - hash: tx.hash.clone(), - transaction_id: tx.transaction_id.clone(), - receipt, - }); - } - Ok(None) | Err(_) => { - failed_txs.push(FailedTransaction { - hash: tx.hash.clone(), - transaction_id: tx.transaction_id.clone(), - }); - } - } - } - - (confirmed_txs, failed_txs) - } - - // ========== HELPER METHODS ========== - async fn estimate_gas_fees( - &self, - chain: &impl Chain, - tx: AlloyTransactionRequest, - ) -> Result { - // Check what fees are missing and need to be estimated - - // If we have gas_price set, we're doing legacy - don't estimate EIP-1559 - if tx.gas_price.is_some() { - return Ok(tx); - } - - // If we have both EIP-1559 fees set, don't estimate - if tx.max_fee_per_gas.is_some() && tx.max_priority_fee_per_gas.is_some() { - return Ok(tx); - } - - // Try EIP-1559 fees first, fall back to legacy if unsupported - match chain.provider().estimate_eip1559_fees().await { - Ok(eip1559_fees) => { - tracing::debug!( - "Using EIP-1559 fees: max_fee={}, max_priority_fee={}", - eip1559_fees.max_fee_per_gas, - eip1559_fees.max_priority_fee_per_gas - ); - - let mut result = tx; - // Only set fees that are missing - if result.max_fee_per_gas.is_none() { - result = result.with_max_fee_per_gas(eip1559_fees.max_fee_per_gas); - } - if result.max_priority_fee_per_gas.is_none() { - result = - result.with_max_priority_fee_per_gas(eip1559_fees.max_priority_fee_per_gas); - } - - Ok(result) - } - Err(eip1559_error) => { - // Check if this is an "unsupported feature" error - if let RpcError::UnsupportedFeature(_) = &eip1559_error { - tracing::debug!("EIP-1559 not supported, falling back to legacy gas price"); - - // Fall back to legacy gas price only if no gas price is set - if tx.authorization_list().is_none() { - match chain.provider().get_gas_price().await { - Ok(gas_price) => { - tracing::debug!("Using legacy gas price: {}", gas_price); - Ok(tx.with_gas_price(gas_price)) - } - Err(legacy_error) => Err(EoaExecutorWorkerError::RpcError { - message: format!( - "Failed to get legacy gas price: {}", - legacy_error - ), - inner_error: legacy_error.to_engine_error(chain), - }), - } - } else { - Err(EoaExecutorWorkerError::TransactionBuildFailed { - message: "EIP7702 transactions not supported on chain".to_string(), - }) - } - } else { - // Other EIP-1559 error - Err(EoaExecutorWorkerError::RpcError { - message: format!("Failed to estimate EIP-1559 fees: {}", eip1559_error), - inner_error: eip1559_error.to_engine_error(chain), - }) - } - } - } - } - - async fn build_typed_transaction( - &self, - tx_data: &TransactionData, - nonce: u64, - chain: &impl Chain, - ) -> Result { - // Build transaction request from stored data - let mut tx_request = AlloyTransactionRequest::default() - .with_from(tx_data.user_request.from) - .with_value(tx_data.user_request.value) - .with_input(tx_data.user_request.data.clone()) - .with_chain_id(tx_data.user_request.chain_id) - .with_nonce(nonce); - - if let Some(to) = tx_data.user_request.to { - tx_request = tx_request.with_to(to); - } - - if let Some(gas_limit) = tx_data.user_request.gas_limit { - tx_request = tx_request.with_gas_limit(gas_limit); - } - - // Handle gas fees - either from user settings or estimation - tx_request = if let Some(type_data) = &tx_data.user_request.transaction_type_data { - // User provided gas settings - respect them first - match type_data { - TransactionTypeData::Eip1559(data) => { - let mut req = tx_request; - if let Some(max_fee) = data.max_fee_per_gas { - req = req.with_max_fee_per_gas(max_fee); - } - if let Some(max_priority) = data.max_priority_fee_per_gas { - req = req.with_max_priority_fee_per_gas(max_priority); - } - - // if either not set, estimate the other one - if req.max_fee_per_gas.is_none() || req.max_priority_fee_per_gas.is_none() { - req = self.estimate_gas_fees(chain, req).await?; - } - - req - } - TransactionTypeData::Legacy(data) => { - if let Some(gas_price) = data.gas_price { - tx_request.with_gas_price(gas_price) - } else { - // User didn't provide gas price, estimate it - self.estimate_gas_fees(chain, tx_request).await? - } - } - TransactionTypeData::Eip7702(data) => { - let mut req = tx_request; - if let Some(authorization_list) = &data.authorization_list { - req = req.with_authorization_list(authorization_list.clone()); - } - if let Some(max_fee) = data.max_fee_per_gas { - req = req.with_max_fee_per_gas(max_fee); - } - if let Some(max_priority) = data.max_priority_fee_per_gas { - req = req.with_max_priority_fee_per_gas(max_priority); - } - - // if either not set, estimate the other one - if req.max_fee_per_gas.is_none() || req.max_priority_fee_per_gas.is_none() { - req = self.estimate_gas_fees(chain, req).await?; - } - - req - } - } - } else { - // No user settings - estimate appropriate fees - self.estimate_gas_fees(chain, tx_request).await? - }; - - // Estimate gas if needed - if tx_request.gas.is_none() { - match chain.provider().estimate_gas(tx_request.clone()).await { - Ok(gas_limit) => { - tx_request = tx_request.with_gas_limit(gas_limit * 110 / 100); // 10% buffer - } - Err(e) => { - // Check if this is a revert - if let RpcError::ErrorResp(error_payload) = &e { - if let Some(revert_data) = error_payload.as_revert_data() { - // This is a revert - the transaction is fundamentally broken - // This should fail the individual transaction, not the worker - return Err(EoaExecutorWorkerError::TransactionSimulationFailed { - message: format!( - "Transaction reverted during gas estimation: {} (revert: {})", - error_payload.message, - hex::encode(&revert_data) - ), - inner_error: e.to_engine_error(chain), - }); - } - } - - // Not a revert - could be RPC issue, this should nack the worker - let engine_error = e.to_engine_error(chain); - return Err(EoaExecutorWorkerError::RpcError { - message: format!("Gas estimation failed: {}", engine_error), - inner_error: engine_error, - }); - } - } - } - - // Build typed transaction - tx_request - .build_typed_tx() - .map_err(|e| EoaExecutorWorkerError::TransactionBuildFailed { - message: format!("Failed to build typed transaction: {:?}", e), - }) - } - - async fn sign_transaction( - &self, - typed_tx: TypedTransaction, - tx_data: &TransactionData, - ) -> Result, EoaExecutorWorkerError> { - let signing_options = EoaSigningOptions { - from: tx_data.user_request.from, - chain_id: Some(tx_data.user_request.chain_id), - }; - - let signature = self - .eoa_signer - .sign_transaction( - signing_options, - typed_tx.clone(), - tx_data.user_request.signing_credential.clone(), - ) - .await - .map_err(|engine_error| EoaExecutorWorkerError::SigningError { - message: format!("Failed to sign transaction: {}", engine_error), - inner_error: engine_error, - })?; - - let signature = signature.parse::().map_err(|e| { - EoaExecutorWorkerError::SignatureParsingFailed { - message: format!("Failed to parse signature: {}", e), - } - })?; - - Ok(typed_tx.into_signed(signature)) - } - - async fn build_and_sign_transaction( - &self, - tx_data: &TransactionData, - nonce: u64, - chain: &impl Chain, - ) -> Result, EoaExecutorWorkerError> { - let typed_tx = self.build_typed_transaction(tx_data, nonce, chain).await?; - self.sign_transaction(typed_tx, tx_data).await - } - - fn apply_gas_bump_to_typed_transaction( - &self, - mut typed_tx: TypedTransaction, - bump_multiplier: u32, // e.g., 120 for 20% increase - ) -> TypedTransaction { - match &mut typed_tx { - TypedTransaction::Eip1559(tx) => { - tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; - tx.max_priority_fee_per_gas = - tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; - } - TypedTransaction::Legacy(tx) => { - tx.gas_price = tx.gas_price * bump_multiplier as u128 / 100; - } - TypedTransaction::Eip2930(tx) => { - tx.gas_price = tx.gas_price * bump_multiplier as u128 / 100; - } - TypedTransaction::Eip7702(tx) => { - tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; - tx.max_priority_fee_per_gas = - tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; - } - TypedTransaction::Eip4844(tx) => match tx { - TxEip4844Variant::TxEip4844(tx) => { - tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; - tx.max_priority_fee_per_gas = - tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; - } - TxEip4844Variant::TxEip4844WithSidecar(TxEip4844WithSidecar { tx, .. }) => { - tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; - tx.max_priority_fee_per_gas = - tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; - } - }, - } - typed_tx - } -} diff --git a/executors/src/eoa/worker/confirm.rs b/executors/src/eoa/worker/confirm.rs new file mode 100644 index 0000000..def76d5 --- /dev/null +++ b/executors/src/eoa/worker/confirm.rs @@ -0,0 +1,360 @@ +use alloy::{primitives::B256, providers::Provider}; +use engine_core::{chain::Chain, error::AlloyRpcErrorToEngineError}; +use serde::{Deserialize, Serialize}; + +use crate::eoa::{ + store::{ + CleanupReport, ConfirmedTransaction, ReplacedTransaction, SubmittedTransaction, + SubmittedTransactionDehydrated, TransactionData, TransactionStoreError, + }, + worker::{ + EoaExecutorWorker, + error::{EoaExecutorWorkerError, should_update_balance_threshold}, + }, +}; + +const NONCE_STALL_TIMEOUT: u64 = 300_000; // 5 minutes in milliseconds - after this time, attempt gas bump + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConfirmedTransactionWithRichReceipt { + pub nonce: u64, + pub hash: String, + pub transaction_id: String, + pub receipt: alloy::rpc::types::TransactionReceipt, +} + +impl EoaExecutorWorker { + // ========== CONFIRM FLOW ========== + #[tracing::instrument(skip_all)] + pub async fn confirm_flow(&self) -> Result { + // Get fresh on-chain transaction count + let current_chain_transaction_count = self + .chain + .provider() + .get_transaction_count(self.eoa) + .await + .map_err(|e| { + let engine_error = e.to_engine_error(&self.chain); + EoaExecutorWorkerError::RpcError { + message: format!("Failed to get transaction count: {}", engine_error), + inner_error: engine_error, + } + })?; + + let cached_transaction_count = match self.store.get_cached_transaction_count().await { + Err(e) => match e { + TransactionStoreError::NonceSyncRequired { .. } => { + self.store + .reset_nonces(current_chain_transaction_count) + .await?; + current_chain_transaction_count + } + _ => return Err(e.into()), + }, + Ok(cached_nonce) => cached_nonce, + }; + + let submitted_count = self.store.get_submitted_transactions_count().await?; + + // no nonce progress + if current_chain_transaction_count <= cached_transaction_count { + let current_health = self.get_eoa_health().await?; + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + // No nonce progress - check if we should attempt gas bumping for stalled nonce + let time_since_movement = now.saturating_sub(current_health.last_nonce_movement_at); + + // if there are waiting transactions, we can attempt a gas bump + if time_since_movement > NONCE_STALL_TIMEOUT && submitted_count > 0 { + tracing::info!( + time_since_movement = time_since_movement, + stall_timeout = NONCE_STALL_TIMEOUT, + current_chain_nonce = current_chain_transaction_count, + "Nonce has been stalled, attempting gas bump" + ); + + // Attempt gas bump for the next expected nonce + if let Err(e) = self + .attempt_gas_bump_for_stalled_nonce(current_chain_transaction_count) + .await + { + tracing::warn!( + error = %e, + "Failed to attempt gas bump for stalled nonce" + ); + } + } + + tracing::debug!("No nonce progress, skipping confirm flow"); + return Ok(CleanupReport::default()); + } + + tracing::info!( + current_chain_nonce = current_chain_transaction_count, + cached_nonce = cached_transaction_count, + "Processing confirmations" + ); + + // Get all pending transactions below the current chain transaction count + // ie, if transaction count is 1, nonce 0 should have mined + let waiting_txs = self + .store + .get_submitted_transactions_below_chain_transaction_count( + current_chain_transaction_count, + ) + .await?; + + if waiting_txs.is_empty() { + tracing::debug!("No waiting transactions to confirm"); + return Ok(CleanupReport::default()); + } + + // Fetch receipts and categorize transactions + let (confirmed_txs, replaced_txs) = + self.fetch_confirmed_transaction_receipts(waiting_txs).await; + + // Process confirmed transactions + let successes: Vec = confirmed_txs + .into_iter() + .map(|tx| { + let receipt_data = match serde_json::to_string(&tx.receipt) { + Ok(receipt_json) => receipt_json, + Err(e) => { + tracing::warn!( + transaction_id = %tx.transaction_id, + hash = %tx.hash, + error = %e, + "Failed to serialize receipt as JSON, using debug format" + ); + format!("{:?}", tx.receipt) + } + }; + + tracing::info!( + transaction_id = %tx.transaction_id, + nonce = tx.nonce, + hash = %tx.hash, + "Transaction confirmed" + ); + + ConfirmedTransaction { + hash: tx.hash, + transaction_id: tx.transaction_id, + receipt: tx.receipt.into(), + receipt_serialized: receipt_data, + } + }) + .collect(); + + let report = self + .store + .clean_submitted_transactions( + &successes, + current_chain_transaction_count - 1, + self.webhook_queue.clone(), + ) + .await?; + + Ok(report) + } + + /// Fetch receipts for all submitted transactions and categorize them + async fn fetch_confirmed_transaction_receipts( + &self, + submitted_txs: Vec, + ) -> ( + Vec, + Vec, + ) { + // Fetch all receipts in parallel + let receipt_futures: Vec<_> = submitted_txs + .iter() + .filter_map(|tx| match tx.hash.parse::() { + Ok(hash_bytes) => Some(async move { + let receipt = self + .chain + .provider() + .get_transaction_receipt(hash_bytes) + .await; + (tx, receipt) + }), + Err(_) => { + tracing::warn!("Invalid hash format: {}, skipping", tx.hash); + None + } + }) + .collect(); + + let receipt_results = futures::future::join_all(receipt_futures).await; + + // Categorize transactions + let mut confirmed_txs = Vec::new(); + let mut failed_txs = Vec::new(); + + for (tx, receipt_result) in receipt_results { + match receipt_result { + Ok(Some(receipt)) => { + confirmed_txs.push(ConfirmedTransactionWithRichReceipt { + nonce: tx.nonce, + hash: tx.hash.clone(), + transaction_id: tx.transaction_id.clone(), + receipt, + }); + } + Ok(None) | Err(_) => { + failed_txs.push(ReplacedTransaction { + hash: tx.hash.clone(), + transaction_id: tx.transaction_id.clone(), + }); + } + } + } + + (confirmed_txs, failed_txs) + } + + // ========== GAS BUMP METHODS ========== + + /// Attempt to gas bump a stalled transaction for the next expected nonce + async fn attempt_gas_bump_for_stalled_nonce( + &self, + expected_nonce: u64, + ) -> Result { + tracing::info!( + nonce = expected_nonce, + "Attempting gas bump for stalled nonce" + ); + + // Get all transaction IDs for this nonce + let submitted_transactions = self + .store + .get_submitted_transactions_for_nonce(expected_nonce) + .await?; + + if submitted_transactions.is_empty() { + tracing::debug!( + nonce = expected_nonce, + "No transactions found for stalled nonce, sending noop" + ); + + let noop_tx = self.send_noop_transaction(expected_nonce).await?; + self.store.process_noop_transactions(&[noop_tx]).await?; + return Ok(true); + } + + // Load transaction data for all IDs and find the newest one + let mut newest_transaction: Option<(String, TransactionData)> = None; + let mut newest_submitted_at = 0u64; + + for SubmittedTransactionDehydrated { transaction_id, .. } in submitted_transactions { + if let Some(tx_data) = self.store.get_transaction_data(&transaction_id).await? { + // Find the most recent attempt for this transaction + if let Some(latest_attempt) = tx_data.attempts.last() { + let submitted_at = latest_attempt.sent_at; + if submitted_at > newest_submitted_at { + newest_submitted_at = submitted_at; + newest_transaction = Some((transaction_id, tx_data)); + } + } + } + } + + if let Some((transaction_id, tx_data)) = newest_transaction { + tracing::info!( + transaction_id = %transaction_id, + nonce = expected_nonce, + "Found newest transaction for gas bump" + ); + + // Get the latest attempt to extract gas values from + // Build typed transaction -> manually bump -> sign + let typed_tx = match self + .build_typed_transaction(&tx_data.user_request, expected_nonce) + .await + { + Ok(tx) => tx, + Err(e) => { + // Check if this is a balance threshold issue during simulation + if let EoaExecutorWorkerError::TransactionSimulationFailed { + inner_error, .. + } = &e + { + if should_update_balance_threshold(inner_error) { + if let Err(e) = self.update_balance_threshold().await { + tracing::error!("Failed to update balance threshold: {}", e); + } + } + } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { + if should_update_balance_threshold(inner_error) { + if let Err(e) = self.update_balance_threshold().await { + tracing::error!("Failed to update balance threshold: {}", e); + } + } + } + + tracing::warn!( + transaction_id = %transaction_id, + nonce = expected_nonce, + error = %e, + "Failed to build typed transaction for gas bump" + ); + return Ok(false); + } + }; + let bumped_typed_tx = self.apply_gas_bump_to_typed_transaction(typed_tx, 120); // 20% increase + let bumped_tx = match self + .sign_transaction(bumped_typed_tx, &tx_data.user_request.signing_credential) + .await + { + Ok(tx) => tx, + Err(e) => { + tracing::warn!( + transaction_id = %transaction_id, + nonce = expected_nonce, + error = %e, + "Failed to sign transaction for gas bump" + ); + return Ok(false); + } + }; + + // Record the gas bump attempt + self.store + .add_gas_bump_attempt( + &SubmittedTransactionDehydrated { + nonce: expected_nonce, + hash: bumped_tx.hash().to_string(), + transaction_id: transaction_id.to_string(), + queued_at: tx_data.created_at, + }, + bumped_tx.clone(), + ) + .await?; + + // Send the bumped transaction + let tx_envelope = bumped_tx.into(); + match self.chain.provider().send_tx_envelope(tx_envelope).await { + Ok(_) => { + tracing::info!( + transaction_id = %transaction_id, + nonce = expected_nonce, + "Successfully sent gas bumped transaction" + ); + return Ok(true); + } + Err(e) => { + tracing::warn!( + transaction_id = %transaction_id, + nonce = expected_nonce, + error = %e, + "Failed to send gas bumped transaction" + ); + // Don't fail the worker, just log the error + return Ok(false); + } + } + } + + Ok(false) + } +} diff --git a/executors/src/eoa/worker/error.rs b/executors/src/eoa/worker/error.rs new file mode 100644 index 0000000..45d39b4 --- /dev/null +++ b/executors/src/eoa/worker/error.rs @@ -0,0 +1,262 @@ +use alloy::transports::{RpcError, TransportErrorKind}; +use engine_core::{ + chain::Chain, + error::{AlloyRpcErrorToEngineError, EngineError, RpcErrorKind}, +}; +use serde::{Deserialize, Serialize}; +use thirdweb_core::iaw::IAWError; +use twmq::{UserCancellable, error::TwmqError}; + +use crate::eoa::{ + EoaTransactionRequest, + store::{ + BorrowedTransaction, BorrowedTransactionData, SubmissionResult, SubmissionResultType, + SubmittedTransaction, TransactionStoreError, + }, + worker::EoaExecutorWorkerResult, +}; + +#[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE", tag = "errorCode")] +pub enum EoaExecutorWorkerError { + #[error("Chain service error for chainId {chain_id}: {message}")] + ChainServiceError { chain_id: u64, message: String }, + + #[error("Store error: {message}")] + StoreError { + message: String, + inner_error: TransactionStoreError, + }, + + #[error("Transaction not found: {transaction_id}")] + TransactionNotFound { transaction_id: String }, + + #[error("Transaction simulation failed: {message}")] + TransactionSimulationFailed { + message: String, + inner_error: EngineError, + }, + + #[error("Transaction build failed: {message}")] + TransactionBuildFailed { message: String }, + + #[error("RPC error encountered during generic operation: {message}")] + RpcError { + message: String, + inner_error: EngineError, + }, + + #[error("Error encountered when broadcasting transaction: {message}")] + TransactionSendError { + message: String, + inner_error: EngineError, + }, + + #[error("Signature parsing failed: {message}")] + SignatureParsingFailed { message: String }, + + #[error("Transaction signing failed: {message}")] + SigningError { + message: String, + inner_error: EngineError, + }, + + #[error("Work still remaining: {result:?}")] + WorkRemaining { result: EoaExecutorWorkerResult }, + + #[error("Internal error: {message}")] + InternalError { message: String }, + + #[error("User cancelled")] + UserCancelled, +} + +impl From for EoaExecutorWorkerError { + fn from(error: TwmqError) -> Self { + EoaExecutorWorkerError::InternalError { + message: format!("Queue error: {}", error), + } + } +} + +impl From for EoaExecutorWorkerError { + fn from(error: TransactionStoreError) -> Self { + EoaExecutorWorkerError::StoreError { + message: error.to_string(), + inner_error: error, + } + } +} + +impl UserCancellable for EoaExecutorWorkerError { + fn user_cancelled() -> Self { + EoaExecutorWorkerError::UserCancelled + } +} + +// ========== SIMPLE ERROR CLASSIFICATION ========== +#[derive(Debug)] +pub enum SendErrorClassification { + PossiblySent, // "nonce too low", "already known" etc + DeterministicFailure, // Invalid signature, malformed tx, insufficient funds etc +} + +#[derive(PartialEq, Eq, Debug)] +pub enum SendContext { + Rebroadcast, + InitialBroadcast, +} + +#[tracing::instrument(skip_all, fields(error = %error, context = ?context))] +pub fn classify_send_error( + error: &RpcError, + context: SendContext, +) -> SendErrorClassification { + if !error.is_error_resp() { + return SendErrorClassification::DeterministicFailure; + } + + let error_str = error.to_string().to_lowercase(); + + // Deterministic failures that didn't consume nonce (spec-compliant) + if error_str.contains("invalid signature") + || error_str.contains("malformed transaction") + || (context == SendContext::InitialBroadcast && error_str.contains("insufficient funds")) + || error_str.contains("invalid transaction format") + || error_str.contains("nonce too high") + // Should trigger nonce reset + { + return SendErrorClassification::DeterministicFailure; + } + + // Transaction possibly made it to mempool (spec-compliant) + if error_str.contains("nonce too low") + || error_str.contains("already known") + || error_str.contains("replacement transaction underpriced") + || error_str.contains("transaction already imported") + { + return SendErrorClassification::PossiblySent; + } + + // Additional common failures that didn't consume nonce + if error_str.contains("malformed") + || error_str.contains("gas limit") + || error_str.contains("intrinsic gas too low") + { + return SendErrorClassification::DeterministicFailure; + } + + tracing::warn!( + "Unknown send error: {}. PLEASE REPORT FOR ADDING CORRECT CLASSIFICATION [NOTIFY]", + error_str + ); + + // Default: assume possibly sent for safety + SendErrorClassification::PossiblySent +} + +pub fn should_trigger_nonce_reset(error: &RpcError) -> bool { + let error_str = error.to_string().to_lowercase(); + + // "nonce too high" should trigger nonce reset as per spec + error_str.contains("nonce too high") +} + +pub fn should_update_balance_threshold(error: &EngineError) -> bool { + match error { + EngineError::RpcError { kind, .. } + | EngineError::PaymasterError { kind, .. } + | EngineError::BundlerError { kind, .. } => match kind { + RpcErrorKind::ErrorResp(resp) => { + let message = resp.message.to_lowercase(); + message.contains("insufficient funds") + || message.contains("insufficient balance") + || message.contains("out of gas") + || message.contains("insufficient eth") + || message.contains("balance too low") + || message.contains("not enough funds") + || message.contains("insufficient native token") + } + _ => false, + }, + _ => false, + } +} + +pub fn is_retryable_rpc_error(kind: &RpcErrorKind) -> bool { + match kind { + RpcErrorKind::TransportHttpError { status, .. } if *status >= 400 && *status < 500 => false, + RpcErrorKind::UnsupportedFeature { .. } => false, + _ => true, + } +} + +pub fn is_retryable_preparation_error(error: &EoaExecutorWorkerError) -> bool { + match error { + EoaExecutorWorkerError::RpcError { inner_error, .. } => { + // extract the RpcErrorKind from the inner error + if let EngineError::RpcError { kind, .. } = inner_error { + is_retryable_rpc_error(kind) + } else { + false + } + } + EoaExecutorWorkerError::ChainServiceError { .. } => true, // Network related + EoaExecutorWorkerError::StoreError { inner_error, .. } => { + matches!(inner_error, TransactionStoreError::RedisError { .. }) + } + EoaExecutorWorkerError::TransactionSimulationFailed { .. } => false, // Deterministic + EoaExecutorWorkerError::TransactionBuildFailed { .. } => false, // Deterministic + EoaExecutorWorkerError::SigningError { inner_error, .. } => match inner_error { + // if vault error, it's not retryable + EngineError::VaultError { .. } => false, + // if iaw error, it's retryable only if it's a network error + EngineError::IawError { error, .. } => matches!(error, IAWError::NetworkError { .. }), + _ => false, + }, + EoaExecutorWorkerError::TransactionNotFound { .. } => false, // Deterministic + EoaExecutorWorkerError::InternalError { .. } => false, // Deterministic + EoaExecutorWorkerError::UserCancelled => false, // Deterministic + EoaExecutorWorkerError::TransactionSendError { .. } => false, // Different context + EoaExecutorWorkerError::SignatureParsingFailed { .. } => false, // Deterministic + EoaExecutorWorkerError::WorkRemaining { .. } => false, // Different context + } +} + +impl SubmissionResult { + /// Convert a send result to a SubmissionResult for batch processing + /// This handles the specific RpcError type from alloy + pub fn from_send_result( + borrowed_transaction: &BorrowedTransaction, + send_result: Result>, + send_context: SendContext, + chain: &impl Chain, + ) -> Self { + match send_result { + Ok(_) => SubmissionResult { + result: SubmissionResultType::Success, + transaction: borrowed_transaction.clone().into(), + }, + Err(ref rpc_error) => { + match classify_send_error(rpc_error, send_context) { + SendErrorClassification::PossiblySent => SubmissionResult { + result: SubmissionResultType::Success, + transaction: borrowed_transaction.clone().into(), + }, + SendErrorClassification::DeterministicFailure => { + // Transaction failed, should be retried + let engine_error = rpc_error.to_engine_error(chain); + let error = EoaExecutorWorkerError::TransactionSendError { + message: format!("Transaction send failed: {}", rpc_error), + inner_error: engine_error, + }; + SubmissionResult { + result: SubmissionResultType::Nack(error), + transaction: borrowed_transaction.clone().into(), + } + } + } + } + } + } +} diff --git a/executors/src/eoa/worker/mod.rs b/executors/src/eoa/worker/mod.rs new file mode 100644 index 0000000..f73ad36 --- /dev/null +++ b/executors/src/eoa/worker/mod.rs @@ -0,0 +1,499 @@ +use alloy::consensus::Transaction; +use alloy::primitives::{Address, U256}; +use alloy::providers::Provider; +use engine_core::{ + chain::{Chain, ChainService}, + credentials::SigningCredential, + error::AlloyRpcErrorToEngineError, + signer::EoaSigner, +}; +use serde::{Deserialize, Serialize}; +use std::{sync::Arc, time::Duration}; +use twmq::Queue; +use twmq::redis::AsyncCommands; +use twmq::redis::aio::ConnectionManager; +use twmq::{ + DurableExecution, FailHookData, NackHookData, SuccessHookData, + hooks::TransactionContext, + job::{BorrowedJob, JobResult, RequeuePosition, ToJobResult}, +}; + +use crate::eoa::store::{ + AtomicEoaExecutorStore, EoaExecutorStore, EoaExecutorStoreKeys, EoaHealth, SubmissionResult, +}; +use crate::webhook::WebhookJobHandler; + +pub mod confirm; +pub mod error; +mod send; +mod transaction; + +use error::{EoaExecutorWorkerError, SendContext}; + +// ========== SPEC-COMPLIANT CONSTANTS ========== +const MAX_INFLIGHT_PER_EOA: u64 = 100; // Default from spec +const MAX_RECYCLED_THRESHOLD: u64 = 50; // Circuit breaker from spec +const TARGET_TRANSACTIONS_PER_EOA: u64 = 10; // Fleet management from spec +const MIN_TRANSACTIONS_PER_EOA: u64 = 1; // Fleet management from spec + +// ========== JOB DATA ========== +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct EoaExecutorWorkerJobData { + pub eoa_address: Address, + pub chain_id: u64, + pub noop_signing_credential: SigningCredential, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct EoaExecutorWorkerResult { + // what we did + /// Number of transactions we recovered from borrowed state + pub recovered_transactions: u32, + + /// Number of transactions we confirmed + pub confirmed_transactions: u32, + + /// Number of transactions we failed due to deterministic errors + pub failed_transactions: u32, + + /// Number of transactions we sent + pub sent_transactions: u32, + + /// Number of transactions that got replaced in the mempool and are now pending + pub replaced_transactions: u32, + + // what we have left + /// Number of transactions currently in the submitted state + pub submitted_transactions: u32, + + /// Number of transactions currently in the pending state + pub pending_transactions: u32, + + /// Number of transactions currently in the borrowed state + pub borrowed_transactions: u32, + + /// Number of recycled nonces + pub recycled_nonces: u32, +} + +impl EoaExecutorWorkerResult { + pub fn is_work_remaining(&self) -> bool { + self.pending_transactions > 0 + || self.borrowed_transactions > 0 + || self.recycled_nonces > 0 + || self.submitted_transactions > 0 + } +} + +// ========== MAIN WORKER ========== +/// EOA Executor Worker +/// +/// ## Core Workflow: +/// 1. **Acquire Lock Aggressively** - Takes over stalled workers using force acquisition. This is a lock over EOA:CHAIN +/// 2. **Crash Recovery** - Rebroadcasts borrowed transactions, handles deterministic failures +/// 3. **Confirmation Flow** - Fetches receipts, confirms transactions, handles nonce sync, requeues replaced transactions +/// 4. **Send Flow** - Processes recycled nonces first, then new transactions with in-flight budget control +/// 5. **Lock Release** - Explicit release in finally pattern as per spec +/// +/// ## Key Features: +/// - **Atomic Operations**: All state transitions use Redis WATCH/MULTI/EXEC for durability +/// - **Borrowed State**: Mid-send crash recovery with atomic pending->borrowed->submitted transitions +/// - **Nonce Management**: Optimistic nonce tracking with recycled nonce priority +/// - **Error Classification**: Spec-compliant deterministic vs. possibly-sent error handling +/// - **Circuit Breakers**: Automatic recycled nonce nuking when threshold exceeded +/// - **Health Monitoring**: Balance checking with configurable thresholds +pub struct EoaExecutorJobHandler +where + CS: ChainService + Send + Sync + 'static, +{ + pub chain_service: Arc, + pub webhook_queue: Arc>, + + pub redis: ConnectionManager, + pub namespace: Option, + + pub eoa_signer: Arc, + pub max_inflight: u64, // Note: Spec uses MAX_INFLIGHT_PER_EOA constant + pub max_recycled_nonces: u64, // Note: Spec uses MAX_RECYCLED_THRESHOLD constant +} + +impl DurableExecution for EoaExecutorJobHandler +where + CS: ChainService + Send + Sync + 'static, +{ + type Output = EoaExecutorWorkerResult; + type ErrorData = EoaExecutorWorkerError; + type JobData = EoaExecutorWorkerJobData; + + #[tracing::instrument(skip_all, fields(eoa = %job.job.data.eoa_address, chain_id = job.job.data.chain_id))] + async fn process( + &self, + job: &BorrowedJob, + ) -> JobResult { + let data = &job.job.data; + + // 1. GET CHAIN + let chain = self + .chain_service + .get_chain(data.chain_id) + .map_err(|e| EoaExecutorWorkerError::ChainServiceError { + chain_id: data.chain_id, + message: format!("Failed to get chain: {}", e), + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; + + // 2. CREATE SCOPED STORE (acquires lock) + let scoped = EoaExecutorStore::new( + self.redis.clone(), + self.namespace.clone(), + data.eoa_address, + data.chain_id, + ) + .acquire_eoa_lock_aggressively(&job.lease_token) + .await + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; + + let worker = EoaExecutorWorker { + store: scoped, + chain, + eoa: data.eoa_address, + chain_id: data.chain_id, + noop_signing_credential: data.noop_signing_credential.clone(), + + max_inflight: self.max_inflight, + max_recycled_nonces: self.max_recycled_nonces, + webhook_queue: self.webhook_queue.clone(), + signer: self.eoa_signer.clone(), + }; + + let result = worker.execute_main_workflow().await?; + worker.release_eoa_lock().await; + + if result.is_work_remaining() { + Err(EoaExecutorWorkerError::WorkRemaining { result }) + .map_err_nack(Some(Duration::from_secs(2)), RequeuePosition::Last) + } else { + Ok(result) + } + + // // initiate health data if doesn't exist + // self.get_eoa_health(&scoped, &chain) + // .await + // .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; + + // // Execute main workflow with proper error handling + // self.execute_main_workflow(&scoped, &chain).await + } + + async fn on_success( + &self, + job: &BorrowedJob, + _success_data: SuccessHookData<'_, Self::Output>, + _tx: &mut TransactionContext<'_>, + ) { + self.soft_release_eoa_lock(&job.job.data).await; + } + + async fn on_nack( + &self, + job: &BorrowedJob, + _nack_data: NackHookData<'_, Self::ErrorData>, + _tx: &mut TransactionContext<'_>, + ) { + self.soft_release_eoa_lock(&job.job.data).await; + } + + async fn on_fail( + &self, + job: &BorrowedJob, + _fail_data: FailHookData<'_, Self::ErrorData>, + _tx: &mut TransactionContext<'_>, + ) { + self.soft_release_eoa_lock(&job.job.data).await; + } +} + +impl EoaExecutorJobHandler +where + CS: ChainService + Send + Sync + 'static, +{ + async fn soft_release_eoa_lock(&self, job_data: &EoaExecutorWorkerJobData) { + let keys = EoaExecutorStoreKeys::new( + job_data.eoa_address, + job_data.chain_id, + self.namespace.clone(), + ); + + let lock_key = keys.eoa_lock_key_name(); + let mut conn = self.redis.clone(); + if let Err(e) = conn.del::<&str, ()>(&lock_key).await { + tracing::error!( + eoa = %job_data.eoa_address, + chain_id = %job_data.chain_id, + error = %e, + "Failed to release EOA lock" + ); + } + } +} + +pub struct EoaExecutorWorker { + pub store: AtomicEoaExecutorStore, + pub chain: C, + + pub eoa: Address, + pub chain_id: u64, + pub noop_signing_credential: SigningCredential, + + pub max_inflight: u64, + pub max_recycled_nonces: u64, + + pub webhook_queue: Arc>, + pub signer: Arc, +} + +impl EoaExecutorWorker { + /// Execute the main EOA worker workflow + async fn execute_main_workflow( + &self, + ) -> JobResult { + // 1. CRASH RECOVERY + let recovered = self + .recover_borrowed_state() + .await + .map_err(|e| { + tracing::error!("Error in recover_borrowed_state: {}", e); + e + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; + + // 2. CONFIRM FLOW + let confirmations_report = self + .confirm_flow() + .await + .map_err(|e| { + tracing::error!("Error in confirm flow: {}", e); + e + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; + + // 3. SEND FLOW + let sent = self + .send_flow() + .await + .map_err(|e| { + tracing::error!("Error in send_flow: {}", e); + e + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; + + // 4. CHECK FOR REMAINING WORK + let pending_count = self + .store + .peek_pending_transactions(1000) + .await + .map_err(|e| { + tracing::error!("Error in peek_pending_transactions: {}", e); + e + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)? + .len(); + let borrowed_count = self + .store + .peek_borrowed_transactions() + .await + .map_err(|e| { + tracing::error!("Error in peek_borrowed_transactions: {}", e); + e + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)? + .len(); + let recycled_count = self + .store + .peek_recycled_nonces() + .await + .map_err(|e| { + tracing::error!("Error in peek_recycled_nonces: {}", e); + e + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)? + .len(); + let submitted_count = self + .store + .get_submitted_transactions_count() + .await + .map_err(|e| { + tracing::error!("Error in get_submitted_transactions_count: {}", e); + e + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; + + Ok(EoaExecutorWorkerResult { + recovered_transactions: recovered, + confirmed_transactions: confirmations_report.moved_to_success as u32, + failed_transactions: confirmations_report.moved_to_pending as u32, + sent_transactions: sent, + + replaced_transactions: confirmations_report.moved_to_pending as u32, + submitted_transactions: submitted_count as u32, + pending_transactions: pending_count as u32, + borrowed_transactions: borrowed_count as u32, + recycled_nonces: recycled_count as u32, + }) + } + + // ========== CRASH RECOVERY ========== + #[tracing::instrument(skip_all)] + async fn recover_borrowed_state(&self) -> Result { + let borrowed_transactions = self.store.peek_borrowed_transactions().await?; + let mut borrowed_transactions = self.store.hydrate_all(borrowed_transactions).await?; + + if borrowed_transactions.is_empty() { + return Ok(0); + } + + tracing::warn!( + "Recovering {} borrowed transactions. This indicates a worker crash or system issue", + borrowed_transactions.len() + ); + + // Sort borrowed transactions by nonce to ensure proper ordering + borrowed_transactions.sort_by_key(|tx| tx.signed_transaction.nonce()); + + // Rebroadcast all transactions in parallel + let rebroadcast_futures: Vec<_> = borrowed_transactions + .iter() + .map(|borrowed| { + let tx_envelope = borrowed.signed_transaction.clone().into(); + let nonce = borrowed.signed_transaction.nonce(); + let transaction_id = borrowed.transaction_id.clone(); + + tracing::info!( + transaction_id = %transaction_id, + nonce = nonce, + "Recovering borrowed transaction" + ); + + async move { + let send_result = self.chain.provider().send_tx_envelope(tx_envelope).await; + (borrowed, send_result) + } + }) + .collect(); + + let rebroadcast_results = futures::future::join_all(rebroadcast_futures).await; + + // Convert results to SubmissionResult for batch processing + let submission_results: Vec = rebroadcast_results + .into_iter() + .map(|(borrowed, send_result)| { + SubmissionResult::from_send_result( + borrowed, + send_result, + SendContext::Rebroadcast, + &self.chain, + ) + }) + .collect(); + + // TODO: Implement post-processing analysis for balance threshold updates and nonce resets + // Currently we lose the granular error handling that was in the individual atomic operations. + // Consider: + // 1. Analyzing submission_results for specific error patterns + // 2. Calling update_balance_threshold if needed + // 3. Detecting nonce reset conditions + // 4. Or move this logic into the batch processor itself + + // Process all results in one batch operation + let report = self + .store + .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) + .await?; + + // TODO: Handle post-processing updates here if needed + // For now, we skip the individual error analysis that was done in the old atomic approach + + tracing::info!( + "Recovered {} transactions: {} submitted, {} recycled, {} failed", + report.total_processed, + report.moved_to_submitted, + report.moved_to_pending, + report.failed_transactions + ); + + Ok(report.total_processed as u32) + } + + // ========== HEALTH ACCESSOR ========== + + /// Get EOA health, initializing it if it doesn't exist + /// This method ensures the health data is always available for the worker + async fn get_eoa_health(&self) -> Result { + let store_health = self.store.get_eoa_health().await?; + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + match store_health { + Some(health) => Ok(health), + None => { + // Initialize with fresh data from chain + let balance = self + .chain + .provider() + .get_balance(self.eoa) + .await + .map_err(|e| { + let engine_error = e.to_engine_error(&self.chain); + EoaExecutorWorkerError::RpcError { + message: format!( + "Failed to get balance during initialization: {}", + engine_error + ), + inner_error: engine_error, + } + })?; + + let health = EoaHealth { + balance, + balance_threshold: U256::ZERO, + balance_fetched_at: now, + last_confirmation_at: now, + last_nonce_movement_at: now, + nonce_resets: Vec::new(), + }; + + // Save to store + self.store.update_health_data(&health).await?; + Ok(health) + } + } + } + + #[tracing::instrument(skip_all, fields(eoa = %self.eoa, chain_id = %self.chain.chain_id()))] + async fn update_balance_threshold(&self) -> Result<(), EoaExecutorWorkerError> { + let mut health = self.get_eoa_health().await?; + + tracing::info!("Updating balance threshold"); + let balance_threshold = self + .chain + .provider() + .get_balance(self.eoa) + .await + .map_err(|e| { + let engine_error = e.to_engine_error(&self.chain); + EoaExecutorWorkerError::RpcError { + message: format!("Failed to get balance: {}", engine_error), + inner_error: engine_error, + } + })?; + + health.balance_threshold = balance_threshold; + self.store.update_health_data(&health).await?; + Ok(()) + } + + async fn release_eoa_lock(self) { + self.store.release_eoa_lock().await; + } +} diff --git a/executors/src/eoa/worker/send.rs b/executors/src/eoa/worker/send.rs new file mode 100644 index 0000000..eb4c384 --- /dev/null +++ b/executors/src/eoa/worker/send.rs @@ -0,0 +1,402 @@ +use alloy::providers::Provider; +use engine_core::{chain::Chain, error::AlloyRpcErrorToEngineError}; + +use crate::eoa::{ + store::{BorrowedTransaction, PendingTransaction, SubmissionResult}, + worker::{ + EoaExecutorWorker, + error::{ + EoaExecutorWorkerError, SendContext, is_retryable_preparation_error, + should_update_balance_threshold, + }, + }, +}; + +const HEALTH_CHECK_INTERVAL: u64 = 300; // 5 minutes in seconds + +impl EoaExecutorWorker { + // ========== SEND FLOW ========== + #[tracing::instrument(skip_all)] + pub async fn send_flow(&self) -> Result { + // 1. Get EOA health (initializes if needed) and check if we should update balance + let mut health = self.get_eoa_health().await?; + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // Update balance if it's stale + // TODO: refactor this, very ugly + if health.balance <= health.balance_threshold { + if now - health.balance_fetched_at > HEALTH_CHECK_INTERVAL { + let balance = self + .chain + .provider() + .get_balance(self.eoa) + .await + .map_err(|e| { + let engine_error = e.to_engine_error(&self.chain); + EoaExecutorWorkerError::RpcError { + message: format!("Failed to get balance: {}", engine_error), + inner_error: engine_error, + } + })?; + + health.balance = balance; + health.balance_fetched_at = now; + self.store.update_health_data(&health).await?; + } + + if health.balance <= health.balance_threshold { + tracing::warn!( + "EOA has insufficient balance (<= {} wei), skipping send flow", + health.balance_threshold + ); + return Ok(0); + } + } + + let mut total_sent = 0; + + // 2. Process recycled nonces first + total_sent += self.process_recycled_nonces().await?; + + // 3. Only proceed to new nonces if we successfully used all recycled nonces + let remaining_recycled = self.store.peek_recycled_nonces().await?.len(); + if remaining_recycled == 0 { + let inflight_budget = self.store.get_inflight_budget(self.max_inflight).await?; + if inflight_budget > 0 { + total_sent += self.process_new_transactions(inflight_budget).await?; + } + } else { + tracing::warn!( + "Still have {} recycled nonces, not sending new transactions", + remaining_recycled + ); + } + + Ok(total_sent) + } + + async fn process_recycled_nonces(&self) -> Result { + let mut total_sent: usize = 0; + let mut is_pending_empty = false; + + // Loop to handle preparation failures and refill with new transactions + for _ in 0..10 { + let recycled_nonces = self.store.clean_and_get_recycled_nonces().await?; + + if recycled_nonces.is_empty() { + return Ok(total_sent as u32); + } + + // Get pending transactions to match with recycled nonces + let pending_txs = self + .store + .peek_pending_transactions(recycled_nonces.len() as u64) + .await?; + + // Pair recycled nonces with pending transactions + let mut build_tasks = Vec::new(); + + for (i, nonce) in recycled_nonces.iter().enumerate() { + if let Some(p_tx) = pending_txs.get(i) { + build_tasks + .push(self.build_and_sign_single_transaction_with_retries(p_tx, *nonce)); + } else { + // No more pending transactions for this recycled nonce + is_pending_empty = true; + break; + } + } + + if build_tasks.is_empty() { + break; + } + + // Build and sign all transactions in parallel + let prepared_results = futures::future::join_all(build_tasks).await; + let prepared_results_with_pending = pending_txs + .iter() + .zip(prepared_results.into_iter()) + .collect::>(); + + let cleaned_results = self + .clean_prepration_results(prepared_results_with_pending) + .await?; + + if cleaned_results.is_empty() { + // No successful preparations, try again with more pending transactions + continue; + } + + // Move prepared transactions to borrowed state with recycled nonces + let moved_count = self + .store + .atomic_move_pending_to_borrowed_with_recycled_nonces( + &cleaned_results + .iter() + .map(|borrowed_tx| borrowed_tx.data.clone()) + .collect::>(), + ) + .await?; + + tracing::debug!( + moved_count = moved_count, + total_prepared = cleaned_results.len(), + "Moved transactions to borrowed state using recycled nonces" + ); + + // Actually send the transactions to the blockchain + let send_tasks: Vec<_> = cleaned_results + .iter() + .map(|borrowed_tx| { + let signed_tx = borrowed_tx.signed_transaction.clone(); + async move { + self.chain + .provider() + .send_tx_envelope(signed_tx.into()) + .await + } + }) + .collect(); + + let send_results = futures::future::join_all(send_tasks).await; + + // Process send results and update states + let submission_results = send_results + .into_iter() + .zip(cleaned_results.into_iter()) + .map(|(send_result, borrowed_tx)| { + SubmissionResult::from_send_result( + &borrowed_tx, + send_result, + SendContext::InitialBroadcast, + &self.chain, + ) + }) + .collect(); + + // Use batch processing to handle all submission results + let processing_report = self + .store + .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) + .await?; + + tracing::debug!( + "Processed {} borrowed transactions: {} moved to submitted, {} moved to pending, {} failed", + processing_report.total_processed, + processing_report.moved_to_submitted, + processing_report.moved_to_pending, + processing_report.failed_transactions + ); + + total_sent += processing_report.moved_to_submitted; + } + + if is_pending_empty { + let recycled_nonces = self.store.clean_and_get_recycled_nonces().await?; + let mut build_tasks = Vec::new(); + + for nonce in recycled_nonces { + build_tasks.push(self.send_noop_transaction(nonce)); + } + + let send_results = futures::future::join_all(build_tasks).await; + + let successful_sends = send_results + .into_iter() + .filter_map(|result| result.ok()) + .collect::>(); + + self.store + .process_noop_transactions(&successful_sends) + .await?; + } + + Ok(total_sent as u32) + } + + async fn clean_prepration_results( + &self, + results: Vec<( + &PendingTransaction, + Result, + )>, + ) -> Result, EoaExecutorWorkerError> { + let mut cleaned_results = Vec::new(); + let mut balance_threshold_update_needed = false; + + for (pending, result) in results.into_iter() { + match result { + Ok(borrowed_data) => { + cleaned_results.push(borrowed_data); + } + Err(e) => { + // Track balance threshold issues + if let EoaExecutorWorkerError::TransactionSimulationFailed { + inner_error, .. + } = &e + { + if should_update_balance_threshold(inner_error) { + balance_threshold_update_needed = true; + } + } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { + if should_update_balance_threshold(inner_error) { + balance_threshold_update_needed = true; + } + } + + // For deterministic build failures, fail the transaction immediately + if !is_retryable_preparation_error(&e) { + self.store + .fail_pending_transaction(pending, e, self.webhook_queue.clone()) + .await?; + } + } + } + } + + if balance_threshold_update_needed { + if let Err(e) = self.update_balance_threshold().await { + tracing::error!("Failed to update balance threshold: {}", e); + } + } + + Ok(cleaned_results) + } + + /// Process new transactions with fixed iterations and simple sequential nonces + async fn process_new_transactions(&self, budget: u64) -> Result { + if budget == 0 { + return Ok(0); + } + + let mut total_sent: usize = 0; + let mut remaining_budget = budget; + + // Fixed number of iterations to avoid infinite loops + for iteration in 0..10 { + if remaining_budget == 0 { + break; + } + + // Get pending transactions + let pending_txs = self + .store + .peek_pending_transactions(remaining_budget) + .await?; + + if pending_txs.is_empty() { + break; + } + + let optimistic_nonce = self.store.get_optimistic_transaction_count().await?; + let batch_size = pending_txs.len().min(remaining_budget as usize); + + tracing::debug!( + iteration = iteration, + batch_size = batch_size, + starting_nonce = optimistic_nonce, + remaining_budget = remaining_budget, + "Processing new transaction batch" + ); + + // Build and sign all transactions in parallel with sequential nonces + let build_tasks: Vec<_> = pending_txs + .iter() + .take(batch_size) + .enumerate() + .map(|(i, tx)| { + let expected_nonce = optimistic_nonce + i as u64; + self.build_and_sign_single_transaction_with_retries(tx, expected_nonce) + }) + .collect(); + + let prepared_results = futures::future::join_all(build_tasks).await; + let prepared_results_with_pending = pending_txs + .iter() + .take(batch_size) + .zip(prepared_results.into_iter()) + .collect::>(); + + // Clean preparation results (handles failures and removes bad transactions) + let cleaned_results = self + .clean_prepration_results(prepared_results_with_pending) + .await?; + + if cleaned_results.is_empty() { + // No successful preparations, reduce budget and continue + continue; + } + + // Move prepared transactions to borrowed state with incremented nonces + let moved_count = self + .store + .atomic_move_pending_to_borrowed_with_incremented_nonces( + &cleaned_results + .iter() + .map(|borrowed_tx| borrowed_tx.data.clone()) + .collect::>(), + ) + .await?; + + tracing::debug!( + moved_count = moved_count, + total_prepared = cleaned_results.len(), + "Moved transactions to borrowed state using incremented nonces" + ); + + // Send the transactions to the blockchain + let send_tasks: Vec<_> = cleaned_results + .iter() + .map(|borrowed_tx| { + let signed_tx = borrowed_tx.signed_transaction.clone(); + async move { + self.chain + .provider() + .send_tx_envelope(signed_tx.into()) + .await + } + }) + .collect(); + + let send_results = futures::future::join_all(send_tasks).await; + + // Process send results and update states + let submission_results = send_results + .into_iter() + .zip(cleaned_results.into_iter()) + .map(|(send_result, borrowed_tx)| { + SubmissionResult::from_send_result( + &borrowed_tx, + send_result, + SendContext::InitialBroadcast, + &self.chain, + ) + }) + .collect(); + + // Use batch processing to handle all submission results + let processing_report = self + .store + .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) + .await?; + + tracing::debug!( + "Processed {} borrowed transactions: {} moved to submitted, {} moved to pending, {} failed", + processing_report.total_processed, + processing_report.moved_to_submitted, + processing_report.moved_to_pending, + processing_report.failed_transactions + ); + + total_sent += processing_report.moved_to_submitted; + remaining_budget = remaining_budget.saturating_sub(moved_count as u64); + + // If we didn't use all our budget in this iteration, we're likely done + if moved_count < batch_size { + break; + } + } + + Ok(total_sent as u32) + } +} diff --git a/executors/src/eoa/worker/transaction.rs b/executors/src/eoa/worker/transaction.rs new file mode 100644 index 0000000..12cd2e7 --- /dev/null +++ b/executors/src/eoa/worker/transaction.rs @@ -0,0 +1,427 @@ +use std::time::Duration; + +use alloy::{ + consensus::{ + SignableTransaction, Signed, TxEip4844Variant, TxEip4844WithSidecar, TypedTransaction, + }, + network::{TransactionBuilder, TransactionBuilder7702}, + primitives::{Bytes, U256}, + providers::Provider, + rpc::types::TransactionRequest as AlloyTransactionRequest, + signers::Signature, + transports::RpcError, +}; +use engine_core::{ + chain::Chain, + credentials::SigningCredential, + error::AlloyRpcErrorToEngineError, + signer::{AccountSigner, EoaSigningOptions}, + transaction::TransactionTypeData, +}; + +use crate::eoa::{ + EoaTransactionRequest, + store::{ + BorrowedTransaction, BorrowedTransactionData, PendingTransaction, SubmittedNoopTransaction, + TransactionData, + }, + worker::{ + EoaExecutorWorker, + error::{EoaExecutorWorkerError, is_retryable_preparation_error}, + }, +}; + +// Retry constants for preparation phase +const MAX_PREPARATION_RETRIES: u32 = 3; +const PREPARATION_RETRY_DELAY_MS: u64 = 100; + +impl EoaExecutorWorker { + pub async fn build_and_sign_single_transaction_with_retries( + &self, + pending_transaction: &PendingTransaction, + nonce: u64, + ) -> Result { + let mut last_error = None; + + // Internal retry loop for retryable errors + for attempt in 0..=MAX_PREPARATION_RETRIES { + if attempt > 0 { + // Simple exponential backoff + let delay = PREPARATION_RETRY_DELAY_MS * (2_u64.pow(attempt - 1)); + tokio::time::sleep(Duration::from_millis(delay)).await; + + tracing::debug!( + transaction_id = %pending_transaction.transaction_id, + attempt = attempt, + "Retrying transaction preparation" + ); + } + + match self + .build_and_sign_single_transaction(pending_transaction, nonce) + .await + { + Ok(result) => return Ok(result), + Err(error) => { + if is_retryable_preparation_error(&error) && attempt < MAX_PREPARATION_RETRIES { + tracing::warn!( + transaction_id = %pending_transaction.transaction_id, + attempt = attempt, + error = %error, + "Retryable error during transaction preparation, will retry" + ); + last_error = Some(error); + continue; + } else { + // Either deterministic error or exceeded max retries + return Err(error); + } + } + } + } + + // This should never be reached, but just in case + Err( + last_error.unwrap_or_else(|| EoaExecutorWorkerError::InternalError { + message: "Unexpected error in retry loop".to_string(), + }), + ) + } + + pub async fn build_and_sign_single_transaction( + &self, + pending_transaction: &PendingTransaction, + nonce: u64, + ) -> Result { + // Get transaction data + let tx_data = pending_transaction.user_request.clone(); + // Build and sign transaction + let signed_tx = self.build_and_sign_transaction(&tx_data, nonce).await?; + + Ok(BorrowedTransaction { + data: BorrowedTransactionData { + transaction_id: pending_transaction.transaction_id.clone(), + hash: signed_tx.hash().to_string(), + signed_transaction: signed_tx, + borrowed_at: chrono::Utc::now().timestamp_millis().max(0) as u64, + queued_at: pending_transaction.queued_at, + }, + user_request: pending_transaction.user_request.clone(), + }) + } + + pub async fn build_and_sign_noop_transaction( + &self, + nonce: u64, + ) -> Result, EoaExecutorWorkerError> { + // Create a minimal transaction to consume the recycled nonce + // Send 0 ETH to self with minimal gas + + // Build no-op transaction (send 0 to self) + let tx_request = AlloyTransactionRequest::default() + .with_from(self.eoa) + .with_to(self.eoa) // Send to self + .with_value(U256::ZERO) // Send 0 value + .with_input(Bytes::new()) // No data + .with_chain_id(self.chain.chain_id()) + .with_nonce(nonce) + .with_gas_limit(21000); // Minimal gas for basic transfer + + let tx_request = self.estimate_gas_fees(tx_request).await?; + let built_tx = tx_request.build_typed_tx().map_err(|e| { + EoaExecutorWorkerError::TransactionBuildFailed { + message: format!("Failed to build typed transaction for no-op: {e:?}"), + } + })?; + + let tx = self + .sign_transaction(built_tx, &self.noop_signing_credential) + .await?; + + Ok(tx) + } + + pub async fn send_noop_transaction( + &self, + nonce: u64, + ) -> Result { + let tx = self.build_and_sign_noop_transaction(nonce).await?; + + self.chain + .provider() + .send_tx_envelope(tx.into()) + .await + .map_err(|e| EoaExecutorWorkerError::TransactionSendError { + message: format!("Failed to send no-op transaction: {e:?}"), + inner_error: e.to_engine_error(&self.chain), + }) + .map(|pending| SubmittedNoopTransaction { + nonce, + hash: pending.tx_hash().to_string(), + }) + } + + async fn estimate_gas_fees( + &self, + tx: AlloyTransactionRequest, + ) -> Result { + // Check what fees are missing and need to be estimated + + // If we have gas_price set, we're doing legacy - don't estimate EIP-1559 + if tx.gas_price.is_some() { + return Ok(tx); + } + + // If we have both EIP-1559 fees set, don't estimate + if tx.max_fee_per_gas.is_some() && tx.max_priority_fee_per_gas.is_some() { + return Ok(tx); + } + + // Try EIP-1559 fees first, fall back to legacy if unsupported + match self.chain.provider().estimate_eip1559_fees().await { + Ok(eip1559_fees) => { + tracing::debug!( + "Using EIP-1559 fees: max_fee={}, max_priority_fee={}", + eip1559_fees.max_fee_per_gas, + eip1559_fees.max_priority_fee_per_gas + ); + + let mut result = tx; + // Only set fees that are missing + if result.max_fee_per_gas.is_none() { + result = result.with_max_fee_per_gas(eip1559_fees.max_fee_per_gas); + } + if result.max_priority_fee_per_gas.is_none() { + result = + result.with_max_priority_fee_per_gas(eip1559_fees.max_priority_fee_per_gas); + } + + Ok(result) + } + Err(eip1559_error) => { + // Check if this is an "unsupported feature" error + if let RpcError::UnsupportedFeature(_) = &eip1559_error { + tracing::debug!("EIP-1559 not supported, falling back to legacy gas price"); + + // Fall back to legacy gas price only if no gas price is set + if tx.authorization_list().is_none() { + match self.chain.provider().get_gas_price().await { + Ok(gas_price) => { + tracing::debug!("Using legacy gas price: {}", gas_price); + Ok(tx.with_gas_price(gas_price)) + } + Err(legacy_error) => Err(EoaExecutorWorkerError::RpcError { + message: format!( + "Failed to get legacy gas price: {}", + legacy_error + ), + inner_error: legacy_error.to_engine_error(&self.chain), + }), + } + } else { + Err(EoaExecutorWorkerError::TransactionBuildFailed { + message: "EIP7702 transactions not supported on chain".to_string(), + }) + } + } else { + // Other EIP-1559 error + Err(EoaExecutorWorkerError::RpcError { + message: format!("Failed to estimate EIP-1559 fees: {}", eip1559_error), + inner_error: eip1559_error.to_engine_error(&self.chain), + }) + } + } + } + } + + pub async fn build_typed_transaction( + &self, + request: &EoaTransactionRequest, + nonce: u64, + ) -> Result { + // Build transaction request from stored data + let mut tx_request = AlloyTransactionRequest::default() + .with_from(request.from) + .with_value(request.value) + .with_input(request.data.clone()) + .with_chain_id(request.chain_id) + .with_nonce(nonce); + + if let Some(to) = request.to { + tx_request = tx_request.with_to(to); + } + + if let Some(gas_limit) = request.gas_limit { + tx_request = tx_request.with_gas_limit(gas_limit); + } + + // Handle gas fees - either from user settings or estimation + tx_request = if let Some(type_data) = &request.transaction_type_data { + // User provided gas settings - respect them first + match type_data { + TransactionTypeData::Eip1559(data) => { + let mut req = tx_request; + if let Some(max_fee) = data.max_fee_per_gas { + req = req.with_max_fee_per_gas(max_fee); + } + if let Some(max_priority) = data.max_priority_fee_per_gas { + req = req.with_max_priority_fee_per_gas(max_priority); + } + + // if either not set, estimate the other one + if req.max_fee_per_gas.is_none() || req.max_priority_fee_per_gas.is_none() { + req = self.estimate_gas_fees(req).await?; + } + + req + } + TransactionTypeData::Legacy(data) => { + if let Some(gas_price) = data.gas_price { + tx_request.with_gas_price(gas_price) + } else { + // User didn't provide gas price, estimate it + self.estimate_gas_fees(tx_request).await? + } + } + TransactionTypeData::Eip7702(data) => { + let mut req = tx_request; + if let Some(authorization_list) = &data.authorization_list { + req = req.with_authorization_list(authorization_list.clone()); + } + if let Some(max_fee) = data.max_fee_per_gas { + req = req.with_max_fee_per_gas(max_fee); + } + if let Some(max_priority) = data.max_priority_fee_per_gas { + req = req.with_max_priority_fee_per_gas(max_priority); + } + + // if either not set, estimate the other one + if req.max_fee_per_gas.is_none() || req.max_priority_fee_per_gas.is_none() { + req = self.estimate_gas_fees(req).await?; + } + + req + } + } + } else { + // No user settings - estimate appropriate fees + self.estimate_gas_fees(tx_request).await? + }; + + // Estimate gas if needed + if tx_request.gas.is_none() { + match self.chain.provider().estimate_gas(tx_request.clone()).await { + Ok(gas_limit) => { + tx_request = tx_request.with_gas_limit(gas_limit * 110 / 100); // 10% buffer + } + Err(e) => { + // Check if this is a revert + if let RpcError::ErrorResp(error_payload) = &e { + if let Some(revert_data) = error_payload.as_revert_data() { + // This is a revert - the transaction is fundamentally broken + // This should fail the individual transaction, not the worker + return Err(EoaExecutorWorkerError::TransactionSimulationFailed { + message: format!( + "Transaction reverted during gas estimation: {} (revert: {})", + error_payload.message, + hex::encode(&revert_data) + ), + inner_error: e.to_engine_error(&self.chain), + }); + } + } + + // Not a revert - could be RPC issue, this should nack the worker + let engine_error = e.to_engine_error(&self.chain); + return Err(EoaExecutorWorkerError::RpcError { + message: format!("Gas estimation failed: {}", engine_error), + inner_error: engine_error, + }); + } + } + } + + // Build typed transaction + tx_request + .build_typed_tx() + .map_err(|e| EoaExecutorWorkerError::TransactionBuildFailed { + message: format!("Failed to build typed transaction: {:?}", e), + }) + } + + pub async fn sign_transaction( + &self, + typed_tx: TypedTransaction, + credential: &SigningCredential, + ) -> Result, EoaExecutorWorkerError> { + let signing_options = EoaSigningOptions { + from: self.eoa, + chain_id: Some(self.chain_id), + }; + + let signature = self + .signer + .sign_transaction(signing_options, &typed_tx, credential) + .await + .map_err(|engine_error| EoaExecutorWorkerError::SigningError { + message: format!("Failed to sign transaction: {}", engine_error), + inner_error: engine_error, + })?; + + let signature = signature.parse::().map_err(|e| { + EoaExecutorWorkerError::SignatureParsingFailed { + message: format!("Failed to parse signature: {}", e), + } + })?; + + Ok(typed_tx.into_signed(signature)) + } + + async fn build_and_sign_transaction( + &self, + request: &EoaTransactionRequest, + nonce: u64, + ) -> Result, EoaExecutorWorkerError> { + let typed_tx = self.build_typed_transaction(request, nonce).await?; + self.sign_transaction(typed_tx, &request.signing_credential) + .await + } + + pub fn apply_gas_bump_to_typed_transaction( + &self, + mut typed_tx: TypedTransaction, + bump_multiplier: u32, // e.g., 120 for 20% increase + ) -> TypedTransaction { + match &mut typed_tx { + TypedTransaction::Eip1559(tx) => { + tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; + tx.max_priority_fee_per_gas = + tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; + } + TypedTransaction::Legacy(tx) => { + tx.gas_price = tx.gas_price * bump_multiplier as u128 / 100; + } + TypedTransaction::Eip2930(tx) => { + tx.gas_price = tx.gas_price * bump_multiplier as u128 / 100; + } + TypedTransaction::Eip7702(tx) => { + tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; + tx.max_priority_fee_per_gas = + tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; + } + TypedTransaction::Eip4844(tx) => match tx { + TxEip4844Variant::TxEip4844(tx) => { + tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; + tx.max_priority_fee_per_gas = + tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; + } + TxEip4844Variant::TxEip4844WithSidecar(TxEip4844WithSidecar { tx, .. }) => { + tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; + tx.max_priority_fee_per_gas = + tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; + } + }, + } + typed_tx + } +} diff --git a/executors/src/external_bundler/confirm.rs b/executors/src/external_bundler/confirm.rs index 1f6e3b6..c1f69af 100644 --- a/executors/src/external_bundler/confirm.rs +++ b/executors/src/external_bundler/confirm.rs @@ -34,7 +34,7 @@ pub struct UserOpConfirmationJobData { pub user_op_hash: Bytes, pub nonce: U256, pub deployment_lock_acquired: bool, - pub webhook_options: Option>, + pub webhook_options: Vec, pub rpc_credentials: RpcCredentials, } @@ -338,7 +338,7 @@ where } impl HasWebhookOptions for UserOpConfirmationJobData { - fn webhook_options(&self) -> Option> { + fn webhook_options(&self) -> Vec { self.webhook_options.clone() } } diff --git a/executors/src/external_bundler/send.rs b/executors/src/external_bundler/send.rs index 0f7e4bc..bb533b6 100644 --- a/executors/src/external_bundler/send.rs +++ b/executors/src/external_bundler/send.rs @@ -49,7 +49,8 @@ pub struct ExternalBundlerSendJobData { pub execution_options: Erc4337ExecutionOptions, pub signing_credential: SigningCredential, - pub webhook_options: Option>, + #[serde(default)] + pub webhook_options: Vec, pub rpc_credentials: RpcCredentials, @@ -59,7 +60,7 @@ pub struct ExternalBundlerSendJobData { } impl HasWebhookOptions for ExternalBundlerSendJobData { - fn webhook_options(&self) -> Option> { + fn webhook_options(&self) -> Vec { self.webhook_options.clone() } } @@ -417,10 +418,12 @@ where // 7.1. Calculate custom call gas limit let custom_call_gas_limit = { - let gas_limits: Vec = job_data.transactions.iter() + let gas_limits: Vec = job_data + .transactions + .iter() .filter_map(|tx| tx.gas_limit) .collect(); - + if gas_limits.len() == job_data.transactions.len() { // All transactions have gas limits specified, sum them up let total_gas: u64 = gas_limits.iter().sum(); diff --git a/executors/src/webhook/envelope.rs b/executors/src/webhook/envelope.rs index 1887615..d351862 100644 --- a/executors/src/webhook/envelope.rs +++ b/executors/src/webhook/envelope.rs @@ -41,6 +41,35 @@ pub struct WebhookNotificationEnvelope { pub delivery_target_url: Option, } +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct BareWebhookNotificationEnvelope { + pub transaction_id: String, + pub event_type: StageEvent, + pub executor_name: String, + pub stage_name: String, + pub payload: T, +} + +impl BareWebhookNotificationEnvelope { + pub fn into_webhook_notification_envelope( + self, + timestamp: u64, + delivery_target_url: String, + ) -> WebhookNotificationEnvelope { + WebhookNotificationEnvelope { + notification_id: Uuid::new_v4().to_string(), + transaction_id: self.transaction_id, + timestamp, + executor_name: self.executor_name, + stage_name: self.stage_name, + event_type: self.event_type, + payload: self.payload, + delivery_target_url: Some(delivery_target_url), + } + } +} + // --- Serializable Hook Data Wrappers --- // These wrap the hook data to make them serializable (removing lifetimes) #[derive(Serialize, Deserialize, Debug, Clone)] @@ -75,7 +104,7 @@ pub trait ExecutorStage { // --- Webhook Options Trait --- pub trait HasWebhookOptions { - fn webhook_options(&self) -> Option>; + fn webhook_options(&self) -> Vec; } pub trait HasTransactionMetadata { @@ -102,10 +131,7 @@ pub trait WebhookCapable: DurableExecution + ExecutorStage { Self::JobData: HasWebhookOptions, Self::Output: Serialize + Clone, { - let webhook_options = match job.job.data.webhook_options() { - Some(w) => w, - None => return Ok(()), // No webhook configured, skip silently - }; + let webhook_options = job.job.data.webhook_options(); for w in webhook_options { let envelope = WebhookNotificationEnvelope { @@ -137,10 +163,7 @@ pub trait WebhookCapable: DurableExecution + ExecutorStage { Self::JobData: HasWebhookOptions, Self::ErrorData: Serialize + Clone, { - let webhook_options = match job.job.data.webhook_options() { - Some(w) => w, - None => return Ok(()), // No webhook configured, skip silently - }; + let webhook_options = job.job.data.webhook_options(); for w in webhook_options { let now: u64 = chrono::Utc::now().timestamp().try_into().unwrap(); let next_retry_at = nack_data.delay.map(|delay| now + delay.as_secs()); @@ -178,10 +201,7 @@ pub trait WebhookCapable: DurableExecution + ExecutorStage { Self::JobData: HasWebhookOptions, Self::ErrorData: Serialize + Clone, { - let webhook_options = match job.job.data.webhook_options() { - Some(w) => w, - None => return Ok(()), // No webhook configured, skip silently - }; + let webhook_options = job.job.data.webhook_options(); for w in webhook_options { let envelope = WebhookNotificationEnvelope { notification_id: Uuid::new_v4().to_string(), diff --git a/executors/src/webhook/mod.rs b/executors/src/webhook/mod.rs index 3408d26..271c4b6 100644 --- a/executors/src/webhook/mod.rs +++ b/executors/src/webhook/mod.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use engine_core::execution_options::WebhookOptions; use hex; use hmac::{Hmac, Mac}; use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; @@ -9,7 +10,9 @@ use serde::{Deserialize, Serialize}; use twmq::error::TwmqError; use twmq::hooks::TransactionContext; use twmq::job::{BorrowedJob, JobError, JobResult, RequeuePosition, ToJobResult}; -use twmq::{DurableExecution, FailHookData, NackHookData, SuccessHookData, UserCancellable}; +use twmq::{DurableExecution, FailHookData, NackHookData, Queue, SuccessHookData, UserCancellable}; + +use crate::webhook::envelope::{BareWebhookNotificationEnvelope, WebhookNotificationEnvelope}; pub mod envelope; @@ -113,7 +116,10 @@ impl DurableExecution for WebhookJobHandler { type JobData = WebhookJobPayload; #[tracing::instrument(skip_all, fields(queue = "webhook", job_id = job.job.id))] - async fn process(&self, job: &BorrowedJob) -> JobResult { + async fn process( + &self, + job: &BorrowedJob, + ) -> JobResult { let payload = &job.job.data; let mut request_headers = HeaderMap::new(); @@ -423,3 +429,77 @@ impl DurableExecution for WebhookJobHandler { ); } } + +pub fn queue_webhook_envelopes( + envelope: BareWebhookNotificationEnvelope, + webhook_options: Vec, + tx: &mut TransactionContext<'_>, + webhook_queue: Arc>, +) -> Result<(), TwmqError> { + let now = chrono::Utc::now().timestamp().max(0) as u64; + let serialised_webhook_envelopes = + webhook_options + .iter() + .map(|webhook_option| { + let webhook_notification_envelope = envelope + .clone() + .into_webhook_notification_envelope(now, webhook_option.url.clone()); + let serialised_envelope = serde_json::to_string(&webhook_notification_envelope)?; + Ok(( + serialised_envelope, + webhook_notification_envelope, + webhook_option.clone(), + )) + }) + .collect::, WebhookOptions)>, + serde_json::Error, + >>()?; + + let webhook_payloads = serialised_webhook_envelopes + .into_iter() + .map( + |(serialised_envelope, webhook_notification_envelope, webhook_option)| { + let payload = WebhookJobPayload { + url: webhook_option.url, + body: serialised_envelope, + headers: Some( + [ + ("Content-Type".to_string(), "application/json".to_string()), + ( + "User-Agent".to_string(), + format!("{}/{}", envelope.executor_name, envelope.stage_name), + ), + ] + .into_iter() + .collect(), + ), + hmac_secret: webhook_option.secret, // TODO: Add HMAC support if needed + http_method: Some("POST".to_string()), + }; + (payload, webhook_notification_envelope) + }, + ) + .collect::>(); + + for (payload, webhook_notification_envelope) in webhook_payloads { + let mut webhook_job = webhook_queue.clone().job(payload); + webhook_job.options.id = format!( + "{}_{}_webhook", + webhook_notification_envelope.transaction_id, + webhook_notification_envelope.notification_id + ); + + tx.queue_job(webhook_job)?; + tracing::info!( + transaction_id = %webhook_notification_envelope.transaction_id, + executor = %webhook_notification_envelope.executor_name, + stage = %webhook_notification_envelope.stage_name, + event = ?webhook_notification_envelope.event_type, + notification_id = %webhook_notification_envelope.notification_id, + "Queued webhook notification" + ); + } + + Ok(()) +} diff --git a/server/src/execution_router/mod.rs b/server/src/execution_router/mod.rs index 446ea07..8d35986 100644 --- a/server/src/execution_router/mod.rs +++ b/server/src/execution_router/mod.rs @@ -18,7 +18,9 @@ use engine_executors::{ confirm::Eip7702ConfirmationHandler, send::{Eip7702SendHandler, Eip7702SendJobData}, }, - eoa::{EoaExecutorStore, EoaExecutorWorker, EoaExecutorWorkerJobData, EoaTransactionRequest}, + eoa::{ + EoaExecutorJobHandler, EoaExecutorStore, EoaExecutorWorkerJobData, EoaTransactionRequest, + }, external_bundler::{ confirm::UserOpConfirmationHandler, send::{ExternalBundlerSendHandler, ExternalBundlerSendJobData}, @@ -26,7 +28,7 @@ use engine_executors::{ transaction_registry::TransactionRegistry, webhook::WebhookJobHandler, }; -use twmq::{Queue, error::TwmqError}; +use twmq::{Queue, error::TwmqError, redis::aio::ConnectionManager}; use vault_sdk::VaultClient; use vault_types::{ RegexRule, Rule, @@ -37,11 +39,12 @@ use vault_types::{ use crate::chains::ThirdwebChainService; pub struct ExecutionRouter { + pub redis: ConnectionManager, + pub namespace: Option, pub webhook_queue: Arc>, pub external_bundler_send_queue: Arc>>, pub userop_confirm_queue: Arc>>, - pub eoa_executor_queue: Arc>>, - pub eoa_executor_store: Arc, + pub eoa_executor_queue: Arc>>, pub eip7702_send_queue: Arc>>, pub eip7702_confirm_queue: Arc>>, pub transaction_registry: Arc, @@ -233,7 +236,7 @@ impl ExecutionRouter { self.execute_eip7702( &execution_request.execution_options.base, eip7702_execution_options, - &execution_request.webhook_options, + execution_request.webhook_options, &execution_request.params, rpc_credentials, signing_credential, @@ -262,7 +265,7 @@ impl ExecutionRouter { self.execute_eoa( &execution_request.execution_options.base, eoa_execution_options, - &execution_request.webhook_options, + execution_request.webhook_options, &execution_request.params, rpc_credentials, signing_credential, @@ -289,7 +292,7 @@ impl ExecutionRouter { &self, base_execution_options: &BaseExecutionOptions, erc4337_execution_options: &Erc4337ExecutionOptions, - webhook_options: &Option>, + webhook_options: &Vec, transactions: &[InnerTransaction], rpc_credentials: RpcCredentials, signing_credential: SigningCredential, @@ -338,7 +341,7 @@ impl ExecutionRouter { &self, base_execution_options: &BaseExecutionOptions, eip7702_execution_options: &Eip7702ExecutionOptions, - webhook_options: &Option>, + webhook_options: Vec, transactions: &[InnerTransaction], rpc_credentials: RpcCredentials, signing_credential: SigningCredential, @@ -349,7 +352,7 @@ impl ExecutionRouter { transactions: transactions.to_vec(), eoa_address: eip7702_execution_options.from, signing_credential, - webhook_options: webhook_options.clone(), + webhook_options, rpc_credentials, nonce: None, // Let the executor handle nonce generation }; @@ -383,7 +386,7 @@ impl ExecutionRouter { &self, base_execution_options: &BaseExecutionOptions, eoa_execution_options: &EoaExecutionOptions, - webhook_options: &Option>, + webhook_options: Vec, transactions: &[InnerTransaction], rpc_credentials: RpcCredentials, signing_credential: SigningCredential, @@ -403,14 +406,21 @@ impl ExecutionRouter { value: transaction.value, data: transaction.data.clone(), gas_limit: transaction.gas_limit, - webhook_options: webhook_options.clone(), - signing_credential, + webhook_options: webhook_options.to_vec(), + signing_credential: signing_credential.clone(), rpc_credentials, transaction_type_data: transaction.transaction_type_data.clone(), }; + let eoa_executor_store = EoaExecutorStore::new( + self.redis.clone(), + self.namespace.clone(), + eoa_execution_options.from, + base_execution_options.chain_id, + ); + // Add transaction to the store - self.eoa_executor_store + eoa_executor_store .add_transaction(eoa_transaction_request) .await .map_err(|e| TwmqError::Runtime { @@ -429,10 +439,7 @@ impl ExecutionRouter { let eoa_job_data = EoaExecutorWorkerJobData { eoa_address: eoa_execution_options.from, chain_id: base_execution_options.chain_id, - worker_id: format!( - "eoa_{}_{}", - eoa_execution_options.from, base_execution_options.chain_id - ), + noop_signing_credential: signing_credential, }; // Create idempotent job for this EOA:chain - only one will exist diff --git a/server/src/http/routes/admin/mod.rs b/server/src/http/routes/admin/mod.rs new file mode 100644 index 0000000..bb7845a --- /dev/null +++ b/server/src/http/routes/admin/mod.rs @@ -0,0 +1 @@ +pub mod queue; \ No newline at end of file diff --git a/server/src/http/routes/admin/queue.rs b/server/src/http/routes/admin/queue.rs new file mode 100644 index 0000000..0026dfc --- /dev/null +++ b/server/src/http/routes/admin/queue.rs @@ -0,0 +1,135 @@ +use axum::{ + debug_handler, + extract::{Path, State}, + http::StatusCode, + response::{IntoResponse, Json}, +}; +use serde::Serialize; +use utoipa::ToSchema; + +use crate::http::{error::ApiEngineError, server::EngineServerState, types::SuccessResponse}; + +// ===== TYPES ===== + +#[derive(Debug, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct EmptyIdempotencySetResponse { + pub queue_name: String, + pub message: String, +} + +// ===== ROUTE HANDLER ===== + +#[utoipa::path( + post, + operation_id = "emptyQueueIdempotencySet", + path = "/admin/queue/{queue_name}/empty-idempotency-set", + tag = "Admin", + responses( + (status = 200, description = "Successfully emptied idempotency set", body = SuccessResponse, content_type = "application/json"), + ), + params( + ("queue_name" = String, Path, description = "Queue name - one of: webhook, external_bundler_send, userop_confirm, eoa_executor, eip7702_send, eip7702_confirm"), + ) +)] +/// Empty Queue Idempotency Set +/// +/// Empty the idempotency set for a specific queue. This removes all job IDs from the deduplication set, +/// allowing duplicate jobs to be submitted again. +#[debug_handler] +pub async fn empty_queue_idempotency_set( + State(state): State, + Path(queue_name): Path, +) -> Result { + tracing::info!( + queue_name = %queue_name, + "Processing empty idempotency set request" + ); + + // Map queue name to the appropriate queue and empty its idempotency set + let result = match queue_name.as_str() { + "webhook" => state.queue_manager.webhook_queue.empty_dedupe_set().await, + "external_bundler_send" => { + state + .queue_manager + .external_bundler_send_queue + .empty_dedupe_set() + .await + } + "userop_confirm" => { + state + .queue_manager + .userop_confirm_queue + .empty_dedupe_set() + .await + } + "eoa_executor" => { + state + .queue_manager + .eoa_executor_queue + .empty_dedupe_set() + .await + } + "eip7702_send" => { + state + .queue_manager + .eip7702_send_queue + .empty_dedupe_set() + .await + } + "eip7702_confirm" => { + state + .queue_manager + .eip7702_confirm_queue + .empty_dedupe_set() + .await + } + _ => { + return Err(ApiEngineError( + engine_core::error::EngineError::ValidationError { + message: format!( + "Invalid queue name '{}'. Valid options are: webhook, external_bundler_send, userop_confirm, eoa_executor, eip7702_send, eip7702_confirm", + queue_name + ), + }, + )); + } + }; + + // Handle the result + match result { + Ok(()) => { + tracing::info!( + queue_name = %queue_name, + "Successfully emptied idempotency set" + ); + + Ok(( + StatusCode::OK, + Json(SuccessResponse::new(EmptyIdempotencySetResponse { + queue_name: queue_name.clone(), + message: format!( + "Successfully emptied idempotency set for queue '{}'", + queue_name + ), + })), + )) + } + Err(e) => { + tracing::error!( + queue_name = %queue_name, + error = %e, + "Failed to empty idempotency set" + ); + + Err(ApiEngineError( + engine_core::error::EngineError::InternalError { + message: format!( + "Failed to empty idempotency set for queue '{}': {}", + queue_name, e + ), + }, + )) + } + } +} diff --git a/server/src/http/routes/contract_write.rs b/server/src/http/routes/contract_write.rs index b0853d8..acd4c41 100644 --- a/server/src/http/routes/contract_write.rs +++ b/server/src/http/routes/contract_write.rs @@ -56,7 +56,8 @@ pub struct WriteContractRequest { /// or as separate transactions if atomic batching is not supported pub params: Vec, - pub webhook_options: Option>, + #[serde(default)] + pub webhook_options: Vec, } // ===== CONVENIENCE METHODS ===== diff --git a/server/src/http/routes/mod.rs b/server/src/http/routes/mod.rs index 6c674db..9f84ca8 100644 --- a/server/src/http/routes/mod.rs +++ b/server/src/http/routes/mod.rs @@ -1,3 +1,4 @@ +pub mod admin; pub mod contract_encode; pub mod contract_read; pub mod contract_write; diff --git a/server/src/http/routes/sign_message.rs b/server/src/http/routes/sign_message.rs index 0306341..6bd1c0a 100644 --- a/server/src/http/routes/sign_message.rs +++ b/server/src/http/routes/sign_message.rs @@ -120,7 +120,7 @@ async fn sign_single_message( eoa_options.clone(), &message_input.message, message_input.format, - signing_credential.clone(), + signing_credential, ) .await } diff --git a/server/src/http/routes/sign_typed_data.rs b/server/src/http/routes/sign_typed_data.rs index 427c3ba..a6f8a97 100644 --- a/server/src/http/routes/sign_typed_data.rs +++ b/server/src/http/routes/sign_typed_data.rs @@ -143,7 +143,7 @@ async fn sign_single_typed_data( // Direct EOA signing state .eoa_signer - .sign_typed_data(eoa_options.clone(), typed_data, signing_credential.clone()) + .sign_typed_data(eoa_options.clone(), typed_data, signing_credential) .await } SigningOptions::ERC4337(smart_account_options) => { diff --git a/server/src/http/server.rs b/server/src/http/server.rs index 44f09c7..18cd704 100644 --- a/server/src/http/server.rs +++ b/server/src/http/server.rs @@ -64,6 +64,9 @@ impl EngineServer { .routes(routes!( crate::http::routes::sign_typed_data::sign_typed_data )) + .routes(routes!( + crate::http::routes::admin::queue::empty_queue_idempotency_set + )) .layer(cors) .layer(TraceLayer::new_for_http()) .with_state(state); diff --git a/server/src/main.rs b/server/src/main.rs index 995317b..cd4dc4c 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -51,9 +51,10 @@ async fn main() -> anyhow::Result<()> { iaw_client: iaw_client.clone(), }); let eoa_signer = Arc::new(EoaSigner::new(vault_client.clone(), iaw_client)); + let redis_client = twmq::redis::Client::open(config.redis.url.as_str())?; let queue_manager = QueueManager::new( - &config.redis, + redis_client.clone(), &config.queue, chains.clone(), signer.clone(), @@ -74,11 +75,12 @@ async fn main() -> anyhow::Result<()> { .build()?; let execution_router = ExecutionRouter { + namespace: config.queue.execution_namespace.clone(), + redis: redis_client.get_connection_manager().await?, webhook_queue: queue_manager.webhook_queue.clone(), external_bundler_send_queue: queue_manager.external_bundler_send_queue.clone(), userop_confirm_queue: queue_manager.userop_confirm_queue.clone(), eoa_executor_queue: queue_manager.eoa_executor_queue.clone(), - eoa_executor_store: queue_manager.eoa_executor_store.clone(), eip7702_send_queue: queue_manager.eip7702_send_queue.clone(), eip7702_confirm_queue: queue_manager.eip7702_confirm_queue.clone(), transaction_registry: queue_manager.transaction_registry.clone(), diff --git a/server/src/queue/manager.rs b/server/src/queue/manager.rs index 04e216e..272ee9b 100644 --- a/server/src/queue/manager.rs +++ b/server/src/queue/manager.rs @@ -5,7 +5,7 @@ use alloy::transports::http::reqwest; use engine_core::error::EngineError; use engine_executors::{ eip7702_executor::{confirm::Eip7702ConfirmationHandler, send::Eip7702SendHandler}, - eoa::{EoaExecutorStore, EoaExecutorWorker}, + eoa::EoaExecutorJobHandler, external_bundler::{ confirm::UserOpConfirmationHandler, deployment::{RedisDeploymentCache, RedisDeploymentLock}, @@ -16,17 +16,13 @@ use engine_executors::{ }; use twmq::{Queue, queue::QueueOptions, shutdown::ShutdownHandle}; -use crate::{ - chains::ThirdwebChainService, - config::{QueueConfig, RedisConfig}, -}; +use crate::{chains::ThirdwebChainService, config::QueueConfig}; pub struct QueueManager { pub webhook_queue: Arc>, pub external_bundler_send_queue: Arc>>, pub userop_confirm_queue: Arc>>, - pub eoa_executor_queue: Arc>>, - pub eoa_executor_store: Arc, + pub eoa_executor_queue: Arc>>, pub eip7702_send_queue: Arc>>, pub eip7702_confirm_queue: Arc>>, pub transaction_registry: Arc, @@ -48,27 +44,18 @@ const EOA_EXECUTOR_QUEUE_NAME: &str = "eoa_executor"; impl QueueManager { pub async fn new( - redis_config: &RedisConfig, + redis_client: twmq::redis::Client, queue_config: &QueueConfig, chain_service: Arc, userop_signer: Arc, eoa_signer: Arc, ) -> Result { - // Create Redis clients - let redis_client = twmq::redis::Client::open(redis_config.url.as_str())?; - // Create transaction registry let transaction_registry = Arc::new(TransactionRegistry::new( redis_client.get_connection_manager().await?, queue_config.execution_namespace.clone(), )); - // Create EOA executor store - let eoa_executor_store = Arc::new(EoaExecutorStore::new( - redis_client.get_connection_manager().await?, - queue_config.execution_namespace.clone(), - )); - // Create deployment cache and lock let deployment_cache = RedisDeploymentCache::new(redis_client.clone()).await?; let deployment_lock = RedisDeploymentLock::new(redis_client.clone()).await?; @@ -219,10 +206,12 @@ impl QueueManager { .arc(); // Create EOA executor queue - let eoa_executor_handler = EoaExecutorWorker { + let eoa_executor_handler = EoaExecutorJobHandler { chain_service: chain_service.clone(), - store: eoa_executor_store.clone(), eoa_signer: eoa_signer.clone(), + webhook_queue: webhook_queue.clone(), + namespace: queue_config.execution_namespace.clone(), + redis: redis_client.get_connection_manager().await?, max_inflight: 100, max_recycled_nonces: 50, }; @@ -241,7 +230,6 @@ impl QueueManager { external_bundler_send_queue, userop_confirm_queue, eoa_executor_queue, - eoa_executor_store, eip7702_send_queue, eip7702_confirm_queue, transaction_registry, diff --git a/thirdweb-core/src/iaw/mod.rs b/thirdweb-core/src/iaw/mod.rs index 9b07542..84b3617 100644 --- a/thirdweb-core/src/iaw/mod.rs +++ b/thirdweb-core/src/iaw/mod.rs @@ -214,9 +214,9 @@ impl IAWClient { /// Sign a message with an EOA pub async fn sign_message( &self, - auth_token: AuthToken, - thirdweb_auth: ThirdwebAuth, - message: String, + auth_token: &AuthToken, + thirdweb_auth: &ThirdwebAuth, + message: &str, _from: Address, _chain_id: Option, format: Option, @@ -296,9 +296,9 @@ impl IAWClient { /// Sign a typed data structure with an EOA pub async fn sign_typed_data( &self, - auth_token: AuthToken, - thirdweb_auth: ThirdwebAuth, - typed_data: TypedData, + auth_token: &AuthToken, + thirdweb_auth: &ThirdwebAuth, + typed_data: &TypedData, _from: Address, ) -> Result { // Get ThirdwebAuth headers for billing/authentication @@ -365,9 +365,9 @@ impl IAWClient { /// Sign a transaction with an EOA pub async fn sign_transaction( &self, - auth_token: AuthToken, - thirdweb_auth: ThirdwebAuth, - transaction: EthereumTypedTransaction, + auth_token: &AuthToken, + thirdweb_auth: &ThirdwebAuth, + transaction: &EthereumTypedTransaction, ) -> Result { // Get ThirdwebAuth headers for billing/authentication let mut headers = thirdweb_auth.to_header_map()?; @@ -435,10 +435,10 @@ impl IAWClient { /// Sign an authorization with an EOA pub async fn sign_authorization( &self, - auth_token: AuthToken, - thirdweb_auth: ThirdwebAuth, + auth_token: &AuthToken, + thirdweb_auth: &ThirdwebAuth, _from: Address, - authorization: Authorization, + authorization: &Authorization, ) -> Result { // Get ThirdwebAuth headers for billing/authentication let mut headers = thirdweb_auth.to_header_map()?; diff --git a/twmq/src/lib.rs b/twmq/src/lib.rs index be6733c..96582bb 100644 --- a/twmq/src/lib.rs +++ b/twmq/src/lib.rs @@ -161,6 +161,15 @@ impl Queue { } } + /// Create a TransactionContext from an existing Redis pipeline + /// This allows queueing jobs atomically within an existing transaction + pub fn transaction_context_from_pipeline<'a>( + &self, + pipeline: &'a mut redis::Pipeline, + ) -> hooks::TransactionContext<'a> { + hooks::TransactionContext::new(pipeline, self.name.clone()) + } + // Get queue name pub fn name(&self) -> &str { &self.name @@ -476,6 +485,7 @@ impl Queue { // Normal polling tick _ = interval.tick() => { let queue_clone = outer_queue_clone.clone(); + let queue_name = queue_clone.name(); // Check available permits for batch size let available_permits = semaphore.available_permits(); @@ -495,7 +505,8 @@ impl Queue { let job_id = job.id().to_string(); let handler_clone = handler_clone.clone(); - tokio::spawn(async move { + tokio::spawn( + async move { // Process job - note we don't pass a context here let result = handler_clone.process(&job).await; @@ -510,7 +521,7 @@ impl Queue { // Release permit when done drop(permit); - }.instrument(tracing::info_span!("twmq_worker", job_id))); + }.instrument(tracing::info_span!("twmq_worker", job_id, queue_name))); } } Err(e) => {