diff --git a/src-ts/proposalPipeline.ts b/src-ts/proposalPipeline.ts index 25b5a5121..36a144cfc 100644 --- a/src-ts/proposalPipeline.ts +++ b/src-ts/proposalPipeline.ts @@ -13,6 +13,7 @@ import { BaseGovernorExecuteStage, L2TimelockExecutionSingleStage, SecurityCouncilManagerTimelockStage, + RedeemFailedError, } from "./proposalStage"; import { Signer, BigNumber } from "ethers"; import { Provider, TransactionReceipt } from "@ethersproject/abstract-provider"; @@ -129,7 +130,7 @@ export class StageTracker extends EventEmitter { this.stageFactory, nStage, this.pollingIntervalMs, - this.writeMode + this.writeMode, ); // propagate events to the listener of this tracker - add some info about the previous stage @@ -184,8 +185,18 @@ export class StageTracker extends EventEmitter { } catch (err) { if (err instanceof ProposalStageError) throw err; if (err instanceof UnreachableCaseError) throw err; + if (err instanceof RedeemFailedError) { + this.emit(TrackerEventName.TRACKER_ERRORED, { + status: currentStatus!, + stage: this.stage.name, + identifier: this.stage.identifier, + error: err, + }); + + } else { + consecutiveErrors++; + } - consecutiveErrors++; const error = err as Error; if (consecutiveErrors > 5) { // emit an error here @@ -203,7 +214,6 @@ export class StageTracker extends EventEmitter { error ); } - await wait(this.pollingIntervalMs); } } diff --git a/src-ts/proposalStage.ts b/src-ts/proposalStage.ts index c8059b385..d7fc27844 100644 --- a/src-ts/proposalStage.ts +++ b/src-ts/proposalStage.ts @@ -82,7 +82,7 @@ export interface ProposalStage { } /** - * Error with additional proposal information + * Fatal error with additional proposal information */ export class ProposalStageError extends Error { constructor(message: string, identifier: string, stageName: string, inner?: Error); @@ -105,6 +105,19 @@ export class UnreachableCaseError extends Error { } } +export class RedeemFailedError extends Error { + constructor( + public readonly retryableId: string, + public readonly since: number, + public readonly inner?: Error + ) { + super(`Retryable redeem for ${retryableId} failing since: ${new Date(since).toISOString()}`); + if (inner) { + this.stack += "\nCaused By: " + inner.stack; + } + } +} + /** * Taken from the IGovernorUpgradeable solidity */ @@ -869,7 +882,7 @@ export class SecurityCouncilManagerTimelockStage extends L2TimelockExecutionSing receipt: TransactionReceipt, arbOneSignerOrProvider: Provider | Signer ): Promise { - const hasManagerEvent = receipt.logs.find(log => log.address === this.managerAddress); + const hasManagerEvent = receipt.logs.find((log) => log.address === this.managerAddress); if (!hasManagerEvent) return []; @@ -878,24 +891,36 @@ export class SecurityCouncilManagerTimelockStage extends L2TimelockExecutionSing const upExecInterface = UpgradeExecutor__factory.createInterface(); const actionInterface = SecurityCouncilMemberSyncAction__factory.createInterface(); - const logs = receipt.logs.filter(log => log.topics[0] === timelockInterface.getEventTopic("CallScheduled")); + const logs = receipt.logs.filter( + (log) => log.topics[0] === timelockInterface.getEventTopic("CallScheduled") + ); if (logs.length === 0) return []; // we take the last log since it has the highest updateNonce const lastLog = logs[logs.length - 1]; - const callScheduledArgs = timelockInterface.parseLog(lastLog).args as CallScheduledEvent["args"]; - const parsedSendTxToL1 = arbSysInterface.decodeFunctionData("sendTxToL1", callScheduledArgs.data); + const callScheduledArgs = timelockInterface.parseLog(lastLog) + .args as CallScheduledEvent["args"]; + const parsedSendTxToL1 = arbSysInterface.decodeFunctionData( + "sendTxToL1", + callScheduledArgs.data + ); const parsedL1ScheduleBatch = L2TimelockExecutionStage.decodeScheduleBatch(parsedSendTxToL1[1]); - const parsedExecute = upExecInterface.decodeFunctionData("execute", parsedL1ScheduleBatch.callDatas[0]); - const parsedPerform = actionInterface.decodeFunctionData("perform", parsedExecute[1]) + const parsedExecute = upExecInterface.decodeFunctionData( + "execute", + parsedL1ScheduleBatch.callDatas[0] + ); + const parsedPerform = actionInterface.decodeFunctionData("perform", parsedExecute[1]); - const newMembers = parsedPerform[1] - const updateNonce = parsedPerform[2] + const newMembers = parsedPerform[1]; + const updateNonce = parsedPerform[2]; - const scheduleSalt = await SecurityCouncilManager__factory.connect(this.managerAddress, arbOneSignerOrProvider).generateSalt(newMembers, updateNonce) + const scheduleSalt = await SecurityCouncilManager__factory.connect( + this.managerAddress, + arbOneSignerOrProvider + ).generateSalt(newMembers, updateNonce); return [ new L2TimelockExecutionSingleStage( @@ -906,8 +931,8 @@ export class SecurityCouncilManagerTimelockStage extends L2TimelockExecutionSing scheduleSalt, lastLog.address, arbOneSignerOrProvider - ) - ] + ), + ]; } } @@ -1317,6 +1342,7 @@ export class L1TimelockExecutionBatchStage export class RetryableExecutionStage implements ProposalStage { public readonly identifier: string; public name: string = "RetryableExecutionStage"; + public failingSince: number = 0; constructor(public readonly l1ToL2Message: L1ToL2MessageReader | L1ToL2MessageWriter) { this.identifier = l1ToL2Message.retryableCreationId; @@ -1384,15 +1410,17 @@ export class RetryableExecutionStage implements ProposalStage { throw new Error("Message is not a writer"); } - while (true) { - try { - await (await this.l1ToL2Message.redeem()).wait(); - break; - } catch { - const id = this.l1ToL2Message.retryableCreationId.toLowerCase(); - console.error(`Failed to redeem retryable ${id}, retrying in 60s`); - await wait(60_000); + try { + await (await this.l1ToL2Message.redeem()).wait(); + } catch (err) { + if (this.failingSince === 0) { + this.failingSince = Date.now(); } + throw new RedeemFailedError( + this.l1ToL2Message.retryableCreationId.toLowerCase(), + this.failingSince, + err as Error + ); } }