From 70ac77627f94eaa20f00c6275cf1b1cf4d1ac485 Mon Sep 17 00:00:00 2001 From: Chris Buckland Date: Fri, 1 Nov 2024 14:17:33 +0000 Subject: [PATCH 1/2] Changed error handling for redeems --- src-ts/proposalPipeline.ts | 16 +++++++++++++--- src-ts/proposalStage.ts | 30 +++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src-ts/proposalPipeline.ts b/src-ts/proposalPipeline.ts index 25b5a5121..36a144cfc 100644 --- a/src-ts/proposalPipeline.ts +++ b/src-ts/proposalPipeline.ts @@ -13,6 +13,7 @@ import { BaseGovernorExecuteStage, L2TimelockExecutionSingleStage, SecurityCouncilManagerTimelockStage, + RedeemFailedError, } from "./proposalStage"; import { Signer, BigNumber } from "ethers"; import { Provider, TransactionReceipt } from "@ethersproject/abstract-provider"; @@ -129,7 +130,7 @@ export class StageTracker extends EventEmitter { this.stageFactory, nStage, this.pollingIntervalMs, - this.writeMode + this.writeMode, ); // propagate events to the listener of this tracker - add some info about the previous stage @@ -184,8 +185,18 @@ export class StageTracker extends EventEmitter { } catch (err) { if (err instanceof ProposalStageError) throw err; if (err instanceof UnreachableCaseError) throw err; + if (err instanceof RedeemFailedError) { + this.emit(TrackerEventName.TRACKER_ERRORED, { + status: currentStatus!, + stage: this.stage.name, + identifier: this.stage.identifier, + error: err, + }); + + } else { + consecutiveErrors++; + } - consecutiveErrors++; const error = err as Error; if (consecutiveErrors > 5) { // emit an error here @@ -203,7 +214,6 @@ export class StageTracker extends EventEmitter { error ); } - await wait(this.pollingIntervalMs); } } diff --git a/src-ts/proposalStage.ts b/src-ts/proposalStage.ts index c8059b385..e6f378671 100644 --- a/src-ts/proposalStage.ts +++ b/src-ts/proposalStage.ts @@ -82,7 +82,7 @@ export interface ProposalStage { } /** - * Error with additional proposal information + * Fatal error with additional proposal information */ export class ProposalStageError extends Error { constructor(message: string, identifier: string, stageName: string, inner?: Error); @@ -105,6 +105,19 @@ export class UnreachableCaseError extends Error { } } +export class RedeemFailedError extends Error { + constructor( + public readonly retryableId: string, + public readonly since: number, + public readonly inner?: Error + ) { + super(`Retryable redeem for ${retryableId} failing since: ${new Date(since).toISOString()}`) + if (inner) { + this.stack += "\nCaused By: " + inner.stack; + } + } +} + /** * Taken from the IGovernorUpgradeable solidity */ @@ -1317,6 +1330,7 @@ export class L1TimelockExecutionBatchStage export class RetryableExecutionStage implements ProposalStage { public readonly identifier: string; public name: string = "RetryableExecutionStage"; + public failingSince: number = 0; constructor(public readonly l1ToL2Message: L1ToL2MessageReader | L1ToL2MessageWriter) { this.identifier = l1ToL2Message.retryableCreationId; @@ -1384,15 +1398,13 @@ export class RetryableExecutionStage implements ProposalStage { throw new Error("Message is not a writer"); } - while (true) { - try { - await (await this.l1ToL2Message.redeem()).wait(); - break; - } catch { - const id = this.l1ToL2Message.retryableCreationId.toLowerCase(); - console.error(`Failed to redeem retryable ${id}, retrying in 60s`); - await wait(60_000); + try { + await (await this.l1ToL2Message.redeem()).wait(); + } catch(err) { + if(this.failingSince === 0) { + this.failingSince = Date.now(); } + throw new RedeemFailedError(this.l1ToL2Message.retryableCreationId.toLowerCase(), this.failingSince, err as Error) } } From e2d1136ad1b08aea443287b8ed26a62f710f7877 Mon Sep 17 00:00:00 2001 From: Chris Buckland Date: Fri, 1 Nov 2024 14:18:09 +0000 Subject: [PATCH 2/2] Formatting --- src-ts/proposalStage.ts | 48 +++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/src-ts/proposalStage.ts b/src-ts/proposalStage.ts index e6f378671..d7fc27844 100644 --- a/src-ts/proposalStage.ts +++ b/src-ts/proposalStage.ts @@ -107,11 +107,11 @@ export class UnreachableCaseError extends Error { export class RedeemFailedError extends Error { constructor( - public readonly retryableId: string, + public readonly retryableId: string, public readonly since: number, public readonly inner?: Error ) { - super(`Retryable redeem for ${retryableId} failing since: ${new Date(since).toISOString()}`) + super(`Retryable redeem for ${retryableId} failing since: ${new Date(since).toISOString()}`); if (inner) { this.stack += "\nCaused By: " + inner.stack; } @@ -882,7 +882,7 @@ export class SecurityCouncilManagerTimelockStage extends L2TimelockExecutionSing receipt: TransactionReceipt, arbOneSignerOrProvider: Provider | Signer ): Promise { - const hasManagerEvent = receipt.logs.find(log => log.address === this.managerAddress); + const hasManagerEvent = receipt.logs.find((log) => log.address === this.managerAddress); if (!hasManagerEvent) return []; @@ -891,24 +891,36 @@ export class SecurityCouncilManagerTimelockStage extends L2TimelockExecutionSing const upExecInterface = UpgradeExecutor__factory.createInterface(); const actionInterface = SecurityCouncilMemberSyncAction__factory.createInterface(); - const logs = receipt.logs.filter(log => log.topics[0] === timelockInterface.getEventTopic("CallScheduled")); + const logs = receipt.logs.filter( + (log) => log.topics[0] === timelockInterface.getEventTopic("CallScheduled") + ); if (logs.length === 0) return []; // we take the last log since it has the highest updateNonce const lastLog = logs[logs.length - 1]; - const callScheduledArgs = timelockInterface.parseLog(lastLog).args as CallScheduledEvent["args"]; - const parsedSendTxToL1 = arbSysInterface.decodeFunctionData("sendTxToL1", callScheduledArgs.data); + const callScheduledArgs = timelockInterface.parseLog(lastLog) + .args as CallScheduledEvent["args"]; + const parsedSendTxToL1 = arbSysInterface.decodeFunctionData( + "sendTxToL1", + callScheduledArgs.data + ); const parsedL1ScheduleBatch = L2TimelockExecutionStage.decodeScheduleBatch(parsedSendTxToL1[1]); - const parsedExecute = upExecInterface.decodeFunctionData("execute", parsedL1ScheduleBatch.callDatas[0]); - const parsedPerform = actionInterface.decodeFunctionData("perform", parsedExecute[1]) + const parsedExecute = upExecInterface.decodeFunctionData( + "execute", + parsedL1ScheduleBatch.callDatas[0] + ); + const parsedPerform = actionInterface.decodeFunctionData("perform", parsedExecute[1]); - const newMembers = parsedPerform[1] - const updateNonce = parsedPerform[2] + const newMembers = parsedPerform[1]; + const updateNonce = parsedPerform[2]; - const scheduleSalt = await SecurityCouncilManager__factory.connect(this.managerAddress, arbOneSignerOrProvider).generateSalt(newMembers, updateNonce) + const scheduleSalt = await SecurityCouncilManager__factory.connect( + this.managerAddress, + arbOneSignerOrProvider + ).generateSalt(newMembers, updateNonce); return [ new L2TimelockExecutionSingleStage( @@ -919,8 +931,8 @@ export class SecurityCouncilManagerTimelockStage extends L2TimelockExecutionSing scheduleSalt, lastLog.address, arbOneSignerOrProvider - ) - ] + ), + ]; } } @@ -1400,11 +1412,15 @@ export class RetryableExecutionStage implements ProposalStage { try { await (await this.l1ToL2Message.redeem()).wait(); - } catch(err) { - if(this.failingSince === 0) { + } catch (err) { + if (this.failingSince === 0) { this.failingSince = Date.now(); } - throw new RedeemFailedError(this.l1ToL2Message.retryableCreationId.toLowerCase(), this.failingSince, err as Error) + throw new RedeemFailedError( + this.l1ToL2Message.retryableCreationId.toLowerCase(), + this.failingSince, + err as Error + ); } }