Change redeem error handling to avoid infinite loop on successful redeem

src-ts/proposalPipeline.ts

@@ -13,6 +13,7 @@ import {
   BaseGovernorExecuteStage,
   L2TimelockExecutionSingleStage,
   SecurityCouncilManagerTimelockStage,
+  RedeemFailedError,
 } from "./proposalStage";
 import { Signer, BigNumber } from "ethers";
 import { Provider, TransactionReceipt } from "@ethersproject/abstract-provider";
@@ -129,7 +130,7 @@ export class StageTracker extends EventEmitter {
                 this.stageFactory,
                 nStage,
                 this.pollingIntervalMs,
-                this.writeMode
+                this.writeMode,
               );
 
               // propagate events to the listener of this tracker - add some info about the previous stage
@@ -184,8 +185,18 @@ export class StageTracker extends EventEmitter {
       } catch (err) {
         if (err instanceof ProposalStageError) throw err;
         if (err instanceof UnreachableCaseError) throw err;
+        if (err instanceof RedeemFailedError) {
+          this.emit(TrackerEventName.TRACKER_ERRORED, {
+            status: currentStatus!,
+            stage: this.stage.name,
+            identifier: this.stage.identifier,
+            error: err,
+          });
+
+        } else {
+          consecutiveErrors++;
+        }
 
-        consecutiveErrors++;
         const error = err as Error;
         if (consecutiveErrors > 5) {
           // emit an error here
@@ -203,7 +214,6 @@ export class StageTracker extends EventEmitter {
             error
           );
         }
-
         await wait(this.pollingIntervalMs);
       }
     }

src-ts/proposalStage.ts

@@ -82,7 +82,7 @@ export interface ProposalStage {
 }
 
 /**
- * Error with additional proposal information
+ * Fatal error with additional proposal information
  */
 export class ProposalStageError extends Error {
   constructor(message: string, identifier: string, stageName: string, inner?: Error);
@@ -105,6 +105,19 @@ export class UnreachableCaseError extends Error {
   }
 }
 
+export class RedeemFailedError extends Error {
+  constructor(
+    public readonly retryableId: string,
+    public readonly since: number,
+    public readonly inner?: Error
+  ) {
+    super(`Retryable redeem for ${retryableId} failing since: ${new Date(since).toISOString()}`);
+    if (inner) {
+      this.stack += "\nCaused By: " + inner.stack;
+    }
+  }
+}
+
 /**
  * Taken from the IGovernorUpgradeable solidity
  */
@@ -869,7 +882,7 @@ export class SecurityCouncilManagerTimelockStage extends L2TimelockExecutionSing
     receipt: TransactionReceipt,
     arbOneSignerOrProvider: Provider | Signer
   ): Promise<L2TimelockExecutionSingleStage[]> {
-    const hasManagerEvent = receipt.logs.find(log => log.address === this.managerAddress);
+    const hasManagerEvent = receipt.logs.find((log) => log.address === this.managerAddress);
 
     if (!hasManagerEvent) return [];
 
@@ -878,24 +891,36 @@ export class SecurityCouncilManagerTimelockStage extends L2TimelockExecutionSing
     const upExecInterface = UpgradeExecutor__factory.createInterface();
     const actionInterface = SecurityCouncilMemberSyncAction__factory.createInterface();
 
-    const logs = receipt.logs.filter(log => log.topics[0] === timelockInterface.getEventTopic("CallScheduled"));
+    const logs = receipt.logs.filter(
+      (log) => log.topics[0] === timelockInterface.getEventTopic("CallScheduled")
+    );
 
     if (logs.length === 0) return [];
 
     // we take the last log since it has the highest updateNonce
     const lastLog = logs[logs.length - 1];
 
-    const callScheduledArgs = timelockInterface.parseLog(lastLog).args as CallScheduledEvent["args"];
-    const parsedSendTxToL1 = arbSysInterface.decodeFunctionData("sendTxToL1", callScheduledArgs.data);
+    const callScheduledArgs = timelockInterface.parseLog(lastLog)
+      .args as CallScheduledEvent["args"];
+    const parsedSendTxToL1 = arbSysInterface.decodeFunctionData(
+      "sendTxToL1",
+      callScheduledArgs.data
+    );
     const parsedL1ScheduleBatch = L2TimelockExecutionStage.decodeScheduleBatch(parsedSendTxToL1[1]);
 
-    const parsedExecute = upExecInterface.decodeFunctionData("execute", parsedL1ScheduleBatch.callDatas[0]);
-    const parsedPerform = actionInterface.decodeFunctionData("perform", parsedExecute[1])
+    const parsedExecute = upExecInterface.decodeFunctionData(
+      "execute",
+      parsedL1ScheduleBatch.callDatas[0]
+    );
+    const parsedPerform = actionInterface.decodeFunctionData("perform", parsedExecute[1]);
 
-    const newMembers = parsedPerform[1]
-    const updateNonce = parsedPerform[2]
+    const newMembers = parsedPerform[1];
+    const updateNonce = parsedPerform[2];
 
-    const scheduleSalt = await SecurityCouncilManager__factory.connect(this.managerAddress, arbOneSignerOrProvider).generateSalt(newMembers, updateNonce)
+    const scheduleSalt = await SecurityCouncilManager__factory.connect(
+      this.managerAddress,
+      arbOneSignerOrProvider
+    ).generateSalt(newMembers, updateNonce);
 
     return [
       new L2TimelockExecutionSingleStage(
@@ -906,8 +931,8 @@ export class SecurityCouncilManagerTimelockStage extends L2TimelockExecutionSing
         scheduleSalt,
         lastLog.address,
         arbOneSignerOrProvider
-      )
-    ]
+      ),
+    ];
   }
 }
 
@@ -1317,6 +1342,7 @@ export class L1TimelockExecutionBatchStage
 export class RetryableExecutionStage implements ProposalStage {
   public readonly identifier: string;
   public name: string = "RetryableExecutionStage";
+  public failingSince: number = 0;
 
   constructor(public readonly l1ToL2Message: L1ToL2MessageReader | L1ToL2MessageWriter) {
     this.identifier = l1ToL2Message.retryableCreationId;
@@ -1384,15 +1410,17 @@ export class RetryableExecutionStage implements ProposalStage {
       throw new Error("Message is not a writer");
     }
 
-    while (true) {
-      try {
-        await (await this.l1ToL2Message.redeem()).wait();
-        break;
-      } catch {
-        const id = this.l1ToL2Message.retryableCreationId.toLowerCase();
-        console.error(`Failed to redeem retryable ${id}, retrying in 60s`);
-        await wait(60_000);
+    try {
+      await (await this.l1ToL2Message.redeem()).wait();
+    } catch (err) {
+      if (this.failingSince === 0) {
+        this.failingSince = Date.now();
       }
+      throw new RedeemFailedError(
+        this.l1ToL2Message.retryableCreationId.toLowerCase(),
+        this.failingSince,
+        err as Error
+      );
     }
   }