replace Pumba with docker-tc, rework old chaos tests

.github/workflows/devenv-ocr2-chaos.yml

@@ -0,0 +1,83 @@
+name: OCR2 Chaos Test
+
+on:
+  schedule:
+    - cron: "0 6 * * *" # Run daily at 6 AM
+  workflow_dispatch:
+
+defaults:
+  run:
+    working-directory: devenv
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  chaos:
+    permissions:
+      id-token: write
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu24.04-16cores-64GB # ghv-ignore!
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+
+      - name: Install Just
+        uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3
+        with:
+          just-version: "1.40.0"
+
+      - name: Configure AWS credentials using OIDC
+        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_IAM_ROLE_SDLC_ECR_READONLY_ARN }}
+          aws-region: us-west-2
+
+      - name: Authenticate to ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
+
+      - name: Set up Go
+        uses: actions/setup-go@v6 # v6
+        with:
+          cache: true
+          go-version-file: devenv/go.mod
+          cache-dependency-path: devenv/go.sum
+
+      - name: Download Go dependencies
+        run: |
+          go mod download
+
+      - name: Set environment variables
+        id: set-env
+        run: |
+          echo "CHAINLINK_IMAGE=${{ secrets.REGISTRY_SDLC }}/chainlink:nightly-$(date +%Y%m%d)-plugins" >> $GITHUB_ENV
+
+      - name: Run OCR2 environment
+        env:
+          FAKE_SERVER_IMAGE: ${{ secrets.FAKE_SERVER_IMAGE }}
+        run: |
+          cd cmd/cl && go install . && cd -
+          cl u env.toml,products/ocr2/basic.toml && cl obs up -f
+
+      - name: Run Chaos tests
+        id: chaos_test
+        working-directory: devenv/tests/ocr2
+        run: |
+          echo "Running tests for: $CHAINLINK_IMAGE, product: OCR2"
+          go test -v -timeout 4h -run TestOCR2Chaos
+
+      - name: Upload Logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: container-logs-smoke
+          path: devenv/tests/ocr2/logs
+          retention-days: 3

.github/workflows/devenv-ocr2.yml → .github/workflows/devenv-ocr2-soak.yml

@@ -72,7 +72,7 @@ jobs:
         working-directory: devenv/tests/ocr2
         run: |
           echo "Running tests for: $CHAINLINK_IMAGE, product: OCR2"
-          go test -v -timeout 4h -run TestOCR2Load/clean
+          go test -v -timeout 4h -run TestOCR2Soak/clean
 
       - name: Upload Logs
         if: always()

devenv/cmd/cl/completion.go

@@ -31,9 +31,9 @@ func getSubCommands(parent string) []prompt.Suggest {
 			{Text: "directrequest TestSmoke", Description: "Run Direct Request test"},
 			{Text: "flux TestSmoke", Description: "Run Flux test"},
 			{Text: "ocr2 TestSmoke/rounds", Description: "Run OCR2 smoke test"},
-			{Text: "ocr2 TestOCR2Load/clean", Description: "Run OCR2 soak test"},
-			{Text: "ocr2 TestOCR2Load/gas-spikes", Description: "Run OCR2 soak test + simulate gas spikes"},
-			{Text: "ocr2 TestOCR2Load/chaos", Description: "Run OCR2 soak test + introduce container kills and latency"},
+			{Text: "ocr2 TestOCR2Soak/clean", Description: "Run OCR2 soak test"},
+			{Text: "ocr2 TestOCR2Soak/gas-spikes", Description: "Run OCR2 soak test + simulate gas spikes"},
+			{Text: "ocr2 TestOCR2Chaos/rpc_latency", Description: "Run OCR2 chaos test + rpc latency"},
 		}
 	case "bs":
 		return []prompt.Suggest{

devenv/env.toml

@@ -40,4 +40,4 @@
   [[nodesets.node_specs]]
 
     [nodesets.node_specs.node]
-      image = "public.ecr.aws/chainlink/chainlink:2.30.0"
+      image = "public.ecr.aws/chainlink/chainlink:2.30.0"

devenv/go.mod

@@ -25,8 +25,8 @@ require (
 	github.com/smartcontractkit/chainlink-evm v0.3.3
 	github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20251211123524-f0c4fe7cfc0a
 	github.com/smartcontractkit/chainlink-protos/job-distributor v0.12.0
-	github.com/smartcontractkit/chainlink-testing-framework/framework v0.14.6
-	github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.10.1-0.20250711120409-5078050f9db4
+	github.com/smartcontractkit/chainlink-testing-framework/framework v0.14.9
+	github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.14.9
 	github.com/smartcontractkit/chainlink-testing-framework/seth v1.51.5
 	github.com/smartcontractkit/chainlink-testing-framework/wasp v1.51.2
 	github.com/smartcontractkit/libocr v0.0.0-20251212213002-0a5e2f907dda

devenv/go.sum

@@ -947,10 +947,10 @@ github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20251021173435-
 github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20251021173435-e86785845942/go.mod h1:2JTBNp3FlRdO/nHc4dsc9bfxxMClMO1Qt8sLJgtreBY=
 github.com/smartcontractkit/chainlink-protos/job-distributor v0.12.0 h1:/bhoALRzNXZkdzxBkNM505pMofNy0K0eW1nCzXw+AUI=
 github.com/smartcontractkit/chainlink-protos/job-distributor v0.12.0/go.mod h1:/dVVLXrsp+V0AbcYGJo3XMzKg3CkELsweA/TTopCsKE=
-github.com/smartcontractkit/chainlink-testing-framework/framework v0.14.6 h1:5hy0m8Vj5TzSU1lW2AZHbbpEQ71BXjLCseLMDnaK+Zs=
-github.com/smartcontractkit/chainlink-testing-framework/framework v0.14.6/go.mod h1:43xdIQuqw/gzfazsqJkBrGdF25TIJDiY/Ak/YrWFTmU=
-github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.10.1-0.20250711120409-5078050f9db4 h1:6iIj+U1SA19xftdEJwubATHBoGm4yc8q+MwWz6rlBDc=
-github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.10.1-0.20250711120409-5078050f9db4/go.mod h1:YEQbZRHFojvlQKeuckG/70t0WkAqOBmArSbkacgHSbc=
+github.com/smartcontractkit/chainlink-testing-framework/framework v0.14.9 h1:a/e5gwPjpzQwqtmMCop2vKXi55mYk+bQBP/5t3y60CI=
+github.com/smartcontractkit/chainlink-testing-framework/framework v0.14.9/go.mod h1:43xdIQuqw/gzfazsqJkBrGdF25TIJDiY/Ak/YrWFTmU=
+github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.14.9 h1:MVDx/Zl7qhikAx5vQgpTiyCkXw6sXgerUAU3WyJgWVY=
+github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.14.9/go.mod h1:1ZKcfw6mNKvM5GNy8AjeviL0tJVZoqhLZbmskcSG68k=
 github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 h1:VIxK8u0Jd0Q/VuhmsNm6Bls6Tb31H/sA3A/rbc5hnhg=
 github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0/go.mod h1:lyAu+oMXdNUzEDScj2DXB2IueY+SDXPPfyl/kb63tMM=
 github.com/smartcontractkit/chainlink-testing-framework/seth v1.51.5 h1:RwZXxdIAOyjp6cwc9Quxgr38k8r7ACz+Lxh9o/A6oH0=

devenv/tests/logpoller/logpoller_test.go

@@ -238,17 +238,8 @@ func XTestLogPollerHeavyLoad(t *testing.T) {
 // with approximate emission of 520-550 logs per second for ~110 seconds
 // 6 filters are registered
 
-/*
-	Chaos runs require Pumba, but that container still speaks Docker API 1.42. Newer Docker daemons reject it unless you pin a
-	minimum API version. If this test never pauses anything, add the entry below to your Docker Engine config (e.g. ~/.docker/daemon.json)
-	and restart Docker:
-
-	"min-api-version": "1.42"
-*/
-
 // Execute both on environment with finalityTagEnabled and with finalityDepth
 func TestLogPollerChaosChainlinkNodes(t *testing.T) {
-	t.Skip("We need to find replacement for Pumba, which doesn't work with Docker API > 1.42")
 	cfg := &Config{
 		General: General{
 			Generator:        "looped",
@@ -274,17 +265,8 @@ func TestLogPollerChaosChainlinkNodes(t *testing.T) {
 // with approximate emission of 520-550 logs per second for ~110 seconds
 // 6 filters are registered
 
-/*
-	Chaos runs require Pumba, but that container still speaks Docker API 1.42. Newer Docker daemons reject it unless you pin a
-	minimum API version. If this test never pauses anything, add the entry below to your Docker Engine config (e.g. ~/.docker/daemon.json)
-	and restart Docker:
-
-	"min-api-version": "1.42"
-*/
-
 // Execute both on environment with finalityTagEnabled and with finalityDepth
 func TestLogPollerChaosPostgres(t *testing.T) {
-	t.Skip("We need to find replacement for Pumba, which doesn't work with Docker API > 1.42")
 	cfg := &Config{
 		General: General{
 			Generator:        "looped",

devenv/tests/logpoller/test_helpers.go

@@ -10,7 +10,6 @@ import (
 	"math"
 	"math/big"
 	"math/rand"
-	"os/exec"
 	"sort"
 	"strings"
 	"sync"
@@ -33,6 +32,7 @@ import (
 	"github.com/smartcontractkit/chainlink-evm/pkg/logpoller"
 	cltypes "github.com/smartcontractkit/chainlink-evm/pkg/types"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/chaos"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/clnode"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/postgres"
 	nodeset "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set"
@@ -624,7 +624,7 @@ func missingLogs(
 
 // printMissingLogsInfo prints various useful information about the missing logs
 func printMissingLogsInfo(missingLogs map[string][]geth_types.Log, l zerolog.Logger, cfg *Config) {
-	var findHumanName = func(topic common.Hash) string {
+	findHumanName := func(topic common.Hash) string {
 		for _, event := range cfg.General.EventsToEmit {
 			if event.ID == topic {
 				return event.Name
@@ -763,7 +763,6 @@ func runWaspGenerator(t *testing.T, cfg *Config, logEmitters []contracts.LogEmit
 	}
 
 	_, err := p.Run(true)
-
 	if err != nil {
 		return 0, err
 	}
@@ -807,9 +806,9 @@ func runLoopedGenerator(cfg *Config, client *seth.Client, logEmitters []contract
 		return 0, err
 	}
 
-	var atomicCounter = atomic.Int32{}
+	atomicCounter := atomic.Int32{}
 
-	var emitAllEventsFn = func(resultCh chan emittedLogsData, errorCh chan error, _ int, task logEmissionTask) {
+	emitAllEventsFn := func(resultCh chan emittedLogsData, errorCh chan error, _ int, task logEmissionTask) {
 		current := atomicCounter.Add(1)
 
 		address := task.emitter.Address().String()
@@ -848,7 +847,6 @@ func runLoopedGenerator(cfg *Config, client *seth.Client, logEmitters []contract
 
 	executor := concurrency.NewConcurrentExecutor[emittedLogsData, emittedLogsData, logEmissionTask](l)
 	r, err := executor.Execute(len(client.Cfg.Network.PrivateKeys)-1, tasks, emitAllEventsFn)
-
 	if err != nil {
 		return 0, err
 	}
@@ -883,7 +881,7 @@ type PauseData struct {
 var ChaosPauses = []PauseData{}
 
 // chaosPauseSyncFn pauses ranom container of the provided type for a random amount of time between 5 and 20 seconds
-func chaosPauseSyncFn(ctx context.Context, l zerolog.Logger, client *seth.Client, nodes *nodeset.Input, targetComponent string) ChaosPauseData {
+func chaosPauseSyncFn(ctx context.Context, dtc *chaos.DockerChaos, l zerolog.Logger, client *seth.Client, nodes *nodeset.Input, targetComponent string) ChaosPauseData {
 	// var component ctf_test_env.EnvComponent
 	var containerName string
 
@@ -910,8 +908,13 @@ func chaosPauseSyncFn(ctx context.Context, l zerolog.Logger, client *seth.Client
 	l.Info().Str("Container", containerName).Int("Pause time", pauseTimeSec).Msg("Pausing component")
 	pauseTimeDur := time.Duration(pauseTimeSec) * time.Second
 
-	if err := pauseContainer(ctx, l, containerName, pauseTimeDur); err != nil {
-		return ChaosPauseData{Err: err}
+	err = dtc.Chaos(containerName, chaos.CmdPause, "")
+	if err != nil {
+		return ChaosPauseData{Err: fmt.Errorf("failed to pause docker container: %s, %w", containerName, err)}
+	}
+	time.Sleep(pauseTimeDur)
+	if err := dtc.RemoveAll(); err != nil {
+		return ChaosPauseData{Err: fmt.Errorf("failed to unpause docker container %s: %w", containerName, err)}
 	}
 	l.Info().Str("Container", containerName).Msg("Component unpaused")
 
@@ -930,39 +933,6 @@ func chaosPauseSyncFn(ctx context.Context, l zerolog.Logger, client *seth.Client
 	}}
 }
 
-func pauseContainer(ctx context.Context, l zerolog.Logger, containerName string, pauseTimeDur time.Duration) error {
-	command := fmt.Sprintf(`docker run -i --rm -v /var/run/docker.sock:/var/run/docker.sock --network %s gaiaadm/pumba --log-level=info pause --duration=%s %s`, framework.DefaultNetworkName, pauseTimeDur.String(), containerName)
-
-	fmt.Println("command: ", command)
-
-	c := strings.Split(command, " ")
-	l.Info().Interface("Command", c).Msg("Executing command")
-	cmd := exec.CommandContext(ctx, c[0], c[1:]...) // #nosec: G204
-	stderr, err := cmd.StderrPipe()
-	if err != nil {
-		return err
-	}
-	stdout, err := cmd.StdoutPipe()
-	if err != nil {
-		return err
-	}
-	if err := cmd.Start(); err != nil {
-		return err
-	}
-	outputFunction := func(m string) {
-		l.Debug().Str("Text", m).Msg("Std Pipe")
-	}
-	go readStdPipe(stderr, outputFunction)
-	go readStdPipe(stdout, outputFunction)
-
-	err = cmd.Wait()
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // readStdPipe continuously read a pipe from the command
 func readStdPipe(pipe io.ReadCloser, outputFunction func(string)) {
 	scanner := bufio.NewScanner(pipe)
@@ -987,6 +957,12 @@ func executeChaosExperiment(ctx context.Context, l zerolog.Logger, nodes *nodese
 		return
 	}
 
+	dtc, err := chaos.NewDockerChaos(ctx)
+	if err != nil {
+		errorCh <- fmt.Errorf("failed to created docker-tc container: %w", err)
+		return
+	}
+
 	chaosChan := make(chan ChaosPauseData, config.ChaosConfig.ExperimentCount)
 	wg := &sync.WaitGroup{}
 
@@ -1005,7 +981,7 @@ func executeChaosExperiment(ctx context.Context, l zerolog.Logger, nodes *nodese
 					current := i + 1
 					l.Info().Str("Current/Total", fmt.Sprintf("%d/%d", current, config.ChaosConfig.ExperimentCount)).Msg("Done with experiment")
 				}()
-				chaosChan <- chaosPauseSyncFn(ctx, l, sethClient, nodes, config.ChaosConfig.TargetComponent)
+				chaosChan <- chaosPauseSyncFn(ctx, dtc, l, sethClient, nodes, config.ChaosConfig.TargetComponent)
 				time.Sleep(10 * time.Second)
 			}()
 		}
@@ -1049,22 +1025,20 @@ const (
 	defaultAmountOfUpkeeps = 2
 )
 
-var (
-	DefaultOCRRegistryConfig = contracts.KeeperRegistrySettings{
-		PaymentPremiumPPB:    uint32(200000000),
-		FlatFeeMicroLINK:     uint32(0),
-		BlockCountPerTurn:    big.NewInt(10),
-		CheckGasLimit:        uint32(2500000),
-		StalenessSeconds:     big.NewInt(90000),
-		GasCeilingMultiplier: uint16(1),
-		MinUpkeepSpend:       big.NewInt(0),
-		MaxPerformGas:        uint32(5000000),
-		FallbackGasPrice:     big.NewInt(2e11),
-		FallbackLinkPrice:    big.NewInt(2e18),
-		MaxCheckDataSize:     uint32(5000),
-		MaxPerformDataSize:   uint32(5000),
-	}
-)
+var DefaultOCRRegistryConfig = contracts.KeeperRegistrySettings{
+	PaymentPremiumPPB:    uint32(200000000),
+	FlatFeeMicroLINK:     uint32(0),
+	BlockCountPerTurn:    big.NewInt(10),
+	CheckGasLimit:        uint32(2500000),
+	StalenessSeconds:     big.NewInt(90000),
+	GasCeilingMultiplier: uint16(1),
+	MinUpkeepSpend:       big.NewInt(0),
+	MaxPerformGas:        uint32(5000000),
+	FallbackGasPrice:     big.NewInt(2e11),
+	FallbackLinkPrice:    big.NewInt(2e18),
+	MaxCheckDataSize:     uint32(5000),
+	MaxPerformDataSize:   uint32(5000),
+}
 
 // uploadLogEmitterContracts uploads the configured number of log emitter contracts
 func uploadLogEmitterContracts(l zerolog.Logger, t *testing.T, client *seth.Client, config *Config) []contracts.LogEmitter {