Skip to content

Commit 469a09b

Browse files
clampleclample
authored
feat: add support for test retries [sc-20570] (#952)
1 parent 89dce39 commit 469a09b

File tree

24 files changed

+355
-123
lines changed

24 files changed

+355
-123
lines changed

‎packages/cli/e2e/__tests__/deploy.spec.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,11 @@ describe('deploy', () => {
125125
apiKey: config.get('apiKey'),
126126
accountId: config.get('accountId'),
127127
directory: path.join(__dirname, 'fixtures', 'deploy-project'),
128-
env: { PROJECT_LOGICAL_ID: projectLogicalId, PRIVATE_LOCATION_SLUG_NAME: privateLocationSlugname },
128+
env: {
129+
PROJECT_LOGICAL_ID: projectLogicalId,
130+
PRIVATE_LOCATION_SLUG_NAME: privateLocationSlugname,
131+
CHECKLY_CLI_VERSION: undefined,
132+
},
129133
})
130134
expect(stderr).toBe('')
131135
// expect the version to be overriden with latest from NPM

‎packages/cli/e2e/__tests__/fixtures/retry-project/checkly.config.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import { defineConfig } from 'checkly'
2+
3+
const config = defineConfig({
4+
projectName: 'Test Project',
5+
logicalId: 'test-project',
6+
repoUrl: 'https://github.com/checkly/checkly-cli',
7+
checks: {
8+
locations: ['us-east-1', 'eu-west-1'],
9+
tags: ['mac'],
10+
runtimeId: '2023.09',
11+
checkMatch: '**/*.check.ts',
12+
browserChecks: {
13+
// using .test.ts suffix (no .spec.ts) to avoid '@playwright/test not found error' when Jest transpile the spec.ts
14+
testMatch: '**/__checks__/*.test.ts',
15+
},
16+
},
17+
cli: {
18+
runLocation: 'us-east-1',
19+
},
20+
})
21+
22+
export default config

‎packages/cli/e2e/__tests__/fixtures/retry-project/src/__checks__/group.check.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/* eslint-disable no-new */
2+
import { CheckGroup, BrowserCheck } from 'checkly/constructs'
3+
4+
const group = new CheckGroup('check-group-1', {
5+
name: 'Group',
6+
activated: true,
7+
muted: false,
8+
locations: ['us-east-1', 'eu-west-1'],
9+
tags: ['mac', 'group'],
10+
environmentVariables: [],
11+
apiCheckDefaults: {},
12+
alertChannels: [],
13+
browserChecks: {
14+
// using .test.ts suffix (no .spec.ts) to avoid '@playwright/test not found error' when Jest transpile the spec.ts
15+
testMatch: '**/*.test.ts',
16+
},
17+
})
18+
19+
new BrowserCheck('group-browser-check-1', {
20+
name: 'Check with group',
21+
activated: false,
22+
groupId: group.ref(),
23+
code: {
24+
content: 'throw new Error("Failing Check Result")',
25+
},
26+
})

‎packages/cli/e2e/__tests__/test.spec.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,4 +191,17 @@ describe('test', () => {
191191
fs.rmSync(snapshotDir, { recursive: true })
192192
}
193193
})
194+
195+
it('Should execute retries', async () => {
196+
const result = await runChecklyCli({
197+
args: ['test', '--retries=3'],
198+
apiKey: config.get('apiKey'),
199+
accountId: config.get('accountId'),
200+
directory: path.join(__dirname, 'fixtures', 'retry-project'),
201+
timeout: 120000, // 2 minutes
202+
})
203+
// The failing check result will have "Failing Check Result" in the output.
204+
// We expect the check to be run 4 times.
205+
expect(result.stdout.match(/Failing Check Result/g)).toHaveLength(4)
206+
})
194207
})

‎packages/cli/e2e/run-checkly.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,9 @@ export function runChecklyCli (options: {
3333
CHECKLY_API_KEY: apiKey,
3434
CHECKLY_ACCOUNT_ID: accountId,
3535
CHECKLY_ENV: process.env.CHECKLY_ENV,
36-
CHECKLY_CLI_VERSION: cliVersion,
36+
// We need the CLI to report 4.8.0 or greater in order for the backend to use the new MQTT topic format.
37+
// Once 4.8.0 has been released, we can remove the 4.8.0 fallback here.
38+
CHECKLY_CLI_VERSION: cliVersion ?? '4.8.0',
3739
CHECKLY_E2E_PROMPTS_INJECTIONS: promptsInjection?.length ? JSON.stringify(promptsInjection) : undefined,
3840
...env,
3941
},

‎packages/cli/src/commands/test.ts‎

Lines changed: 43 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,15 @@ import {
88
Events,
99
RunLocation,
1010
PrivateRunLocation,
11-
CheckRunId,
11+
SequenceId,
1212
DEFAULT_CHECK_RUN_TIMEOUT_SECONDS,
1313
} from '../services/abstract-check-runner'
1414
import TestRunner from '../services/test-runner'
1515
import { loadChecklyConfig } from '../services/checkly-config-loader'
1616
import { filterByFileNamePattern, filterByCheckNamePattern, filterByTags } from '../services/test-filters'
1717
import type { Runtime } from '../rest/runtimes'
1818
import { AuthCommand } from './authCommand'
19-
import { BrowserCheck, Check, HeartbeatCheck, MultiStepCheck, Project, Session } from '../constructs'
19+
import { BrowserCheck, Check, HeartbeatCheck, MultiStepCheck, Project, RetryStrategyBuilder, Session } from '../constructs'
2020
import type { Region } from '..'
2121
import { splitConfigFilePath, getGitInformation, getCiInformation, getEnvs } from '../services/util'
2222
import { createReporters, ReporterType } from '../reporters/reporter'
@@ -26,6 +26,7 @@ import { printLn, formatCheckTitle, CheckStatus } from '../reporters/util'
2626
import { uploadSnapshots } from '../services/snapshot-service'
2727

2828
const DEFAULT_REGION = 'eu-central-1'
29+
const MAX_RETRIES = 3
2930

3031
export default class Test extends AuthCommand {
3132
static coreCommand = true
@@ -100,6 +101,9 @@ export default class Test extends AuthCommand {
100101
description: 'Update any snapshots using the actual result of this test run.',
101102
default: false,
102103
}),
104+
retries: Flags.integer({
105+
description: `[default: 0, max: ${MAX_RETRIES}] How many times to retry a failing test run.`,
106+
}),
103107
}
104108

105109
static args = {
@@ -132,6 +136,7 @@ export default class Test extends AuthCommand {
132136
record: shouldRecord,
133137
'test-session-name': testSessionName,
134138
'update-snapshots': updateSnapshots,
139+
retries,
135140
} = flags
136141
const filePatterns = argv as string[]
137142

@@ -228,6 +233,7 @@ export default class Test extends AuthCommand {
228233
const reporters = createReporters(reporterTypes, location, verbose)
229234
const repoInfo = getGitInformation(project.repoUrl)
230235
const ciInfo = getCiInformation()
236+
const testRetryStrategy = this.prepareTestRetryStrategy(retries, checklyConfig?.cli?.retries)
231237

232238
const runner = new TestRunner(
233239
config.getAccountId(),
@@ -241,34 +247,45 @@ export default class Test extends AuthCommand {
241247
ciInfo.environment,
242248
updateSnapshots,
243249
configDirectory,
250+
testRetryStrategy,
244251
)
245252

246253
runner.on(Events.RUN_STARTED,
247-
(checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId: string) =>
254+
(checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId: string) =>
248255
reporters.forEach(r => r.onBegin(checks, testSessionId)),
249256
)
250257

251-
runner.on(Events.CHECK_INPROGRESS, (check: any, checkRunId: CheckRunId) => {
252-
reporters.forEach(r => r.onCheckInProgress(check, checkRunId))
258+
runner.on(Events.CHECK_INPROGRESS, (check: any, sequenceId: SequenceId) => {
259+
reporters.forEach(r => r.onCheckInProgress(check, sequenceId))
253260
})
254261

255262
runner.on(Events.MAX_SCHEDULING_DELAY_EXCEEDED, () => {
256263
reporters.forEach(r => r.onSchedulingDelayExceeded())
257264
})
258265

259-
runner.on(Events.CHECK_SUCCESSFUL, (checkRunId, check, result, links?: TestResultsShortLinks) => {
260-
if (result.hasFailures) {
261-
process.exitCode = 1
262-
}
263-
264-
reporters.forEach(r => r.onCheckEnd(checkRunId, {
266+
runner.on(Events.CHECK_ATTEMPT_RESULT, (sequenceId: SequenceId, check, result, links?: TestResultsShortLinks) => {
267+
reporters.forEach(r => r.onCheckAttemptResult(sequenceId, {
265268
logicalId: check.logicalId,
266269
sourceFile: check.getSourceFile(),
267270
...result,
268271
}, links))
269272
})
270-
runner.on(Events.CHECK_FAILED, (checkRunId, check, message: string) => {
271-
reporters.forEach(r => r.onCheckEnd(checkRunId, {
273+
274+
runner.on(Events.CHECK_SUCCESSFUL,
275+
(sequenceId: SequenceId, check, result, testResultId, links?: TestResultsShortLinks) => {
276+
if (result.hasFailures) {
277+
process.exitCode = 1
278+
}
279+
280+
reporters.forEach(r => r.onCheckEnd(sequenceId, {
281+
logicalId: check.logicalId,
282+
sourceFile: check.getSourceFile(),
283+
...result,
284+
}, testResultId, links))
285+
})
286+
287+
runner.on(Events.CHECK_FAILED, (sequenceId: SequenceId, check, message: string) => {
288+
reporters.forEach(r => r.onCheckEnd(sequenceId, {
272289
...check,
273290
logicalId: check.logicalId,
274291
sourceFile: check.getSourceFile(),
@@ -337,6 +354,19 @@ export default class Test extends AuthCommand {
337354
}
338355
}
339356

357+
prepareTestRetryStrategy (retries?: number, configRetries?: number) {
358+
const numRetries = retries ?? configRetries ?? 0
359+
if (numRetries > MAX_RETRIES) {
360+
printLn(`Defaulting to the maximum of ${MAX_RETRIES} retries.`)
361+
}
362+
return numRetries
363+
? RetryStrategyBuilder.fixedStrategy({
364+
maxRetries: Math.min(numRetries, MAX_RETRIES),
365+
baseBackoffSeconds: 0,
366+
})
367+
: null
368+
}
369+
340370
private listChecks (checks: Array<Check>) {
341371
// Sort and print the checks in a way that's consistent with AbstractListReporter
342372
const sortedCheckFiles = [...new Set(checks.map((check) => check.getSourceFile()))].sort()

‎packages/cli/src/commands/trigger.ts‎

Lines changed: 43 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,21 @@ import { loadChecklyConfig } from '../services/checkly-config-loader'
77
import { splitConfigFilePath, getEnvs, getGitInformation, getCiInformation } from '../services/util'
88
import type { Region } from '..'
99
import TriggerRunner, { NoMatchingChecksError } from '../services/trigger-runner'
10-
import { RunLocation, Events, PrivateRunLocation, CheckRunId, DEFAULT_CHECK_RUN_TIMEOUT_SECONDS } from '../services/abstract-check-runner'
10+
import {
11+
RunLocation,
12+
Events,
13+
PrivateRunLocation,
14+
SequenceId,
15+
DEFAULT_CHECK_RUN_TIMEOUT_SECONDS,
16+
} from '../services/abstract-check-runner'
1117
import config from '../services/config'
1218
import { createReporters, ReporterType } from '../reporters/reporter'
19+
import { printLn } from '../reporters/util'
1320
import { TestResultsShortLinks } from '../rest/test-sessions'
14-
import { Session } from '../constructs'
21+
import { Session, RetryStrategyBuilder } from '../constructs'
1522

1623
const DEFAULT_REGION = 'eu-central-1'
24+
const MAX_RETRIES = 3
1725

1826
export default class Trigger extends AuthCommand {
1927
static coreCommand = true
@@ -74,6 +82,9 @@ export default class Trigger extends AuthCommand {
7482
char: 'n',
7583
description: 'A name to use when storing results in Checkly with --record.',
7684
}),
85+
retries: Flags.integer({
86+
description: `[default: 0, max: ${MAX_RETRIES}] How many times to retry a check run.`,
87+
}),
7788
}
7889

7990
async run (): Promise<void> {
@@ -90,6 +101,7 @@ export default class Trigger extends AuthCommand {
90101
env,
91102
'env-file': envFile,
92103
'test-session-name': testSessionName,
104+
retries,
93105
} = flags
94106
const envVars = await getEnvs(envFile, env)
95107
const { configDirectory, configFilenames } = splitConfigFilePath(configFilename)
@@ -106,6 +118,7 @@ export default class Trigger extends AuthCommand {
106118
const verbose = this.prepareVerboseFlag(verboseFlag, checklyConfig?.cli?.verbose)
107119
const reporterTypes = this.prepareReportersTypes(reporterFlag as ReporterType, checklyConfig?.cli?.reporters)
108120
const reporters = createReporters(reporterTypes, location, verbose)
121+
const testRetryStrategy = this.prepareTestRetryStrategy(retries, checklyConfig?.cli?.retries)
109122

110123
const repoInfo = getGitInformation()
111124
const ciInfo = getCiInformation()
@@ -121,22 +134,27 @@ export default class Trigger extends AuthCommand {
121134
repoInfo,
122135
ciInfo.environment,
123136
testSessionName,
137+
testRetryStrategy,
124138
)
125139
// TODO: This is essentially the same for `checkly test`. Maybe reuse code.
126140
runner.on(Events.RUN_STARTED,
127-
(checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId: string) =>
141+
(checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId: string) =>
128142
reporters.forEach(r => r.onBegin(checks, testSessionId)))
129-
runner.on(Events.CHECK_INPROGRESS, (check: any, checkRunId: CheckRunId) => {
130-
reporters.forEach(r => r.onCheckInProgress(check, checkRunId))
143+
runner.on(Events.CHECK_INPROGRESS, (check: any, sequenceId: SequenceId) => {
144+
reporters.forEach(r => r.onCheckInProgress(check, sequenceId))
131145
})
132-
runner.on(Events.CHECK_SUCCESSFUL, (checkRunId, _, result, links?: TestResultsShortLinks) => {
133-
if (result.hasFailures) {
134-
process.exitCode = 1
135-
}
136-
reporters.forEach(r => r.onCheckEnd(checkRunId, result, links))
146+
runner.on(Events.CHECK_ATTEMPT_RESULT, (sequenceId: SequenceId, check, result, links?: TestResultsShortLinks) => {
147+
reporters.forEach(r => r.onCheckAttemptResult(sequenceId, result, links))
137148
})
138-
runner.on(Events.CHECK_FAILED, (checkRunId, check, message: string) => {
139-
reporters.forEach(r => r.onCheckEnd(checkRunId, {
149+
runner.on(Events.CHECK_SUCCESSFUL,
150+
(sequenceId: SequenceId, _, result, testResultId, links?: TestResultsShortLinks) => {
151+
if (result.hasFailures) {
152+
process.exitCode = 1
153+
}
154+
reporters.forEach(r => r.onCheckEnd(sequenceId, result, testResultId, links))
155+
})
156+
runner.on(Events.CHECK_FAILED, (sequenceId: SequenceId, check, message: string) => {
157+
reporters.forEach(r => r.onCheckEnd(sequenceId, {
140158
...check,
141159
hasFailures: true,
142160
runError: message,
@@ -209,4 +227,17 @@ export default class Trigger extends AuthCommand {
209227
}
210228
return reporterFlag ? [reporterFlag] : cliReporters
211229
}
230+
231+
prepareTestRetryStrategy (retries?: number, configRetries?: number) {
232+
const numRetries = retries ?? configRetries ?? 0
233+
if (numRetries > MAX_RETRIES) {
234+
printLn(`Defaulting to the maximum of ${MAX_RETRIES} retries.`)
235+
}
236+
return numRetries
237+
? RetryStrategyBuilder.fixedStrategy({
238+
maxRetries: Math.min(numRetries, MAX_RETRIES),
239+
baseBackoffSeconds: 0,
240+
})
241+
: null
242+
}
212243
}

‎packages/cli/src/reporters/__tests__/__snapshots__/json-builder.spec.ts.snap‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ exports[`JsonBuilder renders JSON markdown output with assets & links: json-with
1313
"durationMilliseconds": 6522,
1414
"filename": "src/__checks__/folder/browser.check.ts",
1515
"link": "https://app.checklyhq.com/test-sessions/0c4c64b3-79c5-44a6-ae07-b580ce73f328/results/702961fd-7e2c-45f0-97be-1aa9eabd4d82",
16-
"runError": "Run error"
16+
"runError": "Run error",
17+
"retries": 0
1718
},
1819
{
1920
"result": "Pass",
@@ -22,7 +23,8 @@ exports[`JsonBuilder renders JSON markdown output with assets & links: json-with
2223
"durationMilliseconds": 1234,
2324
"filename": "src/some-other-folder/api.check.ts",
2425
"link": "https://app.checklyhq.com/test-sessions/0c4c64b3-79c5-44a6-ae07-b580ce73f328/results/1c0be612-a5ec-432e-ac1c-837d2f70c010",
25-
"runError": "Run error"
26+
"runError": "Run error",
27+
"retries": 0
2628
}
2729
]
2830
}"
@@ -40,7 +42,8 @@ exports[`JsonBuilder renders basic JSON output with no assets & links: json-basi
4042
"durationMilliseconds": 6522,
4143
"filename": "src/__checks__/folder/browser.check.ts",
4244
"link": null,
43-
"runError": null
45+
"runError": null,
46+
"retries": 0
4447
},
4548
{
4649
"result": "Pass",
@@ -49,7 +52,8 @@ exports[`JsonBuilder renders basic JSON output with no assets & links: json-basi
4952
"durationMilliseconds": 1234,
5053
"filename": "src/some-other-folder/api.check.ts",
5154
"link": null,
52-
"runError": null
55+
"runError": null,
56+
"retries": 0
5357
}
5458
]
5559
}"
@@ -67,7 +71,8 @@ exports[`JsonBuilder renders basic JSON output with run errors: json-basic 1`] =
6771
"durationMilliseconds": 6522,
6872
"filename": "src/__checks__/folder/browser.check.ts",
6973
"link": null,
70-
"runError": "Run error"
74+
"runError": "Run error",
75+
"retries": 0
7176
},
7277
{
7378
"result": "Pass",
@@ -76,7 +81,8 @@ exports[`JsonBuilder renders basic JSON output with run errors: json-basic 1`] =
7681
"durationMilliseconds": 1234,
7782
"filename": "src/some-other-folder/api.check.ts",
7883
"link": null,
79-
"runError": "Run error"
84+
"runError": "Run error",
85+
"retries": 0
8086
}
8187
]
8288
}"

‎packages/cli/src/reporters/__tests__/fixtures/api-check-result.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ export const apiCheckResult = {
55
sourceInfo: {
66
checkRunId: '4f20dfa7-8c66-4a15-8c43-5dc24f6206c6',
77
checkRunSuiteId: '6390a87e-89c7-4295-b6f8-b23e87922ef3',
8+
sequenceId: '72c5d10f-fc68-4361-a779-8543575336ae',
89
ephemeral: true,
910
},
1011
checkRunId: '1c0be612-a5ec-432e-ac1c-837d2f70c010',

0 commit comments

Comments
 (0)