Skip to content

Commit fcecf5a

Browse files
shreyas-pandurangashreyas-panduranga
committed
Adds an option to stop run after an assersion has failed
1 parent 40ef304 commit fcecf5a

File tree

4 files changed

+67
-11
lines changed

4 files changed

+67
-11
lines changed

‎README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ runner.run(collection, {
4242
// calls the `item` and `iteration` callbacks and does not run any further items (requests)
4343
stopOnFailure: true,
4444

45+
// - gracefully halts on test failures only (not on errors).
46+
// calls the `item` and `iteration` callbacks and does not run any further items (requests)
47+
stopOnAssertionFailure: true,
48+
4549
// - abruptly halts the run on errors or test failures, and directly calls the `done` callback
4650
abortOnFailure: true,
4751

‎lib/runner/extensions/event.command.js‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,7 @@ module.exports = {
221221
* execution of a script will stop executing any further scripts
222222
* @param {Boolean} [payload.abortOnFailure] -
223223
* @param {Boolean} [payload.stopOnFailure] -
224+
* @param {Boolean} [payload.stopOnAssertionFailure] -
224225
* @param {Function} next -
225226
*
226227
* @note - in order to raise trigger for the entire event, ensure your extension has registered the triggers
@@ -243,6 +244,7 @@ module.exports = {
243244
// @todo: find a better home for this option processing
244245
abortOnFailure = payload.abortOnFailure,
245246
stopOnFailure = payload.stopOnFailure,
247+
stopOnAssertionFailure = payload.stopOnAssertionFailure,
246248

247249
packageResolver = _.get(this, 'options.script.packageResolver'),
248250

@@ -318,7 +320,7 @@ module.exports = {
318320

319321
// add event listener to trap all assertion events, but only if needed. to avoid needlessly accumulate
320322
// stuff in memory.
321-
(abortOnFailure || stopOnFailure) &&
323+
(abortOnFailure || stopOnFailure || stopOnAssertionFailure) &&
322324
this.host.on(EXECUTION_ASSERTION_EVENT_BASE + executionId, function (scriptCursor, assertions) {
323325
_.forEach(assertions, function (assertion) {
324326
assertion && !assertion.passed && assertionFailed.push(assertion.name);
@@ -581,8 +583,16 @@ module.exports = {
581583
});
582584

583585
// Get the failures. If there was an error running the script itself, that takes precedence
584-
if (!err && (abortOnFailure || stopOnFailure)) {
585-
err = postProcessContext(result, assertionFailed); // also use async assertions
586+
// For stopOnAssertionFailure, we only create an error if there are assertion failures
587+
// For stopOnFailure/abortOnFailure, we create error for any failures
588+
if (!err && (abortOnFailure || stopOnFailure || stopOnAssertionFailure)) {
589+
var assertionError = postProcessContext(result, assertionFailed);
590+
591+
// For stopOnAssertionFailure, only set err if it's an assertion failure
592+
// For stopOnFailure/abortOnFailure, set err for any error
593+
if (abortOnFailure || stopOnFailure || (stopOnAssertionFailure && assertionError)) {
594+
err = assertionError;
595+
}
586596
}
587597

588598
// Ensure that we have SDK instances, not serialized plain objects.
@@ -642,7 +652,21 @@ module.exports = {
642652
}
643653

644654
// move to next script and pass on the results for accumulation
645-
done(((stopOnScriptError || abortOnError || stopOnFailure) && err) ? err : null, _.assign({
655+
// For stopOnAssertionFailure, only stop if the error is an assertion failure
656+
var shouldStop = false;
657+
658+
if (err) {
659+
var isAssertionFailure = err.name === ASSERTION_FAILURE ||
660+
(err.error && err.error.name === ASSERTION_FAILURE);
661+
662+
if (stopOnScriptError || abortOnError || stopOnFailure) {
663+
shouldStop = true;
664+
} else if (stopOnAssertionFailure && isAssertionFailure) {
665+
shouldStop = true;
666+
}
667+
}
668+
669+
done(shouldStop ? err : null, _.assign({
646670
event,
647671
script,
648672
result

‎lib/runner/extensions/item.command.js‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ module.exports = {
119119
// still not sure whether that is the right place for it to be.
120120
abortOnFailure = this.options.abortOnFailure,
121121
stopOnFailure = this.options.stopOnFailure,
122+
stopOnAssertionFailure = this.options.stopOnAssertionFailure,
122123
delay = _.get(this.options, 'delay.item'),
123124

124125
ctxTemplate;
@@ -155,10 +156,12 @@ module.exports = {
155156
// No need to include vaultSecrets here as runtime takes care of tracking internally
156157
trackContext: ['globals', 'environment', 'collectionVariables'],
157158
stopOnScriptError: stopOnError,
158-
stopOnFailure: stopOnFailure
159+
stopOnFailure: stopOnFailure,
160+
stopOnAssertionFailure: stopOnAssertionFailure
159161
}).done(function (prereqExecutions, prereqExecutionError, shouldSkipExecution) {
160162
// if stop on error is marked and script executions had an error,
161163
// do not proceed with more commands, instead we bail out
164+
// Note: stopOnAssertionFailure doesn't stop on execution errors, only assertion failures
162165
if ((stopOnError || stopOnFailure) && prereqExecutionError) {
163166
this.triggers.item(null, coords, item); // @todo - should this trigger receive error?
164167

@@ -239,7 +242,8 @@ module.exports = {
239242
trackContext: ['tests', 'globals', 'environment', 'collectionVariables'],
240243
stopOnScriptError: stopOnError,
241244
abortOnFailure: abortOnFailure,
242-
stopOnFailure: stopOnFailure
245+
stopOnFailure: stopOnFailure,
246+
stopOnAssertionFailure: stopOnAssertionFailure
243247
}).done(function (testExecutions, testExecutionError) {
244248
var visualizerData = extractVisualizerData(prereqExecutions, testExecutions),
245249
visualizerResult;
@@ -272,8 +276,21 @@ module.exports = {
272276
// @note request mutations are not persisted across iterations
273277
item.request = originalRequest;
274278

275-
callback && callback.call(this, ((stopOnError || stopOnFailure) && testExecutionError) ?
276-
testExecutionError : null, {
279+
// Check if we should stop: stopOnError/stopOnFailure stop on any error,
280+
// stopOnAssertionFailure only stops on assertion failures
281+
var shouldStop = false;
282+
if (testExecutionError) {
283+
var isAssertionFailure = testExecutionError.name === 'AssertionFailure' ||
284+
(testExecutionError.error && testExecutionError.error.name === 'AssertionFailure');
285+
286+
if (stopOnError || stopOnFailure) {
287+
shouldStop = true;
288+
} else if (stopOnAssertionFailure && isAssertionFailure) {
289+
shouldStop = true;
290+
}
291+
}
292+
293+
callback && callback.call(this, shouldStop ? testExecutionError : null, {
277294
prerequest: prereqExecutions,
278295
request: request,
279296
response: response,

‎lib/runner/util.js‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,7 @@ module.exports = {
332332
* @param {Object} options.coords - Current coordinates
333333
* @param {Object} options.executions - Execution results (prerequest and test)
334334
* @param {Error} options.executionError - Any execution error
335-
* @param {Object} options.runnerOptions - Runner options (disableSNR, stopOnFailure)
335+
* @param {Object} options.runnerOptions - Runner options (disableSNR, stopOnFailure, stopOnAssertionFailure)
336336
* @param {Object} options.snrHash - SNR lookup hash
337337
* @param {Array} options.items - Collection items for SNR hash preparation
338338
* @returns {Object} Processing result with nextCoords, seekingToStart, stopRunNow flags
@@ -343,7 +343,8 @@ module.exports = {
343343
nextCoords,
344344
seekingToStart,
345345
stopRunNow,
346-
stopOnFailure = runnerOptions.stopOnFailure;
346+
stopOnFailure = runnerOptions.stopOnFailure,
347+
stopOnAssertionFailure = runnerOptions.stopOnAssertionFailure;
347348

348349
if (!executionError) {
349350
// extract set next request
@@ -380,7 +381,17 @@ module.exports = {
380381
(snr.defined || executionError) && (nextCoords.position = nextCoords.length - 1);
381382

382383
// If we need to stop on a run, we set the stop flag to true.
383-
(stopOnFailure && executionError) && (stopRunNow = true);
384+
// stopOnFailure stops on any error, stopOnAssertionFailure only stops on assertion failures
385+
if (executionError) {
386+
var isAssertionFailure = executionError.name === 'AssertionFailure' ||
387+
(executionError.error && executionError.error.name === 'AssertionFailure');
388+
389+
if (stopOnFailure) {
390+
stopRunNow = true;
391+
} else if (stopOnAssertionFailure && isAssertionFailure) {
392+
stopRunNow = true;
393+
}
394+
}
384395
}
385396

386397
// @todo - do this in unhacky way

0 commit comments

Comments
 (0)