diff --git a/README.md b/README.md index 39311ee18..0c4557ec9 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,10 @@ runner.run(collection, { // calls the `item` and `iteration` callbacks and does not run any further items (requests) stopOnFailure: true, + // - gracefully halts on test failures only (not on errors). + // calls the `item` and `iteration` callbacks and does not run any further items (requests) + stopOnAssertionFailure: true, + // - abruptly halts the run on errors or test failures, and directly calls the `done` callback abortOnFailure: true, diff --git a/lib/runner/extensions/event.command.js b/lib/runner/extensions/event.command.js index 9cb47f859..5a5408053 100644 --- a/lib/runner/extensions/event.command.js +++ b/lib/runner/extensions/event.command.js @@ -221,6 +221,7 @@ module.exports = { * execution of a script will stop executing any further scripts * @param {Boolean} [payload.abortOnFailure] - * @param {Boolean} [payload.stopOnFailure] - + * @param {Boolean} [payload.stopOnAssertionFailure] - * @param {Function} next - * * @note - in order to raise trigger for the entire event, ensure your extension has registered the triggers @@ -243,6 +244,7 @@ module.exports = { // @todo: find a better home for this option processing abortOnFailure = payload.abortOnFailure, stopOnFailure = payload.stopOnFailure, + stopOnAssertionFailure = payload.stopOnAssertionFailure, packageResolver = _.get(this, 'options.script.packageResolver'), @@ -318,7 +320,7 @@ module.exports = { // add event listener to trap all assertion events, but only if needed. to avoid needlessly accumulate // stuff in memory. - (abortOnFailure || stopOnFailure) && + (abortOnFailure || stopOnFailure || stopOnAssertionFailure) && this.host.on(EXECUTION_ASSERTION_EVENT_BASE + executionId, function (scriptCursor, assertions) { _.forEach(assertions, function (assertion) { assertion && !assertion.passed && assertionFailed.push(assertion.name); @@ -581,8 +583,16 @@ module.exports = { }); // Get the failures. If there was an error running the script itself, that takes precedence - if (!err && (abortOnFailure || stopOnFailure)) { - err = postProcessContext(result, assertionFailed); // also use async assertions + // For stopOnAssertionFailure, we only create an error if there are assertion failures + // For stopOnFailure/abortOnFailure, we create error for any failures + if (!err && (abortOnFailure || stopOnFailure || stopOnAssertionFailure)) { + var assertionError = postProcessContext(result, assertionFailed); + + // For stopOnAssertionFailure, only set err if it's an assertion failure + // For stopOnFailure/abortOnFailure, set err for any error + if (abortOnFailure || stopOnFailure || (stopOnAssertionFailure && assertionError)) { + err = assertionError; + } } // Ensure that we have SDK instances, not serialized plain objects. @@ -642,7 +652,21 @@ module.exports = { } // move to next script and pass on the results for accumulation - done(((stopOnScriptError || abortOnError || stopOnFailure) && err) ? err : null, _.assign({ + // For stopOnAssertionFailure, only stop if the error is an assertion failure + var shouldStop = false; + + if (err) { + var isAssertionFailure = err.name === ASSERTION_FAILURE || + (err.error && err.error.name === ASSERTION_FAILURE); + + if (stopOnScriptError || abortOnError || stopOnFailure) { + shouldStop = true; + } else if (stopOnAssertionFailure && isAssertionFailure) { + shouldStop = true; + } + } + + done(shouldStop ? err : null, _.assign({ event, script, result diff --git a/lib/runner/extensions/item.command.js b/lib/runner/extensions/item.command.js index ebfa3cc70..899880911 100644 --- a/lib/runner/extensions/item.command.js +++ b/lib/runner/extensions/item.command.js @@ -119,6 +119,7 @@ module.exports = { // still not sure whether that is the right place for it to be. abortOnFailure = this.options.abortOnFailure, stopOnFailure = this.options.stopOnFailure, + stopOnAssertionFailure = this.options.stopOnAssertionFailure, delay = _.get(this.options, 'delay.item'), ctxTemplate; @@ -155,10 +156,12 @@ module.exports = { // No need to include vaultSecrets here as runtime takes care of tracking internally trackContext: ['globals', 'environment', 'collectionVariables'], stopOnScriptError: stopOnError, - stopOnFailure: stopOnFailure + stopOnFailure: stopOnFailure, + stopOnAssertionFailure: stopOnAssertionFailure }).done(function (prereqExecutions, prereqExecutionError, shouldSkipExecution) { // if stop on error is marked and script executions had an error, // do not proceed with more commands, instead we bail out + // Note: stopOnAssertionFailure doesn't stop on execution errors, only assertion failures if ((stopOnError || stopOnFailure) && prereqExecutionError) { this.triggers.item(null, coords, item); // @todo - should this trigger receive error? @@ -239,7 +242,8 @@ module.exports = { trackContext: ['tests', 'globals', 'environment', 'collectionVariables'], stopOnScriptError: stopOnError, abortOnFailure: abortOnFailure, - stopOnFailure: stopOnFailure + stopOnFailure: stopOnFailure, + stopOnAssertionFailure: stopOnAssertionFailure }).done(function (testExecutions, testExecutionError) { var visualizerData = extractVisualizerData(prereqExecutions, testExecutions), visualizerResult; @@ -272,8 +276,21 @@ module.exports = { // @note request mutations are not persisted across iterations item.request = originalRequest; - callback && callback.call(this, ((stopOnError || stopOnFailure) && testExecutionError) ? - testExecutionError : null, { + // Check if we should stop: stopOnError/stopOnFailure stop on any error, + // stopOnAssertionFailure only stops on assertion failures + var shouldStop = false; + if (testExecutionError) { + var isAssertionFailure = testExecutionError.name === 'AssertionFailure' || + (testExecutionError.error && testExecutionError.error.name === 'AssertionFailure'); + + if (stopOnError || stopOnFailure) { + shouldStop = true; + } else if (stopOnAssertionFailure && isAssertionFailure) { + shouldStop = true; + } + } + + callback && callback.call(this, shouldStop ? testExecutionError : null, { prerequest: prereqExecutions, request: request, response: response, diff --git a/lib/runner/util.js b/lib/runner/util.js index e195a1e5a..5a9f90948 100644 --- a/lib/runner/util.js +++ b/lib/runner/util.js @@ -332,7 +332,7 @@ module.exports = { * @param {Object} options.coords - Current coordinates * @param {Object} options.executions - Execution results (prerequest and test) * @param {Error} options.executionError - Any execution error - * @param {Object} options.runnerOptions - Runner options (disableSNR, stopOnFailure) + * @param {Object} options.runnerOptions - Runner options (disableSNR, stopOnFailure, stopOnAssertionFailure) * @param {Object} options.snrHash - SNR lookup hash * @param {Array} options.items - Collection items for SNR hash preparation * @returns {Object} Processing result with nextCoords, seekingToStart, stopRunNow flags @@ -343,7 +343,8 @@ module.exports = { nextCoords, seekingToStart, stopRunNow, - stopOnFailure = runnerOptions.stopOnFailure; + stopOnFailure = runnerOptions.stopOnFailure, + stopOnAssertionFailure = runnerOptions.stopOnAssertionFailure; if (!executionError) { // extract set next request @@ -380,7 +381,17 @@ module.exports = { (snr.defined || executionError) && (nextCoords.position = nextCoords.length - 1); // If we need to stop on a run, we set the stop flag to true. - (stopOnFailure && executionError) && (stopRunNow = true); + // stopOnFailure stops on any error, stopOnAssertionFailure only stops on assertion failures + if (executionError) { + var isAssertionFailure = executionError.name === 'AssertionFailure' || + (executionError.error && executionError.error.name === 'AssertionFailure'); + + if (stopOnFailure) { + stopRunNow = true; + } else if (stopOnAssertionFailure && isAssertionFailure) { + stopRunNow = true; + } + } } // @todo - do this in unhacky way