Adds an option to stop run after an assertion has failed

README.md

@@ -42,6 +42,10 @@ runner.run(collection, {
     //   calls the `item` and `iteration` callbacks and does not run any further items (requests)
     stopOnFailure: true,
 
+    // - gracefully halts on test failures only (not on errors).
+    //   calls the `item` and `iteration` callbacks and does not run any further items (requests)
+    stopOnAssertionFailure: true,
+
     // - abruptly halts the run on errors or test failures, and directly calls the `done` callback
     abortOnFailure: true,
 

lib/runner/extensions/event.command.js

@@ -221,6 +221,7 @@
          * execution of a script will stop executing any further scripts
          * @param {Boolean} [payload.abortOnFailure] -
          * @param {Boolean} [payload.stopOnFailure] -
+         * @param {Boolean} [payload.stopOnAssertionFailure] -
          * @param {Function} next -
          *
          * @note - in order to raise trigger for the entire event, ensure your extension has registered the triggers
@@ -243,6 +244,7 @@
                 // @todo: find a better home for this option processing
                 abortOnFailure = payload.abortOnFailure,
                 stopOnFailure = payload.stopOnFailure,
+                stopOnAssertionFailure = payload.stopOnAssertionFailure,
 
                 packageResolver = _.get(this, 'options.script.packageResolver'),
 
@@ -318,7 +320,7 @@
 
                 // add event listener to trap all assertion events, but only if needed. to avoid needlessly accumulate
                 // stuff in memory.
-                (abortOnFailure || stopOnFailure) &&
+                (abortOnFailure || stopOnFailure || stopOnAssertionFailure) &&
                     this.host.on(EXECUTION_ASSERTION_EVENT_BASE + executionId, function (scriptCursor, assertions) {
                         _.forEach(assertions, function (assertion) {
                             assertion && !assertion.passed && assertionFailed.push(assertion.name);
@@ -581,8 +583,16 @@
                             });
 
                             // Get the failures. If there was an error running the script itself, that takes precedence
-                            if (!err && (abortOnFailure || stopOnFailure)) {
-                                err = postProcessContext(result, assertionFailed); // also use async assertions
+                            // For stopOnAssertionFailure, we only create an error if there are assertion failures
+                            // For stopOnFailure/abortOnFailure, we create error for any failures
+                            if (!err && (abortOnFailure || stopOnFailure || stopOnAssertionFailure)) {
+                                var assertionError = postProcessContext(result, assertionFailed);
+
+                                // For stopOnAssertionFailure, only set err if it's an assertion failure
+                                // For stopOnFailure/abortOnFailure, set err for any error
+                                if (abortOnFailure || stopOnFailure || (stopOnAssertionFailure && assertionError)) {
+                                    err = assertionError;
+                                }
                             }
 
                             // Ensure that we have SDK instances, not serialized plain objects.
@@ -642,7 +652,21 @@
                             }
 
                             // move to next script and pass on the results for accumulation
-                            done(((stopOnScriptError || abortOnError || stopOnFailure) && err) ? err : null, _.assign({
+                            // For stopOnAssertionFailure, only stop if the error is an assertion failure
+                            var shouldStop = false;
+
+                            if (err) {
+                                var isAssertionFailure = err.name === ASSERTION_FAILURE ||
+                                    (err.error && err.error.name === ASSERTION_FAILURE);
+
+                                if (stopOnScriptError || abortOnError || stopOnFailure) {
+                                    shouldStop = true;
+                                } else if (stopOnAssertionFailure && isAssertionFailure) {
+                                    shouldStop = true;
+                                }
+                            }
+
+                            done(shouldStop ? err : null, _.assign({
                                 event,
                                 script,
                                 result

lib/runner/extensions/item.command.js

@@ -119,6 +119,7 @@
                 // still not sure whether that is the right place for it to be.
                 abortOnFailure = this.options.abortOnFailure,
                 stopOnFailure = this.options.stopOnFailure,
+                stopOnAssertionFailure = this.options.stopOnAssertionFailure,
                 delay = _.get(this.options, 'delay.item'),
 
                 ctxTemplate;
@@ -155,10 +156,12 @@
                     // No need to include vaultSecrets here as runtime takes care of tracking internally
                     trackContext: ['globals', 'environment', 'collectionVariables'],
                     stopOnScriptError: stopOnError,
-                    stopOnFailure: stopOnFailure
+                    stopOnFailure: stopOnFailure,
+                    stopOnAssertionFailure: stopOnAssertionFailure
                 }).done(function (prereqExecutions, prereqExecutionError, shouldSkipExecution) {
                     // if stop on error is marked and script executions had an error,
                     // do not proceed with more commands, instead we bail out
+                    // Note: stopOnAssertionFailure doesn't stop on execution errors, only assertion failures
                     if ((stopOnError || stopOnFailure) && prereqExecutionError) {
                         this.triggers.item(null, coords, item); // @todo - should this trigger receive error?
 
@@ -239,7 +242,8 @@
                             trackContext: ['tests', 'globals', 'environment', 'collectionVariables'],
                             stopOnScriptError: stopOnError,
                             abortOnFailure: abortOnFailure,
-                            stopOnFailure: stopOnFailure
+                            stopOnFailure: stopOnFailure,
+                            stopOnAssertionFailure: stopOnAssertionFailure
                         }).done(function (testExecutions, testExecutionError) {
                             var visualizerData = extractVisualizerData(prereqExecutions, testExecutions),
                                 visualizerResult;
@@ -272,8 +276,21 @@
                             // @note request mutations are not persisted across iterations
                             item.request = originalRequest;
 
-                            callback && callback.call(this, ((stopOnError || stopOnFailure) && testExecutionError) ?
-                                testExecutionError : null, {
+                            // Check if we should stop: stopOnError/stopOnFailure stop on any error,
+                            // stopOnAssertionFailure only stops on assertion failures
+                            var shouldStop = false;
+                            if (testExecutionError) {
+                                var isAssertionFailure = testExecutionError.name === 'AssertionFailure' ||
+                                    (testExecutionError.error && testExecutionError.error.name === 'AssertionFailure');
+
+                                if (stopOnError || stopOnFailure) {
+                                    shouldStop = true;
+                                } else if (stopOnAssertionFailure && isAssertionFailure) {
+                                    shouldStop = true;
+                                }
+                            }
+
+                            callback && callback.call(this, shouldStop ? testExecutionError : null, {
                                 prerequest: prereqExecutions,
                                 request: request,
                                 response: response,

lib/runner/util.js

@@ -332,7 +332,7 @@
      * @param {Object} options.coords - Current coordinates
      * @param {Object} options.executions - Execution results (prerequest and test)
      * @param {Error} options.executionError - Any execution error
-     * @param {Object} options.runnerOptions - Runner options (disableSNR, stopOnFailure)
+     * @param {Object} options.runnerOptions - Runner options (disableSNR, stopOnFailure, stopOnAssertionFailure)
      * @param {Object} options.snrHash - SNR lookup hash
      * @param {Array} options.items - Collection items for SNR hash preparation
      * @returns {Object} Processing result with nextCoords, seekingToStart, stopRunNow flags
@@ -343,7 +343,8 @@
             nextCoords,
             seekingToStart,
             stopRunNow,
-            stopOnFailure = runnerOptions.stopOnFailure;
+            stopOnFailure = runnerOptions.stopOnFailure,
+            stopOnAssertionFailure = runnerOptions.stopOnAssertionFailure;
 
         if (!executionError) {
             // extract set next request
@@ -380,7 +381,17 @@
             (snr.defined || executionError) && (nextCoords.position = nextCoords.length - 1);
 
             // If we need to stop on a run, we set the stop flag to true.
-            (stopOnFailure && executionError) && (stopRunNow = true);
+            // stopOnFailure stops on any error, stopOnAssertionFailure only stops on assertion failures
+            if (executionError) {
+                var isAssertionFailure = executionError.name === 'AssertionFailure' ||
+                    (executionError.error && executionError.error.name === 'AssertionFailure');
+
+                if (stopOnFailure) {
+                    stopRunNow = true;
+                } else if (stopOnAssertionFailure && isAssertionFailure) {
+                    stopRunNow = true;
+                }
+            }
         }
 
         // @todo - do this in unhacky way