New built-in, global RandomRunOrderExtension

Supports run order randomization for specifications, features or a
combination of both.

Relates to #1443.

Author: kriegaex

docs/extensions.adoc

@@ -44,6 +44,18 @@ runner {
 
 See the <<parallel_execution.adoc#parallel-execution,Parallel Execution>> section for a detailed description.
 
+=== Random Test Order Configuration
+
+[source,groovy]
+----
+runner {
+  randomizeSpecRunOrder true    // randomize specification run order
+  randomizeFeatureRunOrder true // randomize feature run order within each specification
+}
+----
+
+See the <<#_randomize_run_order>> section for more details.
+
 == Built-In Extensions
 
 Most of Spock's built-in extensions are _annotation-driven_. In other words, they are triggered by annotating a
@@ -709,6 +721,24 @@ runner {
 }
 ----
 
+=== Randomize Run Order
+
+Ideally, automated tests in general and Spock specifications in particular should be independent of each other. The same
+applies to feature methods within a specification. One heuristic way to gain confidence that this is indeed the case, is
+to randomize the order in which specifications and/or features within each specification are executed.
+
+By default, Spock executes both specifications and features in deterministic order, even though users should not rely on
+any particular run order. You can explicitly activate run order randomization separately for specifications and features
+or combine both randomization modes, using the <<spock-configuration-file,Spock Configuration File>>:
+
+[source,groovy]
+----
+runner {
+  randomizeSpecRunOrder true    // randomize specification run order
+  randomizeFeatureRunOrder true // randomize feature run order within each specification
+}
+----
+
 == Third-Party Extensions
 
 You can find a list of third-party extensions in the https://github.com/spockframework/spock/wiki/Third-Party-Extensions[Spock Wiki].

docs/release_notes.adoc

@@ -6,6 +6,8 @@ include::include.adoc[]
 == 2.4 (tbd)
 
 * Support for setting execution order (a.k.a. run order) in `SpecInfo`, can be used by extensions
+* New built-in `RandomRunOrderExtension` supports run order randomization for specifications, features or a combination of both.
+  See manual section <<extensions#_randomize_run_order,Randomize Run Order>>.
 
 == 2.4-M1 (2022-11-30)
 

spock-core/src/main/java/org/spockframework/runtime/extension/builtin/RandomRunOrderExtension.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.spockframework.runtime.extension.builtin;
+
+import org.spockframework.runtime.extension.IGlobalExtension;
+import org.spockframework.runtime.model.FeatureInfo;
+import org.spockframework.runtime.model.SpecInfo;
+import spock.config.RunnerConfiguration;
+
+import java.util.Random;
+
+public class RandomRunOrderExtension implements IGlobalExtension {
+  private static final Random RANDOM = new Random();
+
+  private final RunnerConfiguration config;
+
+  public RandomRunOrderExtension(RunnerConfiguration config) {
+    this.config = config;
+  }
+
+  @Override
+  public void visitSpec(SpecInfo spec) {
+    if (config.randomizeSpecRunOrder)
+      spec.setExecutionOrder(RANDOM.nextInt());
+    if (config.randomizeFeatureRunOrder) {
+      for (FeatureInfo featureInfo : spec.getAllFeatures())
+        featureInfo.setExecutionOrder(RANDOM.nextInt());
+    }
+  }
+}

spock-core/src/main/java/spock/config/RunnerConfiguration.java

@@ -28,7 +28,9 @@
  *     annotation some.pkg.Slow
  *     baseClass IntegrationSpec
  *   }
- *   filterStackTrace true // this is the default
+ *   filterStackTrace true          // this is the default
+ *   randomizeSpecRunOrder false    // this is the default
+ *   randomizeFeatureRunOrder false // this is the default
  * }
  * </pre>
  */
@@ -39,4 +41,6 @@ public class RunnerConfiguration {
   public ParallelConfiguration parallel = new ParallelConfiguration();
   public boolean filterStackTrace = true;
   public boolean optimizeRunOrder = false;
+  public boolean randomizeSpecRunOrder = false;
+  public boolean randomizeFeatureRunOrder = false;
 }

spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension

@@ -1,3 +1,4 @@
 org.spockframework.runtime.extension.builtin.IncludeExcludeExtension
 org.spockframework.runtime.extension.builtin.OptimizeRunOrderExtension
+org.spockframework.runtime.extension.builtin.RandomRunOrderExtension
 org.spockframework.runtime.extension.builtin.UnrollExtension