feat: add burn parameter to repeat filtered tests (#33)

* feat: add burn parameter to repeat filtered tests

* update README

Author: bahmutov

.github/workflows/ci.yml

@@ -125,6 +125,13 @@ jobs:
             --env grepTags=@smoke \
             --expect expects/describe-tags-spec.json
 
+      # repeat the selected test 3 times
+      - name: Burn grepped test 🧪
+        run: |
+          npx cypress-expect \
+            --env grep="hello w",burn=3 \
+            --expect expects/hello-burn.json
+
       - name: Semantic Release 🚀
         uses: cycjimmy/semantic-release-action@v2
         with:

README.md

@@ -212,6 +212,16 @@ You can run tests that match one tag or another using spaces. Make sure to quote
 --env grepTags='@slow @critical'
 ```
 
+## Burn
+
+You can repeat (burn) the filtered tests to make sure they are flake-free
+
+```
+npx cypress run --env grep="hello world",burn=5
+```
+
+You can pass the number of times to run the tests via environment name `burn` or `grepBurn` or `grep-burn`. Note, if a lot of tests match the grep and grep tags, a lot of tests will be burnt!
+
 ## General advice
 
 - keep it simple.

expects/hello-burn.json

@@ -0,0 +1,9 @@
+{
+  "hello world: burning 1 of 3": "passed",
+  "hello world: burning 2 of 3": "passed",
+  "hello world: burning 3 of 3": "passed",
+  "works": "pending",
+  "works 2 @tag1": "pending",
+  "works 2 @tag1 @tag2": "pending",
+  "works @tag2": "pending"
+}

src/support.js

@@ -24,7 +24,15 @@ function cypressGrep() {
     return
   }
 
-  debug('grep %o', { grep, grepTags })
+  /** @type {number} Number of times to repeat each running test */
+  const grepBurn =
+    Cypress.env('grepBurn') ||
+    Cypress.env('grep-burn') ||
+    Cypress.env('burn') ||
+    1
+
+  debug('grep %o', { grep, grepTags, grepBurn })
+  // TODO validate grepBurn value
   const parsedGrep = parseGrep(grep, grepTags)
   debug('parsed grep %o', parsedGrep)
 
@@ -47,6 +55,13 @@ function cypressGrep() {
     const shouldRun = shouldTestRun(parsedGrep, name, configTags)
 
     if (shouldRun) {
+      if (grepBurn > 1) {
+        // repeat the same test to make sure it is solid
+        return Cypress._.times(grepBurn, (k) => {
+          const fullName = `${name}: burning ${k + 1} of ${grepBurn}`
+          _it(fullName, options, callback)
+        })
+      }
       return _it(name, options, callback)
     }