chore(ci): use GitHub Actions instead of SauceLabs (#5548)

wjhsf · web-flow · commit abe6198ac386 · 2025-11-06T15:38:32.000Z
* chore(ci): use GitHub Actions for cross-platform tests, not SuaceLabs Don't delete SauceLabs tho (yet) * chore: not a template string * test(integration): disable flaky tests (temporarily) * chore(integration-tests): remove leading `test-` from filenames It's implicit in the files being in the test directory and long filenames cause problems on windows * chore(integration-tests): `delegate-focus-` -> `delegates-focus-` * chore(integration-tests): rename `tab-index-` to `tabindex-` * chore(integration-tests): remove `-elements` * chore(integration-tests): rename `previous-` to `prev-` * test(integration-tests): await async method * chore(integration-tests): fully remove `test-` prefix * test(integration-tests): derive test names instead of hard-coding easier to maintain and uncovered a few broken tests! * chore(integration-tests): rename way-too-long test name * chore(ci): attempt to run bash on windows and everywhere * chore(ci): declare env vars in more cross-platform manner * chore(ci): run yarn install in bash * chore(ci): give up and use cross-env * chore(ci): try without `exec` * test(wtr): change variable name is SAUCE defined in CI? * chore(ci): remove saucelabs * test(ci): run unit tests on windows * chore(ci): why different? * chore(ci): avoid unnecessasry rebuild * chore(ci): windows compat * Revert "chore(ci): windows compat" This reverts commit e4279e3. * chore(ci): windows compat * chore(ci): update actions versions * chore(ci): install chromedriver as part of setup-chrome * chore(ci): more sloppy windows compat * chore(ci): more windowsy * chore: add package manager * chore(ci): more windowsy * chore(ci): more windowsy * chore(ci): more windowsy * chore(ci): crlf smh * chore(ci): revert setup-chrome@v2 * chore: include call stack * chore: debug * chore: actually, nvm don't need to change this file for this task * test(wtr): jsdoc for the doc gods * chore(ci): update input descriptions * chore: revert unrelated changes * chore: more windowsy * chore: revert unrelated changes * test(wtr): disable tests broken on windows * revert: just log text * chore(ci): remove unnecessary env var * chore(ci): add names for clarity * chore: move `x` to `c` maybe makes windows work * chore: update usage of `x` to `c` maybe makes windows work * chore: update usage of `X-` to `C-` maybe makes windows work * test(wtr): fix scope tokens * test(wtr): order can be different * test(wtr): try to normalize filenames * chore: update yarn.lock * test(wtr): enable browser logs * test(wtr): filter out [WDIO] logs * test(wtr): try to fix windows * test(wtr): no browser logs they're mostly noise * test(wtr): test objects in matrix * Revert "test(wtr): test objects in matrix" This reverts commit 4f1181c. * chore(ci): split jobs to make faster, maybe * chore(ci): fail-fast: false * chore(ci): merge jobs * chore(ci): merge more jobs * chore(ci): reconfigure so it hopefully works * chore(ci): consolidate hydration tests * chore(ci): exclude engine-server flag + SSR v2 runtime combo * chore(ci): add default env hydration tests * chore(ci): use correct SSR package for hydration tests * chore(ci): give up on windows * chore(ci): give up on windows * Revert "chore: move `x` to `c`" This reverts commit 6f32328. * test(wtr): revert usage of `c` to `x` * Apply suggestion from @wjhsf * test(wtr): revert `c-` to `x-` * test(wtr): revert scope tokens * test(wtr): re-enable tests that only fail on windows * chore: revert windows-only changes * chore: remove windows compat file * test(wtr): restore saucelabs manual mode * docs(wtr): add note about saucelabs * chore(ci): clean up integration test matrix * chore(ci): clean up hydration test matrix
diff --git a/.github/workflows/web-test-runner.yml b/.github/workflows/web-test-runner.yml
@@ -11,36 +11,62 @@ on:
     pull_request:
 
 env:
-    SAUCE_USERNAME: ${{secrets.SAUCE_USERNAME}}
-    SAUCE_ACCESS_KEY: ${{secrets.SAUCE_ACCESS_KEY}}
-    GITHUB_RUN_ID: ${{github.run_id}}
     COVERAGE: '1'
     NODE_VERSION: '20.19.4'
 
+defaults:
+    run:
+        working-directory: packages/@lwc/integration-not-karma
+
 jobs:
     # We run tests in parallel to speed up CI, with an arbitrary goal of ~20 minutes per job.
-    # Test grouping is also somewhat arbitrary.
     # TODO: upload result artifacts
 
     integration-tests:
-        name: Integration tests (${{ matrix.shadow_mode }} shadow)
+        name: Integration tests (${{ matrix.shadow_mode }} shadow; ${{ matrix.env || 'default env' }})
+        runs-on: ubuntu-22.04
         strategy:
+            fail-fast: false
             matrix:
                 shadow_mode: [native, synthetic]
+                env:
+                    - ''
+                    # TODO: Fix
+                    # - DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1
+                    - DISABLE_STATIC_CONTENT_OPTIMIZATION=1
+                    - ENABLE_ARIA_REFLECTION_GLOBAL_POLYFILL=1
+                    - NODE_ENV_FOR_TEST=1
+                    - API_VERSION=58
+                    - API_VERSION=59
+                    - API_VERSION=60
+                    - API_VERSION=61
+                    - API_VERSION=62
+                    # No need for 63-65
+                    - API_VERSION=66
+
+                include:
+                    # Test modes for only synthetic or only native
+                    - shadow_mode: synthetic
+                      env: FORCE_NATIVE_SHADOW_MODE_FOR_TEST=1
+                    - shadow_mode: synthetic
+                      env: DISABLE_DETACHED_REHYDRATION=1
+                    # TODO: Fix
+                    # - shadow_mode: synthetic
+                    #   env: DISABLE_DETACHED_REHYDRATION=1 DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1
+                    - shadow_mode: native
+                      env: DISABLE_SYNTHETIC_SHADOW_SUPPORT_IN_COMPILER=1
+                    - shadow_mode: native
+                      env: DISABLE_SYNTHETIC_SHADOW_SUPPORT_IN_COMPILER=1 DISABLE_STATIC_CONTENT_OPTIMIZATION=1
 
-        runs-on: ubuntu-22.04
         env:
-            SAUCE_TUNNEL_ID: gha-${{github.run_id}}-wtr-integration-${{ matrix.shadow_mode }}-1
             SHADOW_MODE_OVERRIDE: ${{ matrix.shadow_mode }}
-        defaults:
-            run:
-                working-directory: ./packages/@lwc/integration-not-karma
+
         steps:
             - name: Checkout repository
-              uses: actions/checkout@v4
+              uses: actions/checkout@v5
 
             - name: Setup Node
-              uses: actions/setup-node@v4
+              uses: actions/setup-node@v6
               with:
                   node-version: ${{ env.NODE_VERSION }}
                   cache: 'yarn'
@@ -49,118 +75,55 @@ jobs:
               run: yarn install --frozen-lockfile
               working-directory: ./
 
-            - uses: saucelabs/sauce-connect-action@v3.0.0
-              with:
-                  username: ${{ secrets.SAUCE_USERNAME }}
-                  accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
-                  tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
-                  region: us
-
-            - run: yarn exec -- playwright install chrome firefox webkit --with-deps
-            - run: yarn test
-            - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 yarn test || true
-            - run: DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test
-            - run: ENABLE_ARIA_REFLECTION_GLOBAL_POLYFILL=1 yarn test
-            - run: NODE_ENV_FOR_TEST=production yarn test
-
-    integration-tests-api-versions:
-        name: Integration tests (${{ matrix.shadow_mode }} shadow) - API versions
-        strategy:
-            matrix:
-                shadow_mode: [native, synthetic]
+            - name: Install playwright dependencies
+              run: yarn playwright install chrome firefox webkit --with-deps
 
-        runs-on: ubuntu-22.04
-        env:
-            SAUCE_TUNNEL_ID: gha-${{github.run_id}}-wtr-integration-${{ matrix.shadow_mode }}-2
-            SHADOW_MODE_OVERRIDE: ${{ matrix.shadow_mode }}
-        defaults:
-            run:
-                working-directory: ./packages/@lwc/integration-not-karma
-        steps:
-            - name: Checkout repository
-              uses: actions/checkout@v4
+            - name: Set env vars
+              run: echo '${{ matrix.env }}' | tr ' ' '\n' >> "$GITHUB_ENV"
 
-            - name: Setup Node
-              uses: actions/setup-node@v4
-              with:
-                  node-version: ${{ env.NODE_VERSION }}
-                  cache: 'yarn'
+            - name: Sanity check env vars
+              run: |
+                  echo "API_VERSION=$API_VERSION"
+                  echo "DISABLE_DETACHED_REHYDRATION=$DISABLE_DETACHED_REHYDRATION"
+                  echo "DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=$DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE"
+                  echo "DISABLE_STATIC_CONTENT_OPTIMIZATION=$DISABLE_STATIC_CONTENT_OPTIMIZATION"
+                  echo "DISABLE_SYNTHETIC_SHADOW_SUPPORT_IN_COMPILER=$DISABLE_SYNTHETIC_SHADOW_SUPPORT_IN_COMPILER"
+                  echo "ENABLE_ARIA_REFLECTION_GLOBAL_POLYFILL=$ENABLE_ARIA_REFLECTION_GLOBAL_POLYFILL"
+                  echo "FORCE_NATIVE_SHADOW_MODE_FOR_TEST=$FORCE_NATIVE_SHADOW_MODE_FOR_TEST"
+                  echo "NODE_ENV_FOR_TEST=$NODE_ENV_FOR_TEST"
 
-            - name: Install dependencies
-              run: yarn install --frozen-lockfile
-              working-directory: ./
+            - name: Run integration tests
+              run: yarn test
 
-            - uses: saucelabs/sauce-connect-action@v3.0.0
-              with:
-                  username: ${{ secrets.SAUCE_USERNAME }}
-                  accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
-                  tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
-                  region: us
-
-            - run: yarn exec -- playwright install chrome firefox webkit --with-deps
-            - run: API_VERSION=58 yarn test
-            - run: API_VERSION=59 yarn test
-            - run: API_VERSION=60 yarn test
-            - run: API_VERSION=61 yarn test
-            - run: API_VERSION=62 yarn test
-            - run: API_VERSION=66 yarn test
-
-    integration-tests-not-both-modes:
-        # Tests that should run in only synthetic or native shadow, not both
-        name: Integration tests (singleton batch)
+    hydration-tests:
+        name: Hydration tests (${{ matrix.engine_server && 'engine-server' || 'SSR v2' }}; ${{ matrix.env || 'default env' }})
         runs-on: ubuntu-22.04
-        env:
-            SAUCE_TUNNEL_ID: gha-${{github.run_id}}-wtr-integration-${{ matrix.shadow_mode }}-3
-            SHADOW_MODE_OVERRIDE: ${{ matrix.shadow_mode }}
-        defaults:
-            run:
-                working-directory: ./packages/@lwc/integration-not-karma
-        steps:
-            - name: Checkout repository
-              uses: actions/checkout@v4
-
-            - name: Setup Node
-              uses: actions/setup-node@v4
-              with:
-                  node-version: ${{ env.NODE_VERSION }}
-                  cache: 'yarn'
-
-            - name: Install dependencies
-              run: yarn install --frozen-lockfile
-              working-directory: ./
+        strategy:
+            fail-fast: false
+            matrix:
+                engine_server: ['', 1]
+                env:
+                    - ''
+                    - SHADOW_MODE_OVERRIDE=synthetic
+                    - NODE_ENV_FOR_TEST=production
+                    - DISABLE_STATIC_CONTENT_OPTIMIZATION=1
+                include:
+                    - engine_server: 1
+                      env: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1
+                    - engine_server: 1
+                      env: DISABLE_DETACHED_REHYDRATION=1
+                    - engine_server: 1
+                      env: DISABLE_DETACHED_REHYDRATION=1 DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1
 
-            - uses: saucelabs/sauce-connect-action@v3.0.0
-              with:
-                  username: ${{ secrets.SAUCE_USERNAME }}
-                  accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
-                  tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
-                  region: us
-
-            - run: yarn exec -- playwright install chrome firefox webkit --with-deps
-            # Synthetic shadow only
-            - run: LEGACY_BROWSERS=1 yarn test || true
-            - run: FORCE_NATIVE_SHADOW_MODE_FOR_TEST=1 yarn test
-            - run: DISABLE_DETACHED_REHYDRATION=1 yarn test
-            - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 DISABLE_DETACHED_REHYDRATION=1 yarn test || true
-
-            # Native shadow only -- don't forget SHADOW_MODE_OVERRIDE!
-            - run: SHADOW_MODE_OVERRIDE=native DISABLE_SYNTHETIC_SHADOW_SUPPORT_IN_COMPILER=1 yarn test
-            - run: SHADOW_MODE_OVERRIDE=native DISABLE_SYNTHETIC_SHADOW_SUPPORT_IN_COMPILER=1 DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test
-
-    hydration-tests-ssr-v2:
-        name: Hydration tests (SSR v2)
-        runs-on: ubuntu-22.04
         env:
-            SAUCE_TUNNEL_ID: gha-${{github.run_id}}-wtr-hydration-1
-        defaults:
-            run:
-                working-directory: ./packages/@lwc/integration-not-karma
+            ENGINE_SERVER: ${{ matrix.engine_server }}
+
         steps:
             - name: Checkout repository
-              uses: actions/checkout@v4
+              uses: actions/checkout@v5
 
             - name: Setup Node
-              uses: actions/setup-node@v4
+              uses: actions/setup-node@v6
               with:
                   node-version: ${{ env.NODE_VERSION }}
                   cache: 'yarn'
@@ -169,56 +132,20 @@ jobs:
               run: yarn install --frozen-lockfile
               working-directory: ./
 
-            - uses: saucelabs/sauce-connect-action@v3.0.0
-              with:
-                  username: ${{ secrets.SAUCE_USERNAME }}
-                  accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
-                  tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
-                  region: us
-
-            - run: yarn exec -- playwright install chrome firefox webkit --with-deps
-            - run: yarn test:hydration
-            - run: SHADOW_MODE_OVERRIDE=synthetic yarn test:hydration
-            - run: NODE_ENV_FOR_TEST=production yarn test:hydration
-            - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 yarn test:hydration
-            - run: DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test:hydration
-
-    hydration-tests-engine-server:
-        name: Hydration tests (engine-server)
-        runs-on: ubuntu-22.04
-        env:
-            SAUCE_TUNNEL_ID: gha-${{github.run_id}}-wtr-hydration-2
-            ENGINE_SERVER: 1
-        defaults:
-            run:
-                working-directory: ./packages/@lwc/integration-not-karma
-        steps:
-            - name: Checkout repository
-              uses: actions/checkout@v4
+            - name: Install playwright dependencies
+              run: yarn playwright install chrome firefox webkit --with-deps
 
-            - name: Setup Node
-              uses: actions/setup-node@v4
-              with:
-                  node-version: ${{ env.NODE_VERSION }}
-                  cache: 'yarn'
+            - name: Set env vars
+              run: echo '${{ matrix.env }}' | tr ' ' '\n' >> "$GITHUB_ENV"
 
-            - name: Install dependencies
-              run: yarn install --frozen-lockfile
-              working-directory: ./
+            - name: Sanity check env vars
+              run: |
+                  echo "SHADOW_MODE_OVERRIDE=$SHADOW_MODE_OVERRIDE"
+                  echo "NODE_ENV_FOR_TEST=$NODE_ENV_FOR_TEST"
+                  echo "DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=$DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE"
+                  echo "DISABLE_STATIC_CONTENT_OPTIMIZATION=$DISABLE_STATIC_CONTENT_OPTIMIZATION"
+                  echo "DISABLE_DETACHED_REHYDRATION=$DISABLE_DETACHED_REHYDRATION"
+                  echo "ENGINE_SERVER=$ENGINE_SERVER"
 
-            - uses: saucelabs/sauce-connect-action@v3.0.0
-              with:
-                  username: ${{ secrets.SAUCE_USERNAME }}
-                  accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
-                  tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
-                  region: us
-
-            - run: yarn exec -- playwright install chrome firefox webkit --with-deps
-            # NOTE: All tests have ENGINE_SERVER=1 already set
-            - run: yarn test:hydration
-            - run: SHADOW_MODE_OVERRIDE=synthetic yarn test:hydration
-            - run: NODE_ENV_FOR_TEST=production yarn test:hydration
-            - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 yarn test:hydration
-            - run: DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test:hydration
-            - run: DISABLE_DETACHED_REHYDRATION=1 yarn test:hydration
-            - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 DISABLE_DETACHED_REHYDRATION=1 yarn test:hydration
+            - name: Run hydration tests
+              run: yarn test:hydration
diff --git a/packages/@lwc/integration-not-karma/README.md b/packages/@lwc/integration-not-karma/README.md
@@ -20,6 +20,10 @@ Environment variables are used as controls to run tests in different modes (e.g
 - `test`: The test directory for integration tests.
 - `test-hydration`: The test directory for hydration tests.
 
+### Test Runners
+
+By default, the tests are run using Playwright. If the `USE_SAUCE` env var is set, then SauceLabs is used instead. To use SauceLabs, the `SAUCE_USERNAME`, `SAUCE_ACCESS_KEY`, and `SAUCE_TUNNEL_ID` env vars must be set.
+
 ### Integration Tests
 
 Integration tests are simply `.spec.js` files that run in the browser. LWC components are transformed by a plugin defined in `serve-integration.js`.
diff --git a/packages/@lwc/integration-not-karma/configs/shared/browsers.js b/packages/@lwc/integration-not-karma/configs/shared/browsers.js
@@ -3,7 +3,7 @@ import { createSauceLabsLauncher } from '@web/test-runner-saucelabs';
 
 /** @type {(options: typeof import('../../helpers/options.js')) => import("@web/test-runner").BrowserLauncher[]} */
 export function getBrowsers(options) {
-    if (options.CI) {
+    if (options.USE_SAUCE) {
         if (!options.SAUCE_USERNAME || !options.SAUCE_ACCESS_KEY || !options.SAUCE_TUNNEL_ID) {
             throw new Error(
                 `SAUCE_USERNAME, SAUCE_ACCESS_KEY, and SAUCE_TUNNEL_ID must be configured in CI`
@@ -18,33 +18,32 @@ export function getBrowsers(options) {
                 tunnelName: options.SAUCE_TUNNEL_ID,
             }
         );
-        return [
-            sauceLabsLauncher({
-                browserName: 'chrome',
-                browserVersion: 'latest',
-            }),
-            // sauceLabsLauncher({
-            //     browserName: 'firefox',
-            //     browserVersion: 'latest',
-            // }),
-            // sauceLabsLauncher({
-            //     browserName: 'safari',
-            //     browserVersion: 'latest',
-            // }),
-            // ...(options.LEGACY_BROWSERS
-            //     ? [
-            //           sauceLabsLauncher({
-            //               browserName: 'chrome',
-            //               browserVersion: 'latest-2',
-            //           }),
-            //           sauceLabsLauncher({
-            //               browserName: 'safari',
-            //               browserVersion: 'latest-2',
-            //               platformName: 'Mac 13', // update with new releases
-            //           }),
-            //       ]
-            //     : []),
-        ];
+        return options.LEGACY_BROWSERS
+            ? [
+                  sauceLabsLauncher({
+                      browserName: 'chrome',
+                      browserVersion: 'latest-2',
+                  }),
+                  sauceLabsLauncher({
+                      browserName: 'safari',
+                      browserVersion: 'latest-2',
+                      platformName: 'Mac 13', // update with new releases
+                  }),
+              ]
+            : [
+                  sauceLabsLauncher({
+                      browserName: 'chrome',
+                      browserVersion: 'latest',
+                  }),
+                  sauceLabsLauncher({
+                      browserName: 'firefox',
+                      browserVersion: 'latest',
+                  }),
+                  sauceLabsLauncher({
+                      browserName: 'safari',
+                      browserVersion: 'latest',
+                  }),
+              ];
     } else {
         return [
             playwrightLauncher({ product: 'chromium' }),
diff --git a/packages/@lwc/integration-not-karma/helpers/options.js b/packages/@lwc/integration-not-karma/helpers/options.js
