fix: respect alwaysSaveActualImage: false for checkScreen methods (#1115)

.changeset/fix-always-save-actual-image.md

@@ -0,0 +1,18 @@
+---
+"@wdio/image-comparison-core": patch
+"@wdio/visual-service": patch
+---
+
+## #1115 Respect `alwaysSaveActualImage: false` for `checkScreen` methods
+
+When using visual matchers like `toMatchScreenSnapshot('tag', 0.9)` with `alwaysSaveActualImage: false`, the actual image was still being saved even when the comparison passed within the threshold.
+
+The root cause was that the matcher's expected threshold was not being passed to the core comparison logic. The core used `saveAboveTolerance` (defaulting to 0) to decide whether to save images, while the matcher used the user-provided threshold to determine pass/fail - these were disconnected.
+
+This fix ensures:
+- When `alwaysSaveActualImage: false` and `saveAboveTolerance` is not explicitly set, actual images are never saved (respecting the literal meaning of the option)
+- When `saveAboveTolerance` is explicitly set (like matchers do internally), actual images are saved only when the mismatch exceeds that threshold
+
+# Committers: 1
+
+-   Wim Selles ([@wswebcreation](https://github.com/wswebcreation))

.changeset/fix-save-methods-always-save.md

@@ -0,0 +1,25 @@
+---
+"@wdio/image-comparison-core": patch
+"@wdio/visual-service": patch
+---
+
+## Fix: `save*` methods now always save files regardless of `alwaysSaveActualImage` setting
+
+Previously, when `alwaysSaveActualImage: false` was set in the configuration, `save*` methods (`saveScreen`, `saveElement`, `saveFullPageScreen`, `saveAppScreen`, `saveAppElement`) were not saving files to disk, causing test failures.
+
+The `alwaysSaveActualImage` option is intended to control whether actual images are saved during `check*` methods (comparison operations), not `save*` methods. Since `save*` methods are explicitly designed to save screenshots, they should always save files regardless of this setting.
+
+This fix ensures:
+- `save*` methods always save files to disk, even when `alwaysSaveActualImage: false` is set in the config
+- `alwaysSaveActualImage: false` continues to work correctly for `check*` methods (as intended for issue #1115)
+- The behavior is now consistent: `save*` = always save, `check*` = respect `alwaysSaveActualImage` setting
+
+**Implementation details:**
+- The visual service overrides `alwaysSaveActualImage: true` when calling `save*` methods directly from the browser API
+- `save*` methods respect whatever `alwaysSaveActualImage` value is passed to them (no special logic needed)
+- `check*` methods pass through the config value (which may be `false`), so `save*` methods respect it when called internally
+- This clean separation ensures `save*` methods work correctly when called directly while still respecting `alwaysSaveActualImage` for `check*` methods
+
+# Committers: 1
+
+-   Wim Selles ([@wswebcreation](https://github.com/wswebcreation))

package.json

@@ -19,7 +19,7 @@
     "test:unit": "vitest --coverage --run",
     "test:unit:ui": "vitest --coverage --ui",
     "test:unit:watch": "vitest --coverage --watch",
-    "test.local.init": "rimraf localBaseline && wdio ./tests/configs/wdio.local.init.conf.ts",
+    "test.local.init": "rimraf localBaseline && SAVE_ACTUAL=true wdio ./tests/configs/wdio.local.init.conf.ts",
     "test.local.desktop": "wdio tests/configs/wdio.local.desktop.conf.ts",
     "test.local.emus.app": "wdio tests/configs/wdio.local.android.emus.app.conf.ts",
     "test.local.emus.web": "wdio tests/configs/wdio.local.android.emus.web.conf.ts",

packages/image-comparison-core/src/__snapshots__/base.test.ts.snap

@@ -21,7 +21,6 @@ exports[`BaseClass > initializes default options correctly 1`] = `
     "ignoreNothing": false,
     "rawMisMatchPercentage": false,
     "returnAllCompareData": false,
-    "saveAboveTolerance": 0,
     "scaleImagesToSameSize": false,
   },
   "disableBlinkingCursor": false,

packages/image-comparison-core/src/commands/__snapshots__/saveFullPageScreen.test.ts.snap

@@ -134,9 +134,55 @@ exports[`saveFullPageScreen > should handle missing dimension values with NaN fa
   "isNativeContext": false,
   "tag": "test-fullpage",
   "wicOptions": {
+    "addIOSBezelCorners": false,
+    "addressBarShadowPadding": 6,
     "alwaysSaveActualImage": true,
+    "autoElementScroll": true,
+    "autoSaveBaseline": false,
+    "clearFolder": false,
+    "compareOptions": {
+      "blockOutSideBar": false,
+      "blockOutStatusBar": false,
+      "blockOutToolBar": false,
+      "createJsonReportFiles": false,
+      "diffPixelBoundingBoxProximity": 5,
+      "ignoreAlpha": false,
+      "ignoreAntialiasing": false,
+      "ignoreColors": false,
+      "ignoreLess": false,
+      "ignoreNothing": false,
+      "rawMisMatchPercentage": false,
+      "returnAllCompareData": false,
+      "saveAboveTolerance": 0,
+      "scaleImagesToSameSize": false,
+    },
+    "disableBlinkingCursor": false,
+    "disableCSSAnimation": false,
+    "enableLayoutTesting": false,
+    "enableLegacyScreenshotMethod": false,
     "formatImageName": "{tag}-{logName}-{width}x{height}-dpr-{dpr}",
+    "fullPageScrollTimeout": 1500,
+    "hideScrollBars": true,
+    "isHybridApp": false,
     "savePerInstance": true,
+    "tabbableOptions": {
+      "circle": {
+        "backgroundColor": "rgba(255, 0, 0, 0.4)",
+        "borderColor": "rgba(255, 0, 0, 1)",
+        "borderWidth": 1,
+        "fontColor": "rgba(0, 0, 0, 1)",
+        "fontFamily": "Arial",
+        "fontSize": 10,
+        "size": 10,
+      },
+      "line": {
+        "color": "rgba(255, 0, 0, 1)",
+        "width": 1,
+      },
+    },
+    "toolBarShadowPadding": 6,
+    "userBasedFullPageScreenshot": false,
+    "waitForFontsLoaded": true,
   },
 }
 `;
@@ -557,9 +603,55 @@ exports[`saveFullPageScreen > should take full page screenshots and return resul
   "isNativeContext": false,
   "tag": "test-fullpage",
   "wicOptions": {
+    "addIOSBezelCorners": false,
+    "addressBarShadowPadding": 6,
     "alwaysSaveActualImage": true,
+    "autoElementScroll": true,
+    "autoSaveBaseline": false,
+    "clearFolder": false,
+    "compareOptions": {
+      "blockOutSideBar": false,
+      "blockOutStatusBar": false,
+      "blockOutToolBar": false,
+      "createJsonReportFiles": false,
+      "diffPixelBoundingBoxProximity": 5,
+      "ignoreAlpha": false,
+      "ignoreAntialiasing": false,
+      "ignoreColors": false,
+      "ignoreLess": false,
+      "ignoreNothing": false,
+      "rawMisMatchPercentage": false,
+      "returnAllCompareData": false,
+      "saveAboveTolerance": 0,
+      "scaleImagesToSameSize": false,
+    },
+    "disableBlinkingCursor": false,
+    "disableCSSAnimation": false,
+    "enableLayoutTesting": false,
+    "enableLegacyScreenshotMethod": false,
     "formatImageName": "{tag}-{logName}-{width}x{height}-dpr-{dpr}",
+    "fullPageScrollTimeout": 1500,
+    "hideScrollBars": true,
+    "isHybridApp": false,
     "savePerInstance": true,
+    "tabbableOptions": {
+      "circle": {
+        "backgroundColor": "rgba(255, 0, 0, 0.4)",
+        "borderColor": "rgba(255, 0, 0, 1)",
+        "borderWidth": 1,
+        "fontColor": "rgba(0, 0, 0, 1)",
+        "fontFamily": "Arial",
+        "fontSize": 10,
+        "size": 10,
+      },
+      "line": {
+        "color": "rgba(255, 0, 0, 1)",
+        "width": 1,
+      },
+    },
+    "toolBarShadowPadding": 6,
+    "userBasedFullPageScreenshot": false,
+    "waitForFontsLoaded": true,
   },
 }
 `;

packages/image-comparison-core/src/commands/__snapshots__/saveWebElement.test.ts.snap

@@ -363,9 +363,55 @@ exports[`saveWebElement > should call takeElementScreenshot with BiDi disabled w
   "isNativeContext": false,
   "tag": "test-element",
   "wicOptions": {
+    "addIOSBezelCorners": false,
+    "addressBarShadowPadding": 6,
     "alwaysSaveActualImage": true,
+    "autoElementScroll": true,
+    "autoSaveBaseline": false,
+    "clearFolder": false,
+    "compareOptions": {
+      "blockOutSideBar": false,
+      "blockOutStatusBar": false,
+      "blockOutToolBar": false,
+      "createJsonReportFiles": false,
+      "diffPixelBoundingBoxProximity": 5,
+      "ignoreAlpha": false,
+      "ignoreAntialiasing": false,
+      "ignoreColors": false,
+      "ignoreLess": false,
+      "ignoreNothing": false,
+      "rawMisMatchPercentage": false,
+      "returnAllCompareData": false,
+      "saveAboveTolerance": 0,
+      "scaleImagesToSameSize": false,
+    },
+    "disableBlinkingCursor": false,
+    "disableCSSAnimation": false,
+    "enableLayoutTesting": false,
+    "enableLegacyScreenshotMethod": false,
     "formatImageName": "{tag}-{logName}-{width}x{height}-dpr-{dpr}",
+    "fullPageScrollTimeout": 1500,
+    "hideScrollBars": true,
+    "isHybridApp": false,
     "savePerInstance": true,
+    "tabbableOptions": {
+      "circle": {
+        "backgroundColor": "rgba(255, 0, 0, 0.4)",
+        "borderColor": "rgba(255, 0, 0, 1)",
+        "borderWidth": 1,
+        "fontColor": "rgba(0, 0, 0, 1)",
+        "fontFamily": "Arial",
+        "fontSize": 10,
+        "size": 10,
+      },
+      "line": {
+        "color": "rgba(255, 0, 0, 1)",
+        "width": 1,
+      },
+    },
+    "toolBarShadowPadding": 6,
+    "userBasedFullPageScreenshot": false,
+    "waitForFontsLoaded": true,
   },
 }
 `;
@@ -581,9 +627,55 @@ exports[`saveWebElement > should call takeElementScreenshot with BiDi disabled w
   "isNativeContext": false,
   "tag": "test-element",
   "wicOptions": {
+    "addIOSBezelCorners": false,
+    "addressBarShadowPadding": 6,
     "alwaysSaveActualImage": true,
+    "autoElementScroll": true,
+    "autoSaveBaseline": false,
+    "clearFolder": false,
+    "compareOptions": {
+      "blockOutSideBar": false,
+      "blockOutStatusBar": false,
+      "blockOutToolBar": false,
+      "createJsonReportFiles": false,
+      "diffPixelBoundingBoxProximity": 5,
+      "ignoreAlpha": false,
+      "ignoreAntialiasing": false,
+      "ignoreColors": false,
+      "ignoreLess": false,
+      "ignoreNothing": false,
+      "rawMisMatchPercentage": false,
+      "returnAllCompareData": false,
+      "saveAboveTolerance": 0,
+      "scaleImagesToSameSize": false,
+    },
+    "disableBlinkingCursor": false,
+    "disableCSSAnimation": false,
+    "enableLayoutTesting": false,
+    "enableLegacyScreenshotMethod": false,
     "formatImageName": "{tag}-{logName}-{width}x{height}-dpr-{dpr}",
+    "fullPageScrollTimeout": 1500,
+    "hideScrollBars": true,
+    "isHybridApp": false,
     "savePerInstance": true,
+    "tabbableOptions": {
+      "circle": {
+        "backgroundColor": "rgba(255, 0, 0, 0.4)",
+        "borderColor": "rgba(255, 0, 0, 1)",
+        "borderWidth": 1,
+        "fontColor": "rgba(0, 0, 0, 1)",
+        "fontFamily": "Arial",
+        "fontSize": 10,
+        "size": 10,
+      },
+      "line": {
+        "color": "rgba(255, 0, 0, 1)",
+        "width": 1,
+      },
+    },
+    "toolBarShadowPadding": 6,
+    "userBasedFullPageScreenshot": false,
+    "waitForFontsLoaded": true,
   },
 }
 `;
@@ -969,9 +1061,55 @@ exports[`saveWebElement > should call takeElementScreenshot with correct options
   "isNativeContext": false,
   "tag": "test-element",
   "wicOptions": {
+    "addIOSBezelCorners": false,
+    "addressBarShadowPadding": 6,
     "alwaysSaveActualImage": true,
+    "autoElementScroll": true,
+    "autoSaveBaseline": false,
+    "clearFolder": false,
+    "compareOptions": {
+      "blockOutSideBar": false,
+      "blockOutStatusBar": false,
+      "blockOutToolBar": false,
+      "createJsonReportFiles": false,
+      "diffPixelBoundingBoxProximity": 5,
+      "ignoreAlpha": false,
+      "ignoreAntialiasing": false,
+      "ignoreColors": false,
+      "ignoreLess": false,
+      "ignoreNothing": false,
+      "rawMisMatchPercentage": false,
+      "returnAllCompareData": false,
+      "saveAboveTolerance": 0,
+      "scaleImagesToSameSize": false,
+    },
+    "disableBlinkingCursor": false,
+    "disableCSSAnimation": false,
+    "enableLayoutTesting": false,
+    "enableLegacyScreenshotMethod": false,
     "formatImageName": "{tag}-{logName}-{width}x{height}-dpr-{dpr}",
+    "fullPageScrollTimeout": 1500,
+    "hideScrollBars": true,
+    "isHybridApp": false,
     "savePerInstance": true,
+    "tabbableOptions": {
+      "circle": {
+        "backgroundColor": "rgba(255, 0, 0, 0.4)",
+        "borderColor": "rgba(255, 0, 0, 1)",
+        "borderWidth": 1,
+        "fontColor": "rgba(0, 0, 0, 1)",
+        "fontFamily": "Arial",
+        "fontSize": 10,
+        "size": 10,
+      },
+      "line": {
+        "color": "rgba(255, 0, 0, 1)",
+        "width": 1,
+      },
+    },
+    "toolBarShadowPadding": 6,
+    "userBasedFullPageScreenshot": false,
+    "waitForFontsLoaded": true,
   },
 }
 `;
@@ -1277,9 +1415,55 @@ exports[`saveWebElement > should handle NaN dimension values correctly 3`] = `
   "isNativeContext": false,
   "tag": "test-element",
   "wicOptions": {
+    "addIOSBezelCorners": false,
+    "addressBarShadowPadding": 6,
     "alwaysSaveActualImage": true,
+    "autoElementScroll": true,
+    "autoSaveBaseline": false,
+    "clearFolder": false,
+    "compareOptions": {
+      "blockOutSideBar": false,
+      "blockOutStatusBar": false,
+      "blockOutToolBar": false,
+      "createJsonReportFiles": false,
+      "diffPixelBoundingBoxProximity": 5,
+      "ignoreAlpha": false,
+      "ignoreAntialiasing": false,
+      "ignoreColors": false,
+      "ignoreLess": false,
+      "ignoreNothing": false,
+      "rawMisMatchPercentage": false,
+      "returnAllCompareData": false,
+      "saveAboveTolerance": 0,
+      "scaleImagesToSameSize": false,
+    },
+    "disableBlinkingCursor": false,
+    "disableCSSAnimation": false,
+    "enableLayoutTesting": false,
+    "enableLegacyScreenshotMethod": false,
     "formatImageName": "{tag}-{logName}-{width}x{height}-dpr-{dpr}",
+    "fullPageScrollTimeout": 1500,
+    "hideScrollBars": true,
+    "isHybridApp": false,
     "savePerInstance": true,
+    "tabbableOptions": {
+      "circle": {
+        "backgroundColor": "rgba(255, 0, 0, 0.4)",
+        "borderColor": "rgba(255, 0, 0, 1)",
+        "borderWidth": 1,
+        "fontColor": "rgba(0, 0, 0, 1)",
+        "fontFamily": "Arial",
+        "fontSize": 10,
+        "size": 10,
+      },
+      "line": {
+        "color": "rgba(255, 0, 0, 1)",
+        "width": 1,
+      },
+    },
+    "toolBarShadowPadding": 6,
+    "userBasedFullPageScreenshot": false,
+    "waitForFontsLoaded": true,
   },
 }
 `;