fix: respect alwaysSaveActualImage: false for checkScreen methods (#1115)

.changeset/fix-always-save-actual-image.md

@@ -0,0 +1,18 @@
+---
+"@wdio/image-comparison-core": patch
+"@wdio/visual-service": patch
+---
+
+## #1115 Respect `alwaysSaveActualImage: false` for `checkScreen` methods
+
+When using visual matchers like `toMatchScreenSnapshot('tag', 0.9)` with `alwaysSaveActualImage: false`, the actual image was still being saved even when the comparison passed within the threshold.
+
+The root cause was that the matcher's expected threshold was not being passed to the core comparison logic. The core used `saveAboveTolerance` (defaulting to 0) to decide whether to save images, while the matcher used the user-provided threshold to determine pass/fail - these were disconnected.
+
+This fix ensures:
+- When `alwaysSaveActualImage: false` and `saveAboveTolerance` is not explicitly set, actual images are never saved (respecting the literal meaning of the option)
+- When `saveAboveTolerance` is explicitly set (like matchers do internally), actual images are saved only when the mismatch exceeds that threshold
+
+# Committers: 1
+
+-   Wim Selles ([@wswebcreation](https://github.com/wswebcreation))

.changeset/fix-save-methods-always-save.md

@@ -0,0 +1,25 @@
+---
+"@wdio/image-comparison-core": patch
+"@wdio/visual-service": patch
+---
+
+## Fix: `save*` methods now always save files regardless of `alwaysSaveActualImage` setting
+
+Previously, when `alwaysSaveActualImage: false` was set in the configuration, `save*` methods (`saveScreen`, `saveElement`, `saveFullPageScreen`, `saveAppScreen`, `saveAppElement`) were not saving files to disk, causing test failures.
+
+The `alwaysSaveActualImage` option is intended to control whether actual images are saved during `check*` methods (comparison operations), not `save*` methods. Since `save*` methods are explicitly designed to save screenshots, they should always save files regardless of this setting.
+
+This fix ensures:
+- `save*` methods always save files to disk, even when `alwaysSaveActualImage: false` is set in the config
+- `alwaysSaveActualImage: false` continues to work correctly for `check*` methods (as intended for issue #1115)
+- The behavior is now consistent: `save*` = always save, `check*` = respect `alwaysSaveActualImage` setting
+
+**Implementation details:**
+- The visual service overrides `alwaysSaveActualImage: true` when calling `save*` methods directly from the browser API
+- `save*` methods respect whatever `alwaysSaveActualImage` value is passed to them (no special logic needed)
+- `check*` methods pass through the config value (which may be `false`), so `save*` methods respect it when called internally
+- This clean separation ensures `save*` methods work correctly when called directly while still respecting `alwaysSaveActualImage` for `check*` methods
+
+# Committers: 1
+
+-   Wim Selles ([@wswebcreation](https://github.com/wswebcreation))

package.json

@@ -19,7 +19,7 @@
     "test:unit": "vitest --coverage --run",
     "test:unit:ui": "vitest --coverage --ui",
     "test:unit:watch": "vitest --coverage --watch",
-    "test.local.init": "rimraf localBaseline && wdio ./tests/configs/wdio.local.init.conf.ts",
+    "test.local.init": "rimraf localBaseline && SAVE_ACTUAL=true wdio ./tests/configs/wdio.local.init.conf.ts",
     "test.local.desktop": "wdio tests/configs/wdio.local.desktop.conf.ts",
     "test.local.emus.app": "wdio tests/configs/wdio.local.android.emus.app.conf.ts",
     "test.local.emus.web": "wdio tests/configs/wdio.local.android.emus.web.conf.ts",

packages/image-comparison-core/src/__snapshots__/base.test.ts.snap

@@ -21,7 +21,6 @@ exports[`BaseClass > initializes default options correctly 1`] = `
     "ignoreNothing": false,
     "rawMisMatchPercentage": false,
     "returnAllCompareData": false,
-    "saveAboveTolerance": 0,
     "scaleImagesToSameSize": false,
   },
   "disableBlinkingCursor": false,

packages/image-comparison-core/src/commands/saveFullPageScreen.test.ts

@@ -318,4 +318,23 @@ describe('saveFullPageScreen', () => {
             fileName: 'test-fullpage.png'
         })
     })
+
+    it('should always save actual image even when alwaysSaveActualImage is false in config', async () => {
+        const options = {
+            ...baseOptions,
+            saveFullPageOptions: {
+                ...baseOptions.saveFullPageOptions,
+                wic: {
+                    ...baseOptions.saveFullPageOptions.wic,
+                    alwaysSaveActualImage: false, // Set to false in config
+                }
+            }
+        }
+
+        await saveFullPageScreen(options)
+
+        expect(buildAfterScreenshotOptionsSpy).toHaveBeenCalled()
+        const buildAfterScreenshotOptionsCall = buildAfterScreenshotOptionsSpy.mock.calls[buildAfterScreenshotOptionsSpy.mock.calls.length - 1]
+        expect(buildAfterScreenshotOptionsCall[0].wicOptions).toHaveProperty('alwaysSaveActualImage', true)
+    })
 })

packages/image-comparison-core/src/commands/saveFullPageScreen.ts

@@ -28,7 +28,6 @@ export default async function saveFullPageScreen(
     }
 
     // 2. Set some variables
-    const { formatImageName, savePerInstance } = saveFullPageOptions.wic
     const enableLegacyScreenshotMethod = getMethodOrWicOption(saveFullPageOptions.method, saveFullPageOptions.wic, 'enableLegacyScreenshotMethod')
     const fullPageScrollTimeout = getMethodOrWicOption(saveFullPageOptions.method, saveFullPageOptions.wic, 'fullPageScrollTimeout')
     const hideAfterFirstScroll: HTMLElement[] = saveFullPageOptions.method.hideAfterFirstScroll || []
@@ -88,9 +87,8 @@ export default async function saveFullPageScreen(
         instanceData,
         enrichedInstanceData,
         beforeOptions,
-        wicOptions: { formatImageName, savePerInstance, alwaysSaveActualImage: saveFullPageOptions.wic.alwaysSaveActualImage }
+        wicOptions: saveFullPageOptions.wic
     })
 
     return afterScreenshot(browserInstance, afterOptions!)
 }
-

packages/image-comparison-core/src/commands/saveWebElement.test.ts

@@ -277,4 +277,23 @@ describe('saveWebElement', () => {
         expect(buildAfterScreenshotOptionsSpy.mock.calls[0][0]).toMatchSnapshot()
         expect(afterScreenshotSpy.mock.calls[0][1]).toMatchSnapshot()
     })
+
+    it('should always save actual image even when alwaysSaveActualImage is false in config', async () => {
+        const options = {
+            ...baseOptions,
+            saveElementOptions: {
+                ...baseOptions.saveElementOptions,
+                wic: {
+                    ...baseOptions.saveElementOptions.wic,
+                    alwaysSaveActualImage: false, // Set to false in config
+                }
+            }
+        }
+
+        await saveWebElement(options)
+
+        expect(buildAfterScreenshotOptionsSpy).toHaveBeenCalled()
+        const buildAfterScreenshotOptionsCall = buildAfterScreenshotOptionsSpy.mock.calls[buildAfterScreenshotOptionsSpy.mock.calls.length - 1]
+        expect(buildAfterScreenshotOptionsCall[0].wicOptions).toHaveProperty('alwaysSaveActualImage', true)
+    })
 })

packages/image-comparison-core/src/commands/saveWebElement.ts

@@ -24,7 +24,7 @@ export default async function saveWebElement(
     }: InternalSaveElementMethodOptions
 ): Promise<ScreenshotOutput> {
     // 1. Set some variables
-    const { addressBarShadowPadding, autoElementScroll, formatImageName, savePerInstance } = saveElementOptions.wic
+    const { addressBarShadowPadding, autoElementScroll } = saveElementOptions.wic
     const enableLegacyScreenshotMethod = getMethodOrWicOption(saveElementOptions.method, saveElementOptions.wic, 'enableLegacyScreenshotMethod')
     const resizeDimensions: ResizeDimensions | number = saveElementOptions.method.resizeDimensions || DEFAULT_RESIZE_DIMENSIONS
 
@@ -81,7 +81,7 @@ export default async function saveWebElement(
         instanceData,
         enrichedInstanceData,
         beforeOptions,
-        wicOptions: { formatImageName, savePerInstance, alwaysSaveActualImage: saveElementOptions.wic.alwaysSaveActualImage }
+        wicOptions: saveElementOptions.wic
     })
 
     return afterScreenshot(browserInstance, afterOptions)

packages/image-comparison-core/src/commands/saveWebScreen.test.ts

@@ -248,4 +248,22 @@ describe('saveWebScreen', () => {
         expect(takeWebScreenshotSpy.mock.calls[0]).toMatchSnapshot()
         expect(afterScreenshotSpy.mock.calls[0]).toMatchSnapshot()
     })
+
+    it('should always save actual image even when alwaysSaveActualImage is false in config', async () => {
+        const options = createTestOptions(baseOptions, {
+            saveScreenOptions: {
+                ...baseOptions.saveScreenOptions,
+                wic: {
+                    ...baseOptions.saveScreenOptions.wic,
+                    alwaysSaveActualImage: false, // Set to false in config
+                }
+            }
+        })
+
+        await saveWebScreen(options)
+
+        expect(afterScreenshotSpy).toHaveBeenCalled()
+        const afterScreenshotCall = afterScreenshotSpy.mock.calls[afterScreenshotSpy.mock.calls.length - 1]
+        expect(afterScreenshotCall[1]).toHaveProperty('alwaysSaveActualImage', true)
+    })
 })

packages/image-comparison-core/src/commands/saveWebScreen.ts

@@ -22,7 +22,7 @@ export default async function saveWebScreen(
     }: InternalSaveScreenMethodOptions
 ): Promise<ScreenshotOutput> {
     // 1. Set some variables
-    const { addIOSBezelCorners, formatImageName, savePerInstance } = saveScreenOptions.wic
+    const { addIOSBezelCorners } = saveScreenOptions.wic
     const enableLegacyScreenshotMethod = getMethodOrWicOption(saveScreenOptions.method, saveScreenOptions.wic, 'enableLegacyScreenshotMethod')
 
     // 2.  Prepare the screenshot
@@ -76,7 +76,7 @@ export default async function saveWebScreen(
         instanceData,
         enrichedInstanceData,
         beforeOptions,
-        wicOptions: { formatImageName, savePerInstance, alwaysSaveActualImage: saveScreenOptions.wic.alwaysSaveActualImage }
+        wicOptions: saveScreenOptions.wic
     })
 
     return afterScreenshot(browserInstance, afterOptions)

packages/image-comparison-core/src/helpers/__snapshots__/options.test.ts.snap

@@ -97,7 +97,6 @@ exports[`options > defaultOptions > should return the default options when no op
     "ignoreNothing": false,
     "rawMisMatchPercentage": false,
     "returnAllCompareData": false,
-    "saveAboveTolerance": 0,
     "scaleImagesToSameSize": false,
   },
   "disableBlinkingCursor": false,

packages/image-comparison-core/src/helpers/constants.ts

@@ -16,7 +16,6 @@ export const DEFAULT_COMPARE_OPTIONS = {
     ignoreNothing: false,
     rawMisMatchPercentage: false,
     returnAllCompareData: false,
-    saveAboveTolerance: 0,
     scaleImagesToSameSize: false,
 }
 export const DEFAULT_FORMAT_STRING = '{tag}-{browserName}-{width}x{height}-dpr-{dpr}'

packages/image-comparison-core/src/helpers/options.interfaces.ts

@@ -461,8 +461,10 @@ export interface CompareOptions {
 
     /**
      * Mismatch percentage threshold above which the image with differences will be saved.
+     * When undefined, actual images won't be saved (respects alwaysSaveActualImage: false).
+     * Matchers set this value internally based on the expected threshold.
      */
-    saveAboveTolerance: number;
+    saveAboveTolerance?: number;
 
     /**
      * Scale images to the same size before comparing them.

packages/image-comparison-core/src/methods/images.executeImageCompare.test.ts

@@ -693,14 +693,60 @@ describe('executeImageCompare', () => {
         expect(fsPromises.writeFile).toHaveBeenCalledWith('/mock/actual/test.png', Buffer.from(base64Image, 'base64'))
     })
 
-    it('should save base64 actual on diff when not always saving', async () => {
+    it('should NOT save base64 actual on diff when alwaysSaveActualImage is false and saveAboveTolerance is not set (#1115)', async () => {
+        // When alwaysSaveActualImage: false and saveAboveTolerance is not explicitly set,
+        // actual images should never be saved - respecting the literal meaning of the option
         const base64Image = Buffer.from('base64-image').toString('base64')
         const optionsWithDiff = {
             ...mockOptions,
             folderOptions: {
                 ...mockOptions.folderOptions,
                 alwaysSaveActualImage: false,
-            }
+            },
+            compareOptions: {
+                ...mockOptions.compareOptions,
+                wic: {
+                    ...mockOptions.compareOptions.wic,
+                    saveAboveTolerance: undefined,
+                },
+            },
+        }
+        vi.mocked(compareImages.default).mockResolvedValue({
+            rawMisMatchPercentage: 0.5,
+            misMatchPercentage: 0.5,
+            getBuffer: vi.fn().mockResolvedValue(Buffer.from('diff-image-data')),
+            diffBounds: { left: 0, top: 0, right: 0, bottom: 0 },
+            analysisTime: 10,
+            diffPixels: []
+        })
+
+        await executeImageCompare({
+            isViewPortScreenshot: true,
+            isNativeContext: false,
+            options: optionsWithDiff,
+            testContext: mockTestContext,
+            actualBase64Image: base64Image,
+        })
+
+        expect(fsPromises.writeFile).not.toHaveBeenCalled()
+    })
+
+    it('should save base64 actual on diff when saveAboveTolerance is explicitly set to 0', async () => {
+        // When saveAboveTolerance is explicitly set (even to 0), save actual images when diff exceeds it
+        const base64Image = Buffer.from('base64-image').toString('base64')
+        const optionsWithDiff = {
+            ...mockOptions,
+            folderOptions: {
+                ...mockOptions.folderOptions,
+                alwaysSaveActualImage: false,
+            },
+            compareOptions: {
+                ...mockOptions.compareOptions,
+                wic: {
+                    ...mockOptions.compareOptions.wic,
+                    saveAboveTolerance: 0,
+                },
+            },
         }
         vi.mocked(compareImages.default).mockResolvedValue({
             rawMisMatchPercentage: 0.5,

packages/image-comparison-core/src/methods/images.ts

@@ -450,10 +450,15 @@ export async function executeImageCompare(
     )
 
     // 6a. Save actual image on failure if alwaysSaveActualImage is false
-    const saveAboveTolerance = imageCompareOptions.saveAboveTolerance ?? 0
-    const hasFailure = rawMisMatchPercentage > saveAboveTolerance
-    if (useBase64Image && hasFailure && actualBase64Image) {
-        // Save the actual image only when comparison fails
+    // Only save when saveAboveTolerance is explicitly set (e.g., by matchers).
+    // When using checkScreen directly without saveAboveTolerance, respect
+    // alwaysSaveActualImage: false literally and don't save actual images.
+    // @see https://github.com/webdriverio/visual-testing/issues/1115
+    const saveAboveTolerance = imageCompareOptions.saveAboveTolerance
+    const shouldSaveOnFailure = saveAboveTolerance !== undefined
+    const hasFailure = rawMisMatchPercentage > (saveAboveTolerance ?? 0)
+    if (useBase64Image && shouldSaveOnFailure && hasFailure && actualBase64Image) {
+        // Save the actual image only when comparison fails and threshold was explicitly set
         await saveBase64Image(actualBase64Image, actualFilePath)
     }
 

packages/visual-service/src/service.ts

@@ -229,6 +229,12 @@ export default class WdioImageComparisonService extends BaseClass {
                         }
                         const isCurrentContextNative = self.contextManager.isNativeContext
 
+                        // save* methods should always save files, regardless of alwaysSaveActualImage config
+                        const isSaveCommand = commandName === 'saveElement'
+                        const wicOptions = isSaveCommand
+                            ? { ...self.defaultOptions, alwaysSaveActualImage: true }
+                            : self.defaultOptions
+
                         return [{
                             browserInstance,
                             element,
@@ -237,7 +243,7 @@ export default class WdioImageComparisonService extends BaseClass {
                             isNativeContext: isCurrentContextNative,
                             tag,
                             [elementOptionsKey]: {
-                                wic: self.defaultOptions,
+                                wic: wicOptions,
                                 method: elementOptions,
                             },
                             testContext: enrichTestContext({
@@ -294,14 +300,20 @@ export default class WdioImageComparisonService extends BaseClass {
                         }
                         const isCurrentContextNative = self.contextManager.isNativeContext
 
+                        // save* methods should always save files, regardless of alwaysSaveActualImage config
+                        const isSaveCommand = commandName === 'saveScreen' || commandName === 'saveFullPageScreen' || commandName === 'saveTabbablePage'
+                        const wicOptions = isSaveCommand
+                            ? { ...self.defaultOptions, alwaysSaveActualImage: true }
+                            : self.defaultOptions
+
                         return [{
                             browserInstance,
                             folders: getFolders(pageOptions, self.folders, self.#getBaselineFolder()),
                             instanceData: updatedInstanceData,
                             isNativeContext: isCurrentContextNative,
                             tag,
                             [pageOptionsKey]: {
-                                wic: self.defaultOptions,
+                                wic: wicOptions,
                                 method: pageOptions,
                             },
                             testContext: enrichTestContext({

tests/configs/wdio.lambdatest.shared.conf.ts

@@ -46,6 +46,7 @@ export const config: WebdriverIO.Config  = {
                 rawMisMatchPercentage: !!process.env.RAW_MISMATCH || false,
                 enableLayoutTesting: true,
                 ignoreAntialiasing: true,
+                alwaysSaveActualImage: false,
             } satisfies VisualServiceOptions,
         ],
     ],

tests/configs/wdio.local.desktop.conf.ts

@@ -72,7 +72,7 @@ export const config: WebdriverIO.Config = {
                 createJsonReportFiles: true,
                 clearRuntimeFolder: true,
                 enableLayoutTesting: true,
-                alwaysSaveActualImage: false,
+                alwaysSaveActualImage: process.env.SAVE_ACTUAL === 'true',
             },
         ]
     ],

tests/specs/desktop.spec.ts

@@ -1,4 +1,6 @@
+import type { ImageCompareResult } from '@wdio/image-comparison-core'
 import { browser, expect } from '@wdio/globals'
+import { fileExists } from '../helpers/fileExists.ts'
 
 describe('@wdio/visual-service desktop', () => {
     // @TODO
@@ -39,4 +41,23 @@ describe('@wdio/visual-service desktop', () => {
             ],
         })
     })
+
+    it(`should not store an actual image for '${browserName}' when the diff is below the threshold (#1115)`, async function () {
+        const tag = 'noActualStoredOnDiff'
+
+        await browser.execute(() => {
+            const el = document.createElement('div')
+            el.id = 'test-diff-element'
+            el.style.cssText = 'position:fixed;top:10px;left:10px;width:500px;height:500px;background:red;z-index:9999;'
+            document.body.appendChild(el)
+        })
+
+        const result = await browser.checkScreen(tag, {
+            returnAllCompareData: true,
+        }) as ImageCompareResult
+
+        expect(result.misMatchPercentage).toBeGreaterThan(0)
+        expect(result.misMatchPercentage).toBeLessThanOrEqual(70)
+        expect(fileExists(result.folders.actual)).toBe(false)
+    })
 })