diff --git a/.changeset/fix-always-save-actual-image.md b/.changeset/fix-always-save-actual-image.md new file mode 100644 index 00000000..b1c1f74a --- /dev/null +++ b/.changeset/fix-always-save-actual-image.md @@ -0,0 +1,18 @@ +--- +"@wdio/image-comparison-core": patch +"@wdio/visual-service": patch +--- + +## #1115 Respect `alwaysSaveActualImage: false` for `checkScreen` methods + +When using visual matchers like `toMatchScreenSnapshot('tag', 0.9)` with `alwaysSaveActualImage: false`, the actual image was still being saved even when the comparison passed within the threshold. + +The root cause was that the matcher's expected threshold was not being passed to the core comparison logic. The core used `saveAboveTolerance` (defaulting to 0) to decide whether to save images, while the matcher used the user-provided threshold to determine pass/fail - these were disconnected. + +This fix ensures: +- When `alwaysSaveActualImage: false` and `saveAboveTolerance` is not explicitly set, actual images are never saved (respecting the literal meaning of the option) +- When `saveAboveTolerance` is explicitly set (like matchers do internally), actual images are saved only when the mismatch exceeds that threshold + +# Committers: 1 + +- Wim Selles ([@wswebcreation](https://github.com/wswebcreation)) diff --git a/.changeset/fix-save-methods-always-save.md b/.changeset/fix-save-methods-always-save.md new file mode 100644 index 00000000..f10914d9 --- /dev/null +++ b/.changeset/fix-save-methods-always-save.md @@ -0,0 +1,25 @@ +--- +"@wdio/image-comparison-core": patch +"@wdio/visual-service": patch +--- + +## Fix: `save*` methods now always save files regardless of `alwaysSaveActualImage` setting + +Previously, when `alwaysSaveActualImage: false` was set in the configuration, `save*` methods (`saveScreen`, `saveElement`, `saveFullPageScreen`, `saveAppScreen`, `saveAppElement`) were not saving files to disk, causing test failures. + +The `alwaysSaveActualImage` option is intended to control whether actual images are saved during `check*` methods (comparison operations), not `save*` methods. Since `save*` methods are explicitly designed to save screenshots, they should always save files regardless of this setting. + +This fix ensures: +- `save*` methods always save files to disk, even when `alwaysSaveActualImage: false` is set in the config +- `alwaysSaveActualImage: false` continues to work correctly for `check*` methods (as intended for issue #1115) +- The behavior is now consistent: `save*` = always save, `check*` = respect `alwaysSaveActualImage` setting + +**Implementation details:** +- The visual service overrides `alwaysSaveActualImage: true` when calling `save*` methods directly from the browser API +- `save*` methods respect whatever `alwaysSaveActualImage` value is passed to them (no special logic needed) +- `check*` methods pass through the config value (which may be `false`), so `save*` methods respect it when called internally +- This clean separation ensures `save*` methods work correctly when called directly while still respecting `alwaysSaveActualImage` for `check*` methods + +# Committers: 1 + +- Wim Selles ([@wswebcreation](https://github.com/wswebcreation)) diff --git a/package.json b/package.json index 8dfee2a5..628e9c92 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test:unit": "vitest --coverage --run", "test:unit:ui": "vitest --coverage --ui", "test:unit:watch": "vitest --coverage --watch", - "test.local.init": "rimraf localBaseline && wdio ./tests/configs/wdio.local.init.conf.ts", + "test.local.init": "rimraf localBaseline && SAVE_ACTUAL=true wdio ./tests/configs/wdio.local.init.conf.ts", "test.local.desktop": "wdio tests/configs/wdio.local.desktop.conf.ts", "test.local.emus.app": "wdio tests/configs/wdio.local.android.emus.app.conf.ts", "test.local.emus.web": "wdio tests/configs/wdio.local.android.emus.web.conf.ts", diff --git a/packages/image-comparison-core/src/__snapshots__/base.test.ts.snap b/packages/image-comparison-core/src/__snapshots__/base.test.ts.snap index 8352d88e..30003706 100644 --- a/packages/image-comparison-core/src/__snapshots__/base.test.ts.snap +++ b/packages/image-comparison-core/src/__snapshots__/base.test.ts.snap @@ -21,7 +21,6 @@ exports[`BaseClass > initializes default options correctly 1`] = ` "ignoreNothing": false, "rawMisMatchPercentage": false, "returnAllCompareData": false, - "saveAboveTolerance": 0, "scaleImagesToSameSize": false, }, "disableBlinkingCursor": false, diff --git a/packages/image-comparison-core/src/commands/__snapshots__/saveFullPageScreen.test.ts.snap b/packages/image-comparison-core/src/commands/__snapshots__/saveFullPageScreen.test.ts.snap index dd853121..e16b13b4 100644 --- a/packages/image-comparison-core/src/commands/__snapshots__/saveFullPageScreen.test.ts.snap +++ b/packages/image-comparison-core/src/commands/__snapshots__/saveFullPageScreen.test.ts.snap @@ -134,9 +134,55 @@ exports[`saveFullPageScreen > should handle missing dimension values with NaN fa "isNativeContext": false, "tag": "test-fullpage", "wicOptions": { + "addIOSBezelCorners": false, + "addressBarShadowPadding": 6, "alwaysSaveActualImage": true, + "autoElementScroll": true, + "autoSaveBaseline": false, + "clearFolder": false, + "compareOptions": { + "blockOutSideBar": false, + "blockOutStatusBar": false, + "blockOutToolBar": false, + "createJsonReportFiles": false, + "diffPixelBoundingBoxProximity": 5, + "ignoreAlpha": false, + "ignoreAntialiasing": false, + "ignoreColors": false, + "ignoreLess": false, + "ignoreNothing": false, + "rawMisMatchPercentage": false, + "returnAllCompareData": false, + "saveAboveTolerance": 0, + "scaleImagesToSameSize": false, + }, + "disableBlinkingCursor": false, + "disableCSSAnimation": false, + "enableLayoutTesting": false, + "enableLegacyScreenshotMethod": false, "formatImageName": "{tag}-{logName}-{width}x{height}-dpr-{dpr}", + "fullPageScrollTimeout": 1500, + "hideScrollBars": true, + "isHybridApp": false, "savePerInstance": true, + "tabbableOptions": { + "circle": { + "backgroundColor": "rgba(255, 0, 0, 0.4)", + "borderColor": "rgba(255, 0, 0, 1)", + "borderWidth": 1, + "fontColor": "rgba(0, 0, 0, 1)", + "fontFamily": "Arial", + "fontSize": 10, + "size": 10, + }, + "line": { + "color": "rgba(255, 0, 0, 1)", + "width": 1, + }, + }, + "toolBarShadowPadding": 6, + "userBasedFullPageScreenshot": false, + "waitForFontsLoaded": true, }, } `; @@ -557,9 +603,55 @@ exports[`saveFullPageScreen > should take full page screenshots and return resul "isNativeContext": false, "tag": "test-fullpage", "wicOptions": { + "addIOSBezelCorners": false, + "addressBarShadowPadding": 6, "alwaysSaveActualImage": true, + "autoElementScroll": true, + "autoSaveBaseline": false, + "clearFolder": false, + "compareOptions": { + "blockOutSideBar": false, + "blockOutStatusBar": false, + "blockOutToolBar": false, + "createJsonReportFiles": false, + "diffPixelBoundingBoxProximity": 5, + "ignoreAlpha": false, + "ignoreAntialiasing": false, + "ignoreColors": false, + "ignoreLess": false, + "ignoreNothing": false, + "rawMisMatchPercentage": false, + "returnAllCompareData": false, + "saveAboveTolerance": 0, + "scaleImagesToSameSize": false, + }, + "disableBlinkingCursor": false, + "disableCSSAnimation": false, + "enableLayoutTesting": false, + "enableLegacyScreenshotMethod": false, "formatImageName": "{tag}-{logName}-{width}x{height}-dpr-{dpr}", + "fullPageScrollTimeout": 1500, + "hideScrollBars": true, + "isHybridApp": false, "savePerInstance": true, + "tabbableOptions": { + "circle": { + "backgroundColor": "rgba(255, 0, 0, 0.4)", + "borderColor": "rgba(255, 0, 0, 1)", + "borderWidth": 1, + "fontColor": "rgba(0, 0, 0, 1)", + "fontFamily": "Arial", + "fontSize": 10, + "size": 10, + }, + "line": { + "color": "rgba(255, 0, 0, 1)", + "width": 1, + }, + }, + "toolBarShadowPadding": 6, + "userBasedFullPageScreenshot": false, + "waitForFontsLoaded": true, }, } `; diff --git a/packages/image-comparison-core/src/commands/__snapshots__/saveWebElement.test.ts.snap b/packages/image-comparison-core/src/commands/__snapshots__/saveWebElement.test.ts.snap index c7b08dcc..fe831462 100644 --- a/packages/image-comparison-core/src/commands/__snapshots__/saveWebElement.test.ts.snap +++ b/packages/image-comparison-core/src/commands/__snapshots__/saveWebElement.test.ts.snap @@ -363,9 +363,55 @@ exports[`saveWebElement > should call takeElementScreenshot with BiDi disabled w "isNativeContext": false, "tag": "test-element", "wicOptions": { + "addIOSBezelCorners": false, + "addressBarShadowPadding": 6, "alwaysSaveActualImage": true, + "autoElementScroll": true, + "autoSaveBaseline": false, + "clearFolder": false, + "compareOptions": { + "blockOutSideBar": false, + "blockOutStatusBar": false, + "blockOutToolBar": false, + "createJsonReportFiles": false, + "diffPixelBoundingBoxProximity": 5, + "ignoreAlpha": false, + "ignoreAntialiasing": false, + "ignoreColors": false, + "ignoreLess": false, + "ignoreNothing": false, + "rawMisMatchPercentage": false, + "returnAllCompareData": false, + "saveAboveTolerance": 0, + "scaleImagesToSameSize": false, + }, + "disableBlinkingCursor": false, + "disableCSSAnimation": false, + "enableLayoutTesting": false, + "enableLegacyScreenshotMethod": false, "formatImageName": "{tag}-{logName}-{width}x{height}-dpr-{dpr}", + "fullPageScrollTimeout": 1500, + "hideScrollBars": true, + "isHybridApp": false, "savePerInstance": true, + "tabbableOptions": { + "circle": { + "backgroundColor": "rgba(255, 0, 0, 0.4)", + "borderColor": "rgba(255, 0, 0, 1)", + "borderWidth": 1, + "fontColor": "rgba(0, 0, 0, 1)", + "fontFamily": "Arial", + "fontSize": 10, + "size": 10, + }, + "line": { + "color": "rgba(255, 0, 0, 1)", + "width": 1, + }, + }, + "toolBarShadowPadding": 6, + "userBasedFullPageScreenshot": false, + "waitForFontsLoaded": true, }, } `; @@ -581,9 +627,55 @@ exports[`saveWebElement > should call takeElementScreenshot with BiDi disabled w "isNativeContext": false, "tag": "test-element", "wicOptions": { + "addIOSBezelCorners": false, + "addressBarShadowPadding": 6, "alwaysSaveActualImage": true, + "autoElementScroll": true, + "autoSaveBaseline": false, + "clearFolder": false, + "compareOptions": { + "blockOutSideBar": false, + "blockOutStatusBar": false, + "blockOutToolBar": false, + "createJsonReportFiles": false, + "diffPixelBoundingBoxProximity": 5, + "ignoreAlpha": false, + "ignoreAntialiasing": false, + "ignoreColors": false, + "ignoreLess": false, + "ignoreNothing": false, + "rawMisMatchPercentage": false, + "returnAllCompareData": false, + "saveAboveTolerance": 0, + "scaleImagesToSameSize": false, + }, + "disableBlinkingCursor": false, + "disableCSSAnimation": false, + "enableLayoutTesting": false, + "enableLegacyScreenshotMethod": false, "formatImageName": "{tag}-{logName}-{width}x{height}-dpr-{dpr}", + "fullPageScrollTimeout": 1500, + "hideScrollBars": true, + "isHybridApp": false, "savePerInstance": true, + "tabbableOptions": { + "circle": { + "backgroundColor": "rgba(255, 0, 0, 0.4)", + "borderColor": "rgba(255, 0, 0, 1)", + "borderWidth": 1, + "fontColor": "rgba(0, 0, 0, 1)", + "fontFamily": "Arial", + "fontSize": 10, + "size": 10, + }, + "line": { + "color": "rgba(255, 0, 0, 1)", + "width": 1, + }, + }, + "toolBarShadowPadding": 6, + "userBasedFullPageScreenshot": false, + "waitForFontsLoaded": true, }, } `; @@ -969,9 +1061,55 @@ exports[`saveWebElement > should call takeElementScreenshot with correct options "isNativeContext": false, "tag": "test-element", "wicOptions": { + "addIOSBezelCorners": false, + "addressBarShadowPadding": 6, "alwaysSaveActualImage": true, + "autoElementScroll": true, + "autoSaveBaseline": false, + "clearFolder": false, + "compareOptions": { + "blockOutSideBar": false, + "blockOutStatusBar": false, + "blockOutToolBar": false, + "createJsonReportFiles": false, + "diffPixelBoundingBoxProximity": 5, + "ignoreAlpha": false, + "ignoreAntialiasing": false, + "ignoreColors": false, + "ignoreLess": false, + "ignoreNothing": false, + "rawMisMatchPercentage": false, + "returnAllCompareData": false, + "saveAboveTolerance": 0, + "scaleImagesToSameSize": false, + }, + "disableBlinkingCursor": false, + "disableCSSAnimation": false, + "enableLayoutTesting": false, + "enableLegacyScreenshotMethod": false, "formatImageName": "{tag}-{logName}-{width}x{height}-dpr-{dpr}", + "fullPageScrollTimeout": 1500, + "hideScrollBars": true, + "isHybridApp": false, "savePerInstance": true, + "tabbableOptions": { + "circle": { + "backgroundColor": "rgba(255, 0, 0, 0.4)", + "borderColor": "rgba(255, 0, 0, 1)", + "borderWidth": 1, + "fontColor": "rgba(0, 0, 0, 1)", + "fontFamily": "Arial", + "fontSize": 10, + "size": 10, + }, + "line": { + "color": "rgba(255, 0, 0, 1)", + "width": 1, + }, + }, + "toolBarShadowPadding": 6, + "userBasedFullPageScreenshot": false, + "waitForFontsLoaded": true, }, } `; @@ -1277,9 +1415,55 @@ exports[`saveWebElement > should handle NaN dimension values correctly 3`] = ` "isNativeContext": false, "tag": "test-element", "wicOptions": { + "addIOSBezelCorners": false, + "addressBarShadowPadding": 6, "alwaysSaveActualImage": true, + "autoElementScroll": true, + "autoSaveBaseline": false, + "clearFolder": false, + "compareOptions": { + "blockOutSideBar": false, + "blockOutStatusBar": false, + "blockOutToolBar": false, + "createJsonReportFiles": false, + "diffPixelBoundingBoxProximity": 5, + "ignoreAlpha": false, + "ignoreAntialiasing": false, + "ignoreColors": false, + "ignoreLess": false, + "ignoreNothing": false, + "rawMisMatchPercentage": false, + "returnAllCompareData": false, + "saveAboveTolerance": 0, + "scaleImagesToSameSize": false, + }, + "disableBlinkingCursor": false, + "disableCSSAnimation": false, + "enableLayoutTesting": false, + "enableLegacyScreenshotMethod": false, "formatImageName": "{tag}-{logName}-{width}x{height}-dpr-{dpr}", + "fullPageScrollTimeout": 1500, + "hideScrollBars": true, + "isHybridApp": false, "savePerInstance": true, + "tabbableOptions": { + "circle": { + "backgroundColor": "rgba(255, 0, 0, 0.4)", + "borderColor": "rgba(255, 0, 0, 1)", + "borderWidth": 1, + "fontColor": "rgba(0, 0, 0, 1)", + "fontFamily": "Arial", + "fontSize": 10, + "size": 10, + }, + "line": { + "color": "rgba(255, 0, 0, 1)", + "width": 1, + }, + }, + "toolBarShadowPadding": 6, + "userBasedFullPageScreenshot": false, + "waitForFontsLoaded": true, }, } `; diff --git a/packages/image-comparison-core/src/commands/saveFullPageScreen.test.ts b/packages/image-comparison-core/src/commands/saveFullPageScreen.test.ts index 09952f19..06843289 100644 --- a/packages/image-comparison-core/src/commands/saveFullPageScreen.test.ts +++ b/packages/image-comparison-core/src/commands/saveFullPageScreen.test.ts @@ -318,4 +318,23 @@ describe('saveFullPageScreen', () => { fileName: 'test-fullpage.png' }) }) + + it('should pass through alwaysSaveActualImage value from options (service overrides for direct save* calls)', async () => { + const options = { + ...baseOptions, + saveFullPageOptions: { + ...baseOptions.saveFullPageOptions, + wic: { + ...baseOptions.saveFullPageOptions.wic, + alwaysSaveActualImage: false, + } + } + } + + await saveFullPageScreen(options) + + expect(buildAfterScreenshotOptionsSpy).toHaveBeenCalled() + const buildAfterScreenshotOptionsCall = buildAfterScreenshotOptionsSpy.mock.calls[buildAfterScreenshotOptionsSpy.mock.calls.length - 1] + expect(buildAfterScreenshotOptionsCall[0].wicOptions).toHaveProperty('alwaysSaveActualImage', false) + }) }) diff --git a/packages/image-comparison-core/src/commands/saveFullPageScreen.ts b/packages/image-comparison-core/src/commands/saveFullPageScreen.ts index 22ea8a0d..f4fe8b39 100644 --- a/packages/image-comparison-core/src/commands/saveFullPageScreen.ts +++ b/packages/image-comparison-core/src/commands/saveFullPageScreen.ts @@ -28,7 +28,6 @@ export default async function saveFullPageScreen( } // 2. Set some variables - const { formatImageName, savePerInstance } = saveFullPageOptions.wic const enableLegacyScreenshotMethod = getMethodOrWicOption(saveFullPageOptions.method, saveFullPageOptions.wic, 'enableLegacyScreenshotMethod') const fullPageScrollTimeout = getMethodOrWicOption(saveFullPageOptions.method, saveFullPageOptions.wic, 'fullPageScrollTimeout') const hideAfterFirstScroll: HTMLElement[] = saveFullPageOptions.method.hideAfterFirstScroll || [] @@ -88,9 +87,8 @@ export default async function saveFullPageScreen( instanceData, enrichedInstanceData, beforeOptions, - wicOptions: { formatImageName, savePerInstance, alwaysSaveActualImage: saveFullPageOptions.wic.alwaysSaveActualImage } + wicOptions: saveFullPageOptions.wic }) return afterScreenshot(browserInstance, afterOptions!) } - diff --git a/packages/image-comparison-core/src/commands/saveWebElement.test.ts b/packages/image-comparison-core/src/commands/saveWebElement.test.ts index b69b774b..61409218 100644 --- a/packages/image-comparison-core/src/commands/saveWebElement.test.ts +++ b/packages/image-comparison-core/src/commands/saveWebElement.test.ts @@ -277,4 +277,23 @@ describe('saveWebElement', () => { expect(buildAfterScreenshotOptionsSpy.mock.calls[0][0]).toMatchSnapshot() expect(afterScreenshotSpy.mock.calls[0][1]).toMatchSnapshot() }) + + it('should pass through alwaysSaveActualImage value from options (service overrides for direct save* calls)', async () => { + const options = { + ...baseOptions, + saveElementOptions: { + ...baseOptions.saveElementOptions, + wic: { + ...baseOptions.saveElementOptions.wic, + alwaysSaveActualImage: false, + } + } + } + + await saveWebElement(options) + + expect(buildAfterScreenshotOptionsSpy).toHaveBeenCalled() + const buildAfterScreenshotOptionsCall = buildAfterScreenshotOptionsSpy.mock.calls[buildAfterScreenshotOptionsSpy.mock.calls.length - 1] + expect(buildAfterScreenshotOptionsCall[0].wicOptions).toHaveProperty('alwaysSaveActualImage', false) + }) }) diff --git a/packages/image-comparison-core/src/commands/saveWebElement.ts b/packages/image-comparison-core/src/commands/saveWebElement.ts index fe9922da..150a1473 100644 --- a/packages/image-comparison-core/src/commands/saveWebElement.ts +++ b/packages/image-comparison-core/src/commands/saveWebElement.ts @@ -24,7 +24,7 @@ export default async function saveWebElement( }: InternalSaveElementMethodOptions ): Promise { // 1. Set some variables - const { addressBarShadowPadding, autoElementScroll, formatImageName, savePerInstance } = saveElementOptions.wic + const { addressBarShadowPadding, autoElementScroll } = saveElementOptions.wic const enableLegacyScreenshotMethod = getMethodOrWicOption(saveElementOptions.method, saveElementOptions.wic, 'enableLegacyScreenshotMethod') const resizeDimensions: ResizeDimensions | number = saveElementOptions.method.resizeDimensions || DEFAULT_RESIZE_DIMENSIONS @@ -81,7 +81,7 @@ export default async function saveWebElement( instanceData, enrichedInstanceData, beforeOptions, - wicOptions: { formatImageName, savePerInstance, alwaysSaveActualImage: saveElementOptions.wic.alwaysSaveActualImage } + wicOptions: saveElementOptions.wic }) return afterScreenshot(browserInstance, afterOptions) diff --git a/packages/image-comparison-core/src/commands/saveWebScreen.test.ts b/packages/image-comparison-core/src/commands/saveWebScreen.test.ts index 4c914a10..04e65f2c 100644 --- a/packages/image-comparison-core/src/commands/saveWebScreen.test.ts +++ b/packages/image-comparison-core/src/commands/saveWebScreen.test.ts @@ -248,4 +248,22 @@ describe('saveWebScreen', () => { expect(takeWebScreenshotSpy.mock.calls[0]).toMatchSnapshot() expect(afterScreenshotSpy.mock.calls[0]).toMatchSnapshot() }) + + it('should pass through alwaysSaveActualImage value from options (service overrides for direct save* calls)', async () => { + const options = createTestOptions(baseOptions, { + saveScreenOptions: { + ...baseOptions.saveScreenOptions, + wic: { + ...baseOptions.saveScreenOptions.wic, + alwaysSaveActualImage: false, + } + } + }) + + await saveWebScreen(options) + + expect(afterScreenshotSpy).toHaveBeenCalled() + const afterScreenshotCall = afterScreenshotSpy.mock.calls[afterScreenshotSpy.mock.calls.length - 1] + expect(afterScreenshotCall[1]).toHaveProperty('alwaysSaveActualImage', false) + }) }) diff --git a/packages/image-comparison-core/src/commands/saveWebScreen.ts b/packages/image-comparison-core/src/commands/saveWebScreen.ts index 80b7bee0..eb41ae72 100644 --- a/packages/image-comparison-core/src/commands/saveWebScreen.ts +++ b/packages/image-comparison-core/src/commands/saveWebScreen.ts @@ -22,7 +22,7 @@ export default async function saveWebScreen( }: InternalSaveScreenMethodOptions ): Promise { // 1. Set some variables - const { addIOSBezelCorners, formatImageName, savePerInstance } = saveScreenOptions.wic + const { addIOSBezelCorners } = saveScreenOptions.wic const enableLegacyScreenshotMethod = getMethodOrWicOption(saveScreenOptions.method, saveScreenOptions.wic, 'enableLegacyScreenshotMethod') // 2. Prepare the screenshot @@ -76,7 +76,7 @@ export default async function saveWebScreen( instanceData, enrichedInstanceData, beforeOptions, - wicOptions: { formatImageName, savePerInstance, alwaysSaveActualImage: saveScreenOptions.wic.alwaysSaveActualImage } + wicOptions: saveScreenOptions.wic }) return afterScreenshot(browserInstance, afterOptions) diff --git a/packages/image-comparison-core/src/helpers/__snapshots__/options.test.ts.snap b/packages/image-comparison-core/src/helpers/__snapshots__/options.test.ts.snap index 77051b1b..9dd979fc 100644 --- a/packages/image-comparison-core/src/helpers/__snapshots__/options.test.ts.snap +++ b/packages/image-comparison-core/src/helpers/__snapshots__/options.test.ts.snap @@ -97,7 +97,6 @@ exports[`options > defaultOptions > should return the default options when no op "ignoreNothing": false, "rawMisMatchPercentage": false, "returnAllCompareData": false, - "saveAboveTolerance": 0, "scaleImagesToSameSize": false, }, "disableBlinkingCursor": false, diff --git a/packages/image-comparison-core/src/helpers/constants.ts b/packages/image-comparison-core/src/helpers/constants.ts index f73c5def..5776b1db 100644 --- a/packages/image-comparison-core/src/helpers/constants.ts +++ b/packages/image-comparison-core/src/helpers/constants.ts @@ -16,7 +16,6 @@ export const DEFAULT_COMPARE_OPTIONS = { ignoreNothing: false, rawMisMatchPercentage: false, returnAllCompareData: false, - saveAboveTolerance: 0, scaleImagesToSameSize: false, } export const DEFAULT_FORMAT_STRING = '{tag}-{browserName}-{width}x{height}-dpr-{dpr}' diff --git a/packages/image-comparison-core/src/helpers/options.interfaces.ts b/packages/image-comparison-core/src/helpers/options.interfaces.ts index aadea678..19b1211b 100644 --- a/packages/image-comparison-core/src/helpers/options.interfaces.ts +++ b/packages/image-comparison-core/src/helpers/options.interfaces.ts @@ -461,8 +461,10 @@ export interface CompareOptions { /** * Mismatch percentage threshold above which the image with differences will be saved. + * When undefined, actual images won't be saved (respects alwaysSaveActualImage: false). + * Matchers set this value internally based on the expected threshold. */ - saveAboveTolerance: number; + saveAboveTolerance?: number; /** * Scale images to the same size before comparing them. diff --git a/packages/image-comparison-core/src/methods/images.executeImageCompare.test.ts b/packages/image-comparison-core/src/methods/images.executeImageCompare.test.ts index 138b5397..fa16e368 100644 --- a/packages/image-comparison-core/src/methods/images.executeImageCompare.test.ts +++ b/packages/image-comparison-core/src/methods/images.executeImageCompare.test.ts @@ -693,14 +693,60 @@ describe('executeImageCompare', () => { expect(fsPromises.writeFile).toHaveBeenCalledWith('/mock/actual/test.png', Buffer.from(base64Image, 'base64')) }) - it('should save base64 actual on diff when not always saving', async () => { + it('should NOT save base64 actual on diff when alwaysSaveActualImage is false and saveAboveTolerance is not set (#1115)', async () => { + // When alwaysSaveActualImage: false and saveAboveTolerance is not explicitly set, + // actual images should never be saved - respecting the literal meaning of the option const base64Image = Buffer.from('base64-image').toString('base64') const optionsWithDiff = { ...mockOptions, folderOptions: { ...mockOptions.folderOptions, alwaysSaveActualImage: false, - } + }, + compareOptions: { + ...mockOptions.compareOptions, + wic: { + ...mockOptions.compareOptions.wic, + saveAboveTolerance: undefined, + }, + }, + } + vi.mocked(compareImages.default).mockResolvedValue({ + rawMisMatchPercentage: 0.5, + misMatchPercentage: 0.5, + getBuffer: vi.fn().mockResolvedValue(Buffer.from('diff-image-data')), + diffBounds: { left: 0, top: 0, right: 0, bottom: 0 }, + analysisTime: 10, + diffPixels: [] + }) + + await executeImageCompare({ + isViewPortScreenshot: true, + isNativeContext: false, + options: optionsWithDiff, + testContext: mockTestContext, + actualBase64Image: base64Image, + }) + + expect(fsPromises.writeFile).not.toHaveBeenCalled() + }) + + it('should save base64 actual on diff when saveAboveTolerance is explicitly set to 0', async () => { + // When saveAboveTolerance is explicitly set (even to 0), save actual images when diff exceeds it + const base64Image = Buffer.from('base64-image').toString('base64') + const optionsWithDiff = { + ...mockOptions, + folderOptions: { + ...mockOptions.folderOptions, + alwaysSaveActualImage: false, + }, + compareOptions: { + ...mockOptions.compareOptions, + wic: { + ...mockOptions.compareOptions.wic, + saveAboveTolerance: 0, + }, + }, } vi.mocked(compareImages.default).mockResolvedValue({ rawMisMatchPercentage: 0.5, diff --git a/packages/image-comparison-core/src/methods/images.ts b/packages/image-comparison-core/src/methods/images.ts index 4f196d9e..d5635ed7 100644 --- a/packages/image-comparison-core/src/methods/images.ts +++ b/packages/image-comparison-core/src/methods/images.ts @@ -450,10 +450,15 @@ export async function executeImageCompare( ) // 6a. Save actual image on failure if alwaysSaveActualImage is false - const saveAboveTolerance = imageCompareOptions.saveAboveTolerance ?? 0 - const hasFailure = rawMisMatchPercentage > saveAboveTolerance - if (useBase64Image && hasFailure && actualBase64Image) { - // Save the actual image only when comparison fails + // Only save when saveAboveTolerance is explicitly set (e.g., by matchers). + // When using checkScreen directly without saveAboveTolerance, respect + // alwaysSaveActualImage: false literally and don't save actual images. + // @see https://github.com/webdriverio/visual-testing/issues/1115 + const saveAboveTolerance = imageCompareOptions.saveAboveTolerance + const shouldSaveOnFailure = saveAboveTolerance !== undefined + const hasFailure = rawMisMatchPercentage > (saveAboveTolerance ?? 0) + if (useBase64Image && shouldSaveOnFailure && hasFailure && actualBase64Image) { + // Save the actual image only when comparison fails and threshold was explicitly set await saveBase64Image(actualBase64Image, actualFilePath) } diff --git a/packages/visual-service/src/service.ts b/packages/visual-service/src/service.ts index 2759794e..7fc43450 100644 --- a/packages/visual-service/src/service.ts +++ b/packages/visual-service/src/service.ts @@ -229,6 +229,12 @@ export default class WdioImageComparisonService extends BaseClass { } const isCurrentContextNative = self.contextManager.isNativeContext + // save* methods should always save files, regardless of alwaysSaveActualImage config + const isSaveCommand = commandName === 'saveElement' + const wicOptions = isSaveCommand + ? { ...self.defaultOptions, alwaysSaveActualImage: true } + : self.defaultOptions + return [{ browserInstance, element, @@ -237,7 +243,7 @@ export default class WdioImageComparisonService extends BaseClass { isNativeContext: isCurrentContextNative, tag, [elementOptionsKey]: { - wic: self.defaultOptions, + wic: wicOptions, method: elementOptions, }, testContext: enrichTestContext({ @@ -294,6 +300,12 @@ export default class WdioImageComparisonService extends BaseClass { } const isCurrentContextNative = self.contextManager.isNativeContext + // save* methods should always save files, regardless of alwaysSaveActualImage config + const isSaveCommand = commandName === 'saveScreen' || commandName === 'saveFullPageScreen' || commandName === 'saveTabbablePage' + const wicOptions = isSaveCommand + ? { ...self.defaultOptions, alwaysSaveActualImage: true } + : self.defaultOptions + return [{ browserInstance, folders: getFolders(pageOptions, self.folders, self.#getBaselineFolder()), @@ -301,7 +313,7 @@ export default class WdioImageComparisonService extends BaseClass { isNativeContext: isCurrentContextNative, tag, [pageOptionsKey]: { - wic: self.defaultOptions, + wic: wicOptions, method: pageOptions, }, testContext: enrichTestContext({ diff --git a/packages/visual-service/tests/service.test.ts b/packages/visual-service/tests/service.test.ts index b54715f7..7a8954e7 100644 --- a/packages/visual-service/tests/service.test.ts +++ b/packages/visual-service/tests/service.test.ts @@ -3,6 +3,7 @@ import logger from '@wdio/logger' import { expect as wdioExpect } from '@wdio/globals' import { beforeEach, describe, expect, it, vi } from 'vitest' import VisualService from '../src/index.js' +import { saveScreen } from '@wdio/image-comparison-core' const log = logger('test') vi.mock('@wdio/logger', () => import(join(process.cwd(), '__mocks__', '@wdio/logger'))) @@ -17,6 +18,7 @@ vi.mock('@wdio/image-comparison-core', () => ({ saveTabbablePage: vi.fn(), checkTabbablePage: vi.fn(), DEFAULT_TEST_CONTEXT: {}, + FOLDERS: { ACTUAL: 'actual', DIFF: 'diff', TEMP_FULL_SCREEN: 'tempFullScreen', DEFAULT: { BASE: './__snapshots__/', SCREENSHOTS: '.tmp/' } }, NOT_KNOWN: 'not_known', DEVICE_RECTANGLES: { bottomBar: { y: 0, x: 0, width: 0, height: 0 }, @@ -172,5 +174,30 @@ describe('@wdio/visual-service', () => { expect(log.warn).toMatchSnapshot() }) + + it('should pass alwaysSaveActualImage: true to core for direct saveScreen calls when config is false', async () => { + vi.mocked(saveScreen).mockResolvedValue({} as any) + const service = new VisualService({ alwaysSaveActualImage: false }, {}, {} as unknown as WebdriverIO.Config) + // Mocked BaseClass does not set defaultOptions/folders; set them so the command can run + ;(service as any).defaultOptions = { alwaysSaveActualImage: false } + ;(service as any).folders = { baselineFolder: './__snapshots__/' } + const browser = { + isMultiremote: false, + addCommand: vi.fn((name, fn) => { + (browser as any)[name] = fn + }), + capabilities: {}, + requestedCapabilities: {}, + on: vi.fn(), + execute: vi.fn().mockResolvedValue(1), + } as any as WebdriverIO.Browser + + await service.before({}, [], browser) + await (browser as any).saveScreen('tag') + + expect(saveScreen).toHaveBeenCalledTimes(1) + const [saveScreenOptions] = vi.mocked(saveScreen).mock.calls[0] + expect((saveScreenOptions as any).saveScreenOptions?.wic?.alwaysSaveActualImage).toBe(true) + }) }) }) diff --git a/tests/configs/wdio.lambdatest.shared.conf.ts b/tests/configs/wdio.lambdatest.shared.conf.ts index 5e81cd09..1ad06c1d 100644 --- a/tests/configs/wdio.lambdatest.shared.conf.ts +++ b/tests/configs/wdio.lambdatest.shared.conf.ts @@ -46,6 +46,7 @@ export const config: WebdriverIO.Config = { rawMisMatchPercentage: !!process.env.RAW_MISMATCH || false, enableLayoutTesting: true, ignoreAntialiasing: true, + alwaysSaveActualImage: false, } satisfies VisualServiceOptions, ], ], diff --git a/tests/configs/wdio.local.desktop.conf.ts b/tests/configs/wdio.local.desktop.conf.ts index d7ec4494..7fd12534 100644 --- a/tests/configs/wdio.local.desktop.conf.ts +++ b/tests/configs/wdio.local.desktop.conf.ts @@ -72,7 +72,7 @@ export const config: WebdriverIO.Config = { createJsonReportFiles: true, clearRuntimeFolder: true, enableLayoutTesting: true, - alwaysSaveActualImage: false, + alwaysSaveActualImage: process.env.SAVE_ACTUAL === 'true', }, ] ], diff --git a/tests/lambdaTestBaseline/desktop_chrome/fullPage-chrome-latest-320x658.png b/tests/lambdaTestBaseline/desktop_chrome/fullPage-chrome-latest-320x658.png new file mode 100644 index 00000000..5e6a1030 Binary files /dev/null and b/tests/lambdaTestBaseline/desktop_chrome/fullPage-chrome-latest-320x658.png differ diff --git a/tests/lambdaTestBaseline/desktop_chrome/noActualStoredOnDiff-chrome-latest-1366x768.png b/tests/lambdaTestBaseline/desktop_chrome/noActualStoredOnDiff-chrome-latest-1366x768.png new file mode 100644 index 00000000..14cd80de Binary files /dev/null and b/tests/lambdaTestBaseline/desktop_chrome/noActualStoredOnDiff-chrome-latest-1366x768.png differ diff --git a/tests/lambdaTestBaseline/desktop_chrome/noActualStoredOnDiff-chrome-latest-320x658.png b/tests/lambdaTestBaseline/desktop_chrome/noActualStoredOnDiff-chrome-latest-320x658.png new file mode 100644 index 00000000..84c82a27 Binary files /dev/null and b/tests/lambdaTestBaseline/desktop_chrome/noActualStoredOnDiff-chrome-latest-320x658.png differ diff --git a/tests/lambdaTestBaseline/desktop_chrome/tabbable-chrome-latest-320x658.png b/tests/lambdaTestBaseline/desktop_chrome/tabbable-chrome-latest-320x658.png new file mode 100644 index 00000000..e91b2622 Binary files /dev/null and b/tests/lambdaTestBaseline/desktop_chrome/tabbable-chrome-latest-320x658.png differ diff --git a/tests/lambdaTestBaseline/desktop_chrome/viewportScreenshot-chrome-latest-320x658.png b/tests/lambdaTestBaseline/desktop_chrome/viewportScreenshot-chrome-latest-320x658.png new file mode 100644 index 00000000..84c82a27 Binary files /dev/null and b/tests/lambdaTestBaseline/desktop_chrome/viewportScreenshot-chrome-latest-320x658.png differ diff --git a/tests/lambdaTestBaseline/desktop_chrome/wdioLogo-chrome-latest-320x658.png b/tests/lambdaTestBaseline/desktop_chrome/wdioLogo-chrome-latest-320x658.png new file mode 100644 index 00000000..a0bce2d5 Binary files /dev/null and b/tests/lambdaTestBaseline/desktop_chrome/wdioLogo-chrome-latest-320x658.png differ diff --git a/tests/lambdaTestBaseline/desktop_firefox/noActualStoredOnDiff-Firefox_latest-1366x768.png b/tests/lambdaTestBaseline/desktop_firefox/noActualStoredOnDiff-Firefox_latest-1366x768.png new file mode 100644 index 00000000..ea5bf6fa Binary files /dev/null and b/tests/lambdaTestBaseline/desktop_firefox/noActualStoredOnDiff-Firefox_latest-1366x768.png differ diff --git a/tests/lambdaTestBaseline/desktop_microsoftedge/noActualStoredOnDiff-Microsoft_Edge_latest-1366x768.png b/tests/lambdaTestBaseline/desktop_microsoftedge/noActualStoredOnDiff-Microsoft_Edge_latest-1366x768.png new file mode 100644 index 00000000..4823e514 Binary files /dev/null and b/tests/lambdaTestBaseline/desktop_microsoftedge/noActualStoredOnDiff-Microsoft_Edge_latest-1366x768.png differ diff --git a/tests/lambdaTestBaseline/desktop_safari/noActualStoredOnDiff-SafariLatest-1366x768.png b/tests/lambdaTestBaseline/desktop_safari/noActualStoredOnDiff-SafariLatest-1366x768.png new file mode 100644 index 00000000..ab304908 Binary files /dev/null and b/tests/lambdaTestBaseline/desktop_safari/noActualStoredOnDiff-SafariLatest-1366x768.png differ diff --git a/tests/specs/desktop.spec.ts b/tests/specs/desktop.spec.ts index f4ff46b8..bf276dc5 100644 --- a/tests/specs/desktop.spec.ts +++ b/tests/specs/desktop.spec.ts @@ -1,4 +1,6 @@ +import type { ImageCompareResult } from '@wdio/image-comparison-core' import { browser, expect } from '@wdio/globals' +import { fileExists } from '../helpers/fileExists.ts' describe('@wdio/visual-service desktop', () => { // @TODO @@ -39,4 +41,23 @@ describe('@wdio/visual-service desktop', () => { ], }) }) + + it(`should not store an actual image for '${browserName}' when the diff is below the threshold (#1115)`, async function () { + const tag = 'noActualStoredOnDiff' + + await browser.execute(() => { + const el = document.createElement('div') + el.id = 'test-diff-element' + el.style.cssText = 'position:fixed;top:10px;left:10px;width:500px;height:500px;background:red;z-index:9999;' + document.body.appendChild(el) + }) + + const result = await browser.checkScreen(tag, { + returnAllCompareData: true, + }) as ImageCompareResult + + expect(result.misMatchPercentage).toBeGreaterThan(0) + expect(result.misMatchPercentage).toBeLessThanOrEqual(70) + expect(fileExists(result.folders.actual)).toBe(false) + }) })