From 6b5882970481cef9153193ffaeffd62232077134 Mon Sep 17 00:00:00 2001 From: Adam Setch Date: Fri, 16 Oct 2026 17:59:19 -0400 Subject: [PATCH 1/3] ci: enforce playwright container image alignment Renovate cannot keep the visual-regression Playwright container image in lockstep with the playwright devDependency: they are separate datasources with independent release timelines, and MCR exposes no release timestamp for minimumReleaseAge to compare. Grouping them still resolves to different versions and opens image-only PRs that strand the screenshot baselines (see #3437, #3299). Make package.json the source of truth and fail fast in Run Tests whenever the workflow image tag drifts from the devDependency. --- renovate.json | 9 +++--- src/main/visualRegressionImage.test.ts | 45 ++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 src/main/visualRegressionImage.test.ts diff --git a/renovate.json b/renovate.json index 7e1d62942..f6581b7a6 100644 --- a/renovate.json +++ b/renovate.json @@ -13,11 +13,10 @@ "allowedVersions": "<12" }, { - "description": "Group Playwright with the browser container image used by visual regression tests, and wait for both updates to pass the release-age check.", - "matchPackageNames": ["playwright", "mcr.microsoft.com/playwright"], - "groupName": "playwright", - "minimumReleaseAge": "3 days", - "internalChecksFilter": "strict" + "description": "Leave the visual-regression Playwright container image to a deliberate bump rather than Renovate. It must match the playwright devDependency exactly (both ship the same browser build), but the npm package and the MCR image are separate datasources and MCR exposes no release timestamp for minimumReleaseAge to compare, so grouping them still drifts the versions and opens image-only PRs that strand the screenshot baselines. package.json is the source of truth; src/main/visualRegressionImage.test.ts enforces the match and scripts/visual.sh derives the same image locally. The digest stays pinned for zizmor.", + "matchDatasources": ["docker"], + "matchPackageNames": ["mcr.microsoft.com/playwright"], + "enabled": false }, { "description": "Give core runtime dependency updates a distinct scope so release-please can include them in the changelog.", diff --git a/src/main/visualRegressionImage.test.ts b/src/main/visualRegressionImage.test.ts new file mode 100644 index 000000000..4fd81cb04 --- /dev/null +++ b/src/main/visualRegressionImage.test.ts @@ -0,0 +1,45 @@ +import { readFileSync } from 'node:fs'; + +/** + * The visual-regression job runs inside a pinned Playwright container image. + * Screenshots are only comparable against the exact browser build that image + * ships, so its version must match the `playwright` devDependency exactly — the + * same coupling `scripts/visual.sh` applies to local runs. + * + * Renovate cannot keep the two in lockstep on its own: the npm package and the + * MCR image are separate datasources, and MCR exposes no release timestamp for + * `minimumReleaseAge` to compare, so grouping them still drifts the versions and + * opens image-only PRs. This check fails fast — in the Run Tests job rather than + * the slower Visual Regression job — whenever they diverge. + */ +const repoRoot = new URL('../../', import.meta.url); + +const PLAYWRIGHT_IMAGE = + /container:\s+mcr\.microsoft\.com\/playwright:v([^@\s]+)-noble@(sha256:[0-9a-f]{64})/; + +describe('CI visual-regression Playwright image', () => { + const { devDependencies } = JSON.parse( + readFileSync(new URL('package.json', repoRoot), 'utf8'), + ) as { devDependencies: Record }; + + const workflow = readFileSync(new URL('.github/workflows/test.yml', repoRoot), 'utf8'); + + it('is pinned to the playwright devDependency version', () => { + const pinned = workflow.match(PLAYWRIGHT_IMAGE); + const playwright = devDependencies.playwright; + + expect( + pinned, + 'Expected a digest-pinned mcr.microsoft.com/playwright container in the visual-regression job of .github/workflows/test.yml.', + ).not.toBeNull(); + + expect( + pinned?.[1], + [ + `The visual-regression container must run mcr.microsoft.com/playwright:v${playwright}-noble to match the playwright devDependency.`, + 'Update the tag and refresh the pinned digest with:', + ` docker buildx imagetools inspect mcr.microsoft.com/playwright:v${playwright}-noble --format '{{.Manifest.Digest}}'`, + ].join('\n'), + ).toBe(playwright); + }); +}); From 566a569f7d1106a2c10050085dce12219a109c03 Mon Sep 17 00:00:00 2001 From: Adam Setch Date: Fri, 16 Oct 2026 18:11:51 -0400 Subject: [PATCH 2/3] test: generalize the Playwright guard into CI dependency compatibility Also assert the exact vitest/vite couplings CI depends on: @vitest/browser-playwright and @vitest/coverage-v8 peer on the vitest version that vite-plus bundles, and vite must resolve to the vite-plus-core build that vite-plus ships. --- .github/workflows/test.yml | 5 +- renovate.json | 2 +- src/main/ciDependencyCompatibility.test.ts | 79 ++++++++++++++++++++++ src/main/visualRegressionImage.test.ts | 45 ------------ 4 files changed, 83 insertions(+), 48 deletions(-) create mode 100644 src/main/ciDependencyCompatibility.test.ts delete mode 100644 src/main/visualRegressionImage.test.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b9283fbf0..7448c0e13 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,8 +49,9 @@ jobs: # font stack, which a bare runner does not provide. # # This tag must match the `playwright` devDependency, which ships the - # matching browser build. Renovate groups the two so they always move - # together; `scripts/visual.sh` reads the version straight from package.json. + # matching browser build. `scripts/visual.sh` derives the image from it for + # local runs, and `src/main/ciDependencyCompatibility.test.ts` asserts the + # two stay in sync so the version cannot drift unnoticed. container: mcr.microsoft.com/playwright:v1.64.0-noble@sha256:06a9939e57531807f8d5fd76ce44b53165ffb7d7501d87ab10e285c20b1e971f steps: diff --git a/renovate.json b/renovate.json index f6581b7a6..1b247a00d 100644 --- a/renovate.json +++ b/renovate.json @@ -13,7 +13,7 @@ "allowedVersions": "<12" }, { - "description": "Leave the visual-regression Playwright container image to a deliberate bump rather than Renovate. It must match the playwright devDependency exactly (both ship the same browser build), but the npm package and the MCR image are separate datasources and MCR exposes no release timestamp for minimumReleaseAge to compare, so grouping them still drifts the versions and opens image-only PRs that strand the screenshot baselines. package.json is the source of truth; src/main/visualRegressionImage.test.ts enforces the match and scripts/visual.sh derives the same image locally. The digest stays pinned for zizmor.", + "description": "Leave the visual-regression Playwright container image to a deliberate bump rather than Renovate. It must match the playwright devDependency exactly (both ship the same browser build), but the npm package and the MCR image are separate datasources and MCR exposes no release timestamp for minimumReleaseAge to compare, so grouping them still drifts the versions and opens image-only PRs that strand the screenshot baselines. package.json is the source of truth; src/main/ciDependencyCompatibility.test.ts enforces the match and scripts/visual.sh derives the same image locally. The digest stays pinned for zizmor.", "matchDatasources": ["docker"], "matchPackageNames": ["mcr.microsoft.com/playwright"], "enabled": false diff --git a/src/main/ciDependencyCompatibility.test.ts b/src/main/ciDependencyCompatibility.test.ts new file mode 100644 index 000000000..94f6c6bd0 --- /dev/null +++ b/src/main/ciDependencyCompatibility.test.ts @@ -0,0 +1,79 @@ +import { readFileSync } from 'node:fs'; + +/** + * Guards version couplings that CI depends on but that no single manifest + * expresses. Renovate (or a manual edit) can move one half of a pair and leave + * the other behind, which otherwise only surfaces as a confusing failure deep in + * the test or visual-regression run. Asserting the couplings here — against the + * versions actually installed — turns that into a fast, explicit failure. + * + * The vitest couplings are exact peer dependencies: `@vitest/browser-playwright` + * and `@vitest/coverage-v8` peer on the same `vitest` version that `vite-plus` + * bundles, and `vite` must resolve to the `@voidzero-dev/vite-plus-core` build + * that `vite-plus` ships. + */ +const repoRoot = new URL('../../', import.meta.url); + +function readJson(path: string): T { + return JSON.parse(readFileSync(new URL(path, repoRoot), 'utf8')) as T; +} + +type PackageJson = { + name?: string; + version: string; + dependencies?: Record; +}; + +const workflow = readFileSync(new URL('.github/workflows/test.yml', repoRoot), 'utf8'); + +const playwright = readJson('node_modules/playwright/package.json'); +const vitest = readJson('node_modules/vitest/package.json'); +const browserPlaywright = readJson( + 'node_modules/@vitest/browser-playwright/package.json', +); +const coverageV8 = readJson('node_modules/@vitest/coverage-v8/package.json'); +const vitePlus = readJson('node_modules/vite-plus/package.json'); +const vite = readJson('node_modules/vite/package.json'); + +const PINNED_PLAYWRIGHT_IMAGE = + /container:\s+mcr\.microsoft\.com\/playwright:v([^@\s]+)-noble@(sha256:[0-9a-f]{64})/; + +describe('CI dependency compatibility', () => { + describe('playwright', () => { + it('pins the visual-regression image to the installed playwright version', () => { + const pinned = workflow.match(PINNED_PLAYWRIGHT_IMAGE); + + expect( + pinned, + 'Expected a digest-pinned mcr.microsoft.com/playwright container in the visual-regression job of .github/workflows/test.yml.', + ).not.toBeNull(); + + expect( + pinned?.[1], + [ + `The visual-regression container must run mcr.microsoft.com/playwright:v${playwright.version}-noble to match the installed playwright (driven by the package.json devDependency).`, + 'Update the tag and refresh the pinned digest with:', + ` docker buildx imagetools inspect mcr.microsoft.com/playwright:v${playwright.version}-noble --format '{{.Manifest.Digest}}'`, + ].join('\n'), + ).toBe(playwright.version); + }); + }); + + describe('vitest family', () => { + it('installs the browser provider and coverage reporter at the vitest version', () => { + expect(browserPlaywright.version).toBe(vitest.version); + expect(coverageV8.version).toBe(vitest.version); + }); + + it('installs vitest at the version bundled by vite-plus', () => { + expect(vitest.version).toBe(vitePlus.dependencies?.vitest); + }); + }); + + describe('vite family', () => { + it('resolves vite to the vite-plus-core build that vite-plus ships', () => { + expect(vite.name).toBe('@voidzero-dev/vite-plus-core'); + expect(vitePlus.dependencies?.vite).toBe(`npm:${vite.name}@${vite.version}`); + }); + }); +}); diff --git a/src/main/visualRegressionImage.test.ts b/src/main/visualRegressionImage.test.ts deleted file mode 100644 index 4fd81cb04..000000000 --- a/src/main/visualRegressionImage.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { readFileSync } from 'node:fs'; - -/** - * The visual-regression job runs inside a pinned Playwright container image. - * Screenshots are only comparable against the exact browser build that image - * ships, so its version must match the `playwright` devDependency exactly — the - * same coupling `scripts/visual.sh` applies to local runs. - * - * Renovate cannot keep the two in lockstep on its own: the npm package and the - * MCR image are separate datasources, and MCR exposes no release timestamp for - * `minimumReleaseAge` to compare, so grouping them still drifts the versions and - * opens image-only PRs. This check fails fast — in the Run Tests job rather than - * the slower Visual Regression job — whenever they diverge. - */ -const repoRoot = new URL('../../', import.meta.url); - -const PLAYWRIGHT_IMAGE = - /container:\s+mcr\.microsoft\.com\/playwright:v([^@\s]+)-noble@(sha256:[0-9a-f]{64})/; - -describe('CI visual-regression Playwright image', () => { - const { devDependencies } = JSON.parse( - readFileSync(new URL('package.json', repoRoot), 'utf8'), - ) as { devDependencies: Record }; - - const workflow = readFileSync(new URL('.github/workflows/test.yml', repoRoot), 'utf8'); - - it('is pinned to the playwright devDependency version', () => { - const pinned = workflow.match(PLAYWRIGHT_IMAGE); - const playwright = devDependencies.playwright; - - expect( - pinned, - 'Expected a digest-pinned mcr.microsoft.com/playwright container in the visual-regression job of .github/workflows/test.yml.', - ).not.toBeNull(); - - expect( - pinned?.[1], - [ - `The visual-regression container must run mcr.microsoft.com/playwright:v${playwright}-noble to match the playwright devDependency.`, - 'Update the tag and refresh the pinned digest with:', - ` docker buildx imagetools inspect mcr.microsoft.com/playwright:v${playwright}-noble --format '{{.Manifest.Digest}}'`, - ].join('\n'), - ).toBe(playwright); - }); -}); From 783dbabb110d8eb8479ea387d8ad0b7b9fdda814 Mon Sep 17 00:00:00 2001 From: Adam Setch Date: Fri, 16 Oct 2026 18:22:57 -0400 Subject: [PATCH 3/3] chore(renovate): drop the Playwright group rule It tried to keep the npm package and the MCR image in lockstep, but Renovate can't compare the two release timelines (MCR exposes no release timestamp for minimumReleaseAge to read), so it never actually grouped them. The CI dependency compatibility test now guards the pairing. --- renovate.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/renovate.json b/renovate.json index 1b247a00d..55fb04f13 100644 --- a/renovate.json +++ b/renovate.json @@ -12,12 +12,6 @@ "matchPackageNames": ["got"], "allowedVersions": "<12" }, - { - "description": "Leave the visual-regression Playwright container image to a deliberate bump rather than Renovate. It must match the playwright devDependency exactly (both ship the same browser build), but the npm package and the MCR image are separate datasources and MCR exposes no release timestamp for minimumReleaseAge to compare, so grouping them still drifts the versions and opens image-only PRs that strand the screenshot baselines. package.json is the source of truth; src/main/ciDependencyCompatibility.test.ts enforces the match and scripts/visual.sh derives the same image locally. The digest stays pinned for zizmor.", - "matchDatasources": ["docker"], - "matchPackageNames": ["mcr.microsoft.com/playwright"], - "enabled": false - }, { "description": "Give core runtime dependency updates a distinct scope so release-please can include them in the changelog.", "matchPackageNames": [