From fb19ea7e9312169efa7feaebf6c482b94328d148 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 11 May 2026 19:31:39 +0200 Subject: [PATCH 01/81] chore: Add visual regression testing --- .github/workflows/visual-regression.yml | 86 +++++++++++++++ build-tools/tasks/visual.js | 107 +++++++++++++++++++ build-tools/visual/global-setup.js | 4 + build-tools/visual/global-teardown.js | 4 + build-tools/visual/setup.js | 18 ++++ docs/RUNNING_TESTS.md | 59 +++++++++- eslint.config.mjs | 2 +- gulpfile.js | 2 + jest.visual.config.js | 25 +++++ package.json | 1 + test/visual/compare-screenshots.ts | 76 +++++++++++++ test/visual/definitions/alert.ts | 15 +++ test/visual/definitions/button.ts | 15 +++ test/visual/definitions/date-range-picker.ts | 40 +++++++ test/visual/definitions/index.ts | 12 +++ test/visual/definitions/table.ts | 15 +++ test/visual/types.ts | 23 ++++ test/visual/visual.test.ts | 6 ++ 18 files changed, 504 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/visual-regression.yml create mode 100644 build-tools/tasks/visual.js create mode 100644 build-tools/visual/global-setup.js create mode 100644 build-tools/visual/global-teardown.js create mode 100644 build-tools/visual/setup.js create mode 100644 jest.visual.config.js create mode 100644 test/visual/compare-screenshots.ts create mode 100644 test/visual/definitions/alert.ts create mode 100644 test/visual/definitions/button.ts create mode 100644 test/visual/definitions/date-range-picker.ts create mode 100644 test/visual/definitions/index.ts create mode 100644 test/visual/definitions/table.ts create mode 100644 test/visual/types.ts create mode 100644 test/visual/visual.test.ts diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml new file mode 100644 index 0000000000..a211178f4c --- /dev/null +++ b/.github/workflows/visual-regression.yml @@ -0,0 +1,86 @@ +name: Visual Regression Tests + +on: + pull_request: + branches: + - main + +defaults: + run: + shell: bash + +permissions: + id-token: write + contents: read + +jobs: + visual: + name: Visual regression + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 18 + cache: npm + + - name: Install ChromeDriver + run: npm install -g chromedriver + + # ── Build PR (test) pages ────────────────────────────────────────────── + # Install the PR's dependencies and build its pages. + - name: Install PR dependencies + run: npm ci + + - name: Build PR pages + run: npx gulp quick-build + env: + NODE_ENV: production + + - name: Bundle PR pages + run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-default + env: + NODE_ENV: production + + # ── Build baseline (main) pages ──────────────────────────────────────── + # Use a git worktree so the baseline has its own directory and its own + # node_modules. This means a PR that changes package-lock.json will still + # produce a correct baseline: the baseline installs from main's lockfile + # and the PR build installs from the PR's lockfile, so both sides use the + # dependency versions that are correct for their respective source trees. + - name: Create baseline worktree from origin/main + run: git worktree add /tmp/baseline origin/main + + - name: Install baseline dependencies + run: npm ci + working-directory: /tmp/baseline + + - name: Build baseline pages + run: npx gulp quick-build + working-directory: /tmp/baseline + env: + NODE_ENV: production + + - name: Bundle baseline pages + run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path ${{ github.workspace }}/pages/lib/static-visual-baseline + working-directory: /tmp/baseline + env: + NODE_ENV: production + + # ── Run tests ───────────────────────────────────────────────────────── + - name: Run visual regression tests + run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js + env: + TZ: UTC + + - name: Upload diff artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: visual-regression-diffs + path: visual-regression-output/ + retention-days: 14 diff --git a/build-tools/tasks/visual.js b/build-tools/tasks/visual.js new file mode 100644 index 0000000000..0864790afb --- /dev/null +++ b/build-tools/tasks/visual.js @@ -0,0 +1,107 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +const execa = require('execa'); +const path = require('path'); +const fs = require('fs'); +const waitOn = require('wait-on'); +const { task } = require('../utils/gulp-utils.js'); +const { parseArgs } = require('node:util'); + +const BASELINE_WORKTREE = '/tmp/visual-baseline'; +const BASELINE_OUTPUT = path.resolve('pages/lib/static-visual-baseline'); +const TEST_OUTPUT = path.resolve('pages/lib/static-default'); + +// Port assignments: +// 8080 — test build (PR / local changes) +// 8081 — baseline build (main branch) +const TEST_PORT = 8080; +const BASELINE_PORT = 8081; + +/** + * Serves a pre-built static directory using webpack-dev-server in static mode. + */ +function serveStatic(dir, port) { + return execa( + 'node_modules/.bin/webpack', + ['serve', '--config', 'pages/webpack.config.integ.cjs', '--port', String(port), '--static', dir, '--no-hot'], + { env: { ...process.env, NODE_ENV: 'development' } } + ); +} + +/** + * Builds the dev pages from the source tree at `cwd` into `outputPath`. + * Uses the node_modules present in `cwd`. + */ +async function buildPages(cwd, outputPath) { + await execa('npx', ['gulp', 'quick-build'], { + stdio: 'inherit', + cwd, + env: { ...process.env, NODE_ENV: 'production' }, + }); + await execa( + path.join(cwd, 'node_modules/.bin/webpack'), + ['--config', 'pages/webpack.config.integ.cjs', '--output-path', outputPath], + { stdio: 'inherit', cwd, env: { ...process.env, NODE_ENV: 'production' } } + ); +} + +module.exports = task('test:visual', async () => { + const options = { + shard: { type: 'string' }, + // Pass --skip-build to skip the build steps when artifacts are already present. + skipBuild: { type: 'boolean' }, + }; + const { shard, skipBuild } = parseArgs({ options, strict: false }).values; + + const cwd = process.cwd(); + + if (!skipBuild) { + // ── 1. Build the test (PR) pages ──────────────────────────────────────── + console.log('Building test pages (current branch)…'); + await buildPages(cwd, TEST_OUTPUT); + + // ── 2. Build the baseline (main) pages ────────────────────────────────── + // Create a worktree for origin/main so it gets its own node_modules. + // This correctly handles PRs that change package-lock.json: each side + // installs from its own lockfile. + console.log('Setting up baseline worktree from origin/main…'); + if (fs.existsSync(BASELINE_WORKTREE)) { + await execa('git', ['worktree', 'remove', '--force', BASELINE_WORKTREE]); + } + await execa('git', ['worktree', 'add', BASELINE_WORKTREE, 'origin/main']); + + try { + console.log('Installing baseline dependencies…'); + await execa('npm', ['ci'], { stdio: 'inherit', cwd: BASELINE_WORKTREE }); + + console.log('Building baseline pages (origin/main)…'); + await buildPages(BASELINE_WORKTREE, BASELINE_OUTPUT); + } finally { + await execa('git', ['worktree', 'remove', '--force', BASELINE_WORKTREE]); + } + } + + // ── 3. Start both static servers ────────────────────────────────────────── + console.log(`Starting test server on :${TEST_PORT} (${TEST_OUTPUT})…`); + const testServer = serveStatic(TEST_OUTPUT, TEST_PORT); + + console.log(`Starting baseline server on :${BASELINE_PORT} (${BASELINE_OUTPUT})…`); + const baselineServer = serveStatic(BASELINE_OUTPUT, BASELINE_PORT); + + try { + await waitOn({ resources: [`http://localhost:${TEST_PORT}`, `http://localhost:${BASELINE_PORT}`] }); + + // ── 4. Run visual tests ────────────────────────────────────────────────── + const jestArgs = ['-c', 'jest.visual.config.js']; + if (shard) { + jestArgs.push(`--shard=${shard}`); + } + await execa('jest', jestArgs, { + stdio: 'inherit', + env: { ...process.env, NODE_OPTIONS: '--experimental-vm-modules' }, + }); + } finally { + testServer.cancel(); + baselineServer.cancel(); + } +}); diff --git a/build-tools/visual/global-setup.js b/build-tools/visual/global-setup.js new file mode 100644 index 0000000000..075ee6e398 --- /dev/null +++ b/build-tools/visual/global-setup.js @@ -0,0 +1,4 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +const { startWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); +module.exports = () => startWebdriver(); diff --git a/build-tools/visual/global-teardown.js b/build-tools/visual/global-teardown.js new file mode 100644 index 0000000000..57ad21b454 --- /dev/null +++ b/build-tools/visual/global-teardown.js @@ -0,0 +1,4 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +const { shutdownWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); +module.exports = () => shutdownWebdriver(); diff --git a/build-tools/visual/setup.js b/build-tools/visual/setup.js new file mode 100644 index 0000000000..2625a43809 --- /dev/null +++ b/build-tools/visual/setup.js @@ -0,0 +1,18 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +/* global jest */ +const { configure } = require('@cloudscape-design/browser-test-tools/use-browser'); + +// The PR build (the code under test) is served on port 8080. +// The baseline build (main branch, same node_modules) is served on port 8081. +configure({ + browserName: 'ChromeHeadlessIntegration', + browserCreatorOptions: { + seleniumUrl: 'http://localhost:9515', + }, + webdriverOptions: { + baseUrl: 'http://localhost:8080', + }, +}); + +jest.retryTimes(2, { logErrorsBeforeRetry: true }); diff --git a/docs/RUNNING_TESTS.md b/docs/RUNNING_TESTS.md index 525cdf181d..c5c483e981 100644 --- a/docs/RUNNING_TESTS.md +++ b/docs/RUNNING_TESTS.md @@ -60,11 +60,60 @@ TZ=UTC npx jest -u -c jest.unit.config.js src/ ``` ## Visual Regression Tests -> **Note:** The components repository does not have visual regression tests on GitHub. This section applies to other repositories such as chat-components, code-view, chart-components, and board-components. +Visual regression tests run automatically when opening a pull request in GitHub (see `.github/workflows/visual-regression.yml`). -Visual regression tests for permutation pages run automatically when opening a pull request in GitHub. +They compare permutation pages between the PR build and a baseline build of `main`, both served locally in the same CI job. Each side installs from its own `package-lock.json` via a git worktree, so dependency changes in the PR are handled correctly and unpinned updates in sister repositories affect both sides equally. -To check results: look at the "Visual Regression Tests" action in the PR. The "Test for regressions" step logs which pages failed. For a full report, download the `visual-regression-snapshots-results` artifact from the action summary. +### How it works -If there are unexpected regressions, fix your pull request. -If the changes are expected, call this out in your pull request comments. +1. The PR pages are built and served on port 8080. +2. A git worktree of `origin/main` is created, its dependencies installed, and its pages built and served on port 8081. +3. The single test runner (`test/visual/visual.test.ts`) iterates over all test definitions, captures the `.screenshot-area` element from both servers for each test, and fails if any pixels differ. + +### Running locally + +``` +npm run test:visual +``` + +This handles the full build and comparison in one command. If both outputs are already built, skip the build step: + +``` +NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js +``` + +(Requires both servers to be running — start the PR build with `npm run start:integ` on port 8080 and the baseline build on port 8081, or set `NEW_HOST` / `OLD_HOST` env vars to point at different hosts.) + +### Adding tests for a new component + +Create `test/visual/definitions/.ts`: + +```ts +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'my-component', + tests: [ + { + description: 'permutations', + path: 'my-component/permutations', + }, + ], +}; + +export default suite; +``` + +Then import and add it to `test/visual/definitions/index.ts`: + +```ts +import myComponent from './my-component'; + +export const allSuites: TestSuite[] = [..., myComponent]; +``` + +### Reviewing failures + +If the CI job fails, download the `visual-regression-diffs` artifact from the Actions summary. + +If the diff is expected (intentional visual change), note it in your PR description. diff --git a/eslint.config.mjs b/eslint.config.mjs index 0d9423aa5b..f03eb9ce60 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -225,7 +225,7 @@ export default tsEslint.config( }, }, { - files: ['**/__integ__/**', '**/__motion__/**', '**/__a11y__/**'], + files: ['**/__integ__/**', '**/__motion__/**', '**/__a11y__/**', 'test/visual/**'], rules: { // useBrowser is not a hook 'react-hooks/rules-of-hooks': 'off', diff --git a/gulpfile.js b/gulpfile.js index 6e3f389082..221c065982 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -19,6 +19,7 @@ const { generateI18nMessages, integ, motion, + visual, copyFiles, themeableSource, bundleVendorFiles, @@ -43,6 +44,7 @@ exports['test:unit'] = unit; exports['test:integ'] = integ; exports['test:a11y'] = a11y; exports['test:motion'] = motion; +exports['test:visual'] = visual; exports.watch = () => { watch( diff --git a/jest.visual.config.js b/jest.visual.config.js new file mode 100644 index 0000000000..7f1d020880 --- /dev/null +++ b/jest.visual.config.js @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +const path = require('path'); +const os = require('os'); + +module.exports = { + verbose: true, + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': [ + 'ts-jest', + { + tsconfig: 'tsconfig.integ.json', + }, + ], + }, + reporters: ['default', 'github-actions'], + testTimeout: 120_000, // 2min — pages can be tall and slow to capture + maxWorkers: os.cpus().length * (process.env.GITHUB_ACTION ? 3 : 1), + globalSetup: '/build-tools/visual/global-setup.js', + globalTeardown: '/build-tools/visual/global-teardown.js', + setupFilesAfterEnv: [path.join(__dirname, 'build-tools', 'visual', 'setup.js')], + moduleFileExtensions: ['js', 'ts'], + testMatch: ['/test/visual/visual.test.ts'], +}; diff --git a/package.json b/package.json index fda995324a..5a7a02c59f 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "test:a11y": "gulp test:a11y", "test:integ": "gulp test:integ", "test:motion": "gulp test:motion", + "test:visual": "gulp test:visual", "lint": "npm-run-all --parallel lint:*", "lint:eslint": "eslint .", "lint:stylelint": "stylelint --ignore-path .gitignore '{src,pages}/**/*.{css,scss}'", diff --git a/test/visual/compare-screenshots.ts b/test/visual/compare-screenshots.ts new file mode 100644 index 0000000000..51254363ac --- /dev/null +++ b/test/visual/compare-screenshots.ts @@ -0,0 +1,76 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import pixelmatch from 'pixelmatch'; +import { PNG } from 'pngjs'; + +import { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; +import useBrowser from '@cloudscape-design/browser-test-tools/use-browser'; + +import { TestDefinition, TestSuite } from './types'; + +const screenshotAreaSelector = '.screenshot-area'; +const defaultWindowSize = { width: 1600, height: 800 }; + +// NEW_HOST serves the PR's pages, OLD_HOST serves the baseline (main) pages. +const newHost = process.env.NEW_HOST || 'http://localhost:8080'; +const oldHost = process.env.OLD_HOST || 'http://localhost:8081'; + +async function captureScreenshot( + browser: WebdriverIO.Browser, + url: string, + setup?: (page: ScreenshotPageObject) => Promise +): Promise { + await browser.url(url); + const page = new ScreenshotPageObject(browser); + await page.waitForVisible(screenshotAreaSelector); + if (setup) { + await setup(page); + } + const { image } = await page.captureBySelector(screenshotAreaSelector); + return image; +} + +function buildUrl(host: string, path: string, queryParams?: Record): string { + const params = new URLSearchParams(queryParams); + const qs = params.toString(); + return `${host}/#/${path}${qs ? `?${qs}` : ''}`; +} + +function compareImages(newImage: PNG, oldImage: PNG): number { + const { width, height } = newImage; + const diff = new PNG({ width, height }); + return pixelmatch(newImage.data, oldImage.data, diff.data, width, height, { threshold: 0.1 }); +} + +function isTestDefinition(item: TestDefinition | TestSuite): item is TestDefinition { + return (item as TestDefinition).path !== undefined; +} + +export function runTestSuites(suites: Array) { + for (const item of suites) { + if (isTestDefinition(item)) { + runSingleTest(item); + } else { + describe(item.description, () => { + runTestSuites(item.tests); + }); + } + } +} + +function runSingleTest(testDef: TestDefinition) { + const windowSize = { ...defaultWindowSize, ...testDef.configuration }; + + test( + testDef.description, + useBrowser(windowSize, async browser => { + const newUrl = buildUrl(newHost, testDef.path, testDef.queryParams); + const newScreenshot = await captureScreenshot(browser, newUrl, testDef.setup); + + const oldUrl = buildUrl(oldHost, testDef.path, testDef.queryParams); + const oldScreenshot = await captureScreenshot(browser, oldUrl, testDef.setup); + const diffPixels = compareImages(newScreenshot, oldScreenshot); + expect(diffPixels).toBe(0); + }) + ); +} diff --git a/test/visual/definitions/alert.ts b/test/visual/definitions/alert.ts new file mode 100644 index 0000000000..37f0985f24 --- /dev/null +++ b/test/visual/definitions/alert.ts @@ -0,0 +1,15 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'alert', + tests: [ + { + description: 'permutations', + path: 'alert/permutations', + }, + ], +}; + +export default suite; diff --git a/test/visual/definitions/button.ts b/test/visual/definitions/button.ts new file mode 100644 index 0000000000..cb7d590a59 --- /dev/null +++ b/test/visual/definitions/button.ts @@ -0,0 +1,15 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'button', + tests: [ + { + description: 'permutations', + path: 'button/permutations', + }, + ], +}; + +export default suite; diff --git a/test/visual/definitions/date-range-picker.ts b/test/visual/definitions/date-range-picker.ts new file mode 100644 index 0000000000..6800eebc90 --- /dev/null +++ b/test/visual/definitions/date-range-picker.ts @@ -0,0 +1,40 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'date-range-picker', + tests: [ + { + description: 'with value', + path: 'date-range-picker/with-value', + }, + { + description: 'range calendar', + path: 'date-range-picker/range-calendar', + }, + { + description: 'month calendar permutations', + path: 'date-range-picker/month-calendar-permutations', + }, + { + description: 'year calendar permutations', + path: 'date-range-picker/year-calendar-permutations', + }, + { + description: 'in small viewport', + path: 'date-range-picker/small-viewport', + configuration: { width: 400 }, + }, + { + description: 'with dropdown open', + path: 'date-range-picker/with-value', + setup: async page => { + await page.click('[data-testid="date-range-picker-trigger"]'); + await page.waitForVisible('.awsui-context-content-header'); + }, + }, + ], +}; + +export default suite; diff --git a/test/visual/definitions/index.ts b/test/visual/definitions/index.ts new file mode 100644 index 0000000000..27f5c9e7ca --- /dev/null +++ b/test/visual/definitions/index.ts @@ -0,0 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Each component has its own test definition file. +// Import them here manually to form the full test suite. +import { TestSuite } from '../types'; +import alert from './alert'; +import button from './button'; +import dateRangePicker from './date-range-picker'; +import table from './table'; + +export const allSuites: TestSuite[] = [alert, button, dateRangePicker, table]; diff --git a/test/visual/definitions/table.ts b/test/visual/definitions/table.ts new file mode 100644 index 0000000000..6529046d7b --- /dev/null +++ b/test/visual/definitions/table.ts @@ -0,0 +1,15 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'table', + tests: [ + { + description: 'permutations', + path: 'table/permutations', + }, + ], +}; + +export default suite; diff --git a/test/visual/types.ts b/test/visual/types.ts new file mode 100644 index 0000000000..7c4c90809a --- /dev/null +++ b/test/visual/types.ts @@ -0,0 +1,23 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; + +export interface ScreenshotTestConfiguration { + width?: number; + height?: number; +} + +export type TestCallback = (page: ScreenshotPageObject) => Promise; + +export interface TestDefinition { + description: string; + path: string; + queryParams?: Record; + configuration?: ScreenshotTestConfiguration; + setup?: TestCallback; +} + +export interface TestSuite { + description: string; + tests: Array; +} diff --git a/test/visual/visual.test.ts b/test/visual/visual.test.ts new file mode 100644 index 0000000000..06ef0fa8a4 --- /dev/null +++ b/test/visual/visual.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from './compare-screenshots'; +import { allSuites } from './definitions'; + +runTestSuites(allSuites); From 47b7f087e49491d7619f6fa7af13d88fe74a70ca Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 11 May 2026 19:36:06 +0200 Subject: [PATCH 02/81] Install dependencies with npm i --- .github/workflows/visual-regression.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index a211178f4c..33e01c79fd 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -34,7 +34,7 @@ jobs: # ── Build PR (test) pages ────────────────────────────────────────────── # Install the PR's dependencies and build its pages. - name: Install PR dependencies - run: npm ci + run: npm i - name: Build PR pages run: npx gulp quick-build @@ -56,7 +56,7 @@ jobs: run: git worktree add /tmp/baseline origin/main - name: Install baseline dependencies - run: npm ci + run: npm i working-directory: /tmp/baseline - name: Build baseline pages From fb78d3a9e2478e883e1be8a12042acbd0df17cc9 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 11 May 2026 20:05:09 +0200 Subject: [PATCH 03/81] Use node 20 --- .github/workflows/visual-regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 33e01c79fd..642ab08bac 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -25,7 +25,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: 18 + node-version: 20 cache: npm - name: Install ChromeDriver From 16d40d7ede15e33af175c5fca3dea3e291aa97de Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 11 May 2026 20:12:53 +0200 Subject: [PATCH 04/81] Add pixelmatch types --- package-lock.json | 11 +++++++++++ package.json | 1 + 2 files changed, 12 insertions(+) diff --git a/package-lock.json b/package-lock.json index 1c7a32ecfd..660e509f79 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,6 +52,7 @@ "@types/jest": "^29.5.13", "@types/lodash": "^4.14.176", "@types/node": "^20.17.14", + "@types/pixelmatch": "^5.2.6", "@types/react": "^16.14.20", "@types/react-dom": "^16.9.14", "@types/react-is": "^18.2.0", @@ -4846,6 +4847,16 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/pixelmatch": { + "version": "5.2.6", + "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.6.tgz", + "integrity": "sha512-wC83uexE5KGuUODn6zkm9gMzTwdY5L0chiK+VrKcDfEjzxh1uadlWTvOmAbCpnM9zx/Ww3f8uKlYQVnO/TrqVg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/pngjs": { "version": "6.0.5", "dev": true, diff --git a/package.json b/package.json index 5a7a02c59f..7884df953b 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "@types/jest": "^29.5.13", "@types/lodash": "^4.14.176", "@types/node": "^20.17.14", + "@types/pixelmatch": "^5.2.6", "@types/react": "^16.14.20", "@types/react-dom": "^16.9.14", "@types/react-is": "^18.2.0", From b6f63856503ae099a4b78b7ceab67fbbbe3bc2a9 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 11 May 2026 20:44:04 +0200 Subject: [PATCH 05/81] Install Chromedriver in CI --- .github/workflows/visual-regression.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 642ab08bac..90af77becf 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -28,8 +28,10 @@ jobs: node-version: 20 cache: npm - - name: Install ChromeDriver - run: npm install -g chromedriver + - name: Setup Chrome and ChromeDriver + uses: browser-actions/setup-chrome@v1 + with: + chrome-version: stable # ── Build PR (test) pages ────────────────────────────────────────────── # Install the PR's dependencies and build its pages. From 0df42fc04f9a3b30358303d5c5ffc9bcd2ec4b51 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 11 May 2026 21:01:25 +0200 Subject: [PATCH 06/81] Start servers --- .github/workflows/visual-regression.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 90af77becf..7ee7d42669 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -74,6 +74,19 @@ jobs: NODE_ENV: production # ── Run tests ───────────────────────────────────────────────────────── + - name: Start test server (port 8080) + run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8080 --static pages/lib/static-default --no-hot & + env: + NODE_ENV: development + + - name: Start baseline server (port 8081) + run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8081 --static pages/lib/static-visual-baseline --no-hot & + env: + NODE_ENV: development + + - name: Wait for servers to be ready + run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081 + - name: Run visual regression tests run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js env: From 632df9b7ddf17b02d565a675d070c655279f96e2 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 11 May 2026 21:39:49 +0200 Subject: [PATCH 07/81] Install Puppeteer --- package-lock.json | 71 +++++++++++++++++++++++++++++++++++++++++++++-- package.json | 1 + 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 660e509f79..555d3fabdc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -97,6 +97,7 @@ "mockdate": "^3.0.5", "npm-run-all": "^4.1.5", "prettier": "^3.6.1", + "puppeteer-core": "^24.43.1", "react": "^16.14.0", "react-dom": "^16.14.0", "react-dom18": "npm:react-dom@^18.3.1", @@ -3522,9 +3523,9 @@ } }, "node_modules/@puppeteer/browsers": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.13.0.tgz", - "integrity": "sha512-46BZJYJjc/WwmKjsvDFykHtXrtomsCIrwYQPOP7VfMJoZY2bsDF9oROBABR3paDjDcmkUye1Pb1BqdcdiipaWA==", + "version": "2.13.2", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.13.2.tgz", + "integrity": "sha512-5EUZSUIc37H6aIXyWO0Z4y8NlF8NnjgmqeQgOGiswAU7pY0HOo16ho4+alIWmSfdZnjqBRawMsP3I5YqLSn6kw==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -7273,6 +7274,20 @@ "node": ">=6.0" } }, + "node_modules/chromium-bidi": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-14.0.0.tgz", + "integrity": "sha512-9gYlLtS6tStdRWzrtXaTMnqcM4dudNegMXJxkR0I/CXObHalYeYcAMPrL19eroNZHtJ8DQmu1E+ZNOYu/IXMXw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/ci-info": { "version": "3.9.0", "dev": true, @@ -8789,6 +8804,13 @@ "dev": true, "license": "MIT" }, + "node_modules/devtools-protocol": { + "version": "0.0.1608973", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1608973.tgz", + "integrity": "sha512-Tpm17fxYzt+J7VrGdc1k8YdRqS3YV7se/M6KeemEqvUbq/n7At1rWVuXMxQgpWkdwSdIEKYbU//Bve+Shm4YNQ==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/diff-sequences": { "version": "29.6.3", "dev": true, @@ -17746,6 +17768,25 @@ "node": ">=6" } }, + "node_modules/puppeteer-core": { + "version": "24.43.1", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.43.1.tgz", + "integrity": "sha512-T5ScUMAsmhdNbgDR41AGESYeS6V9MSgetkSnVhhW+gXvzC42VesKCn5ld87gAZDJ6vLHL9GkRvY9WtQWSnwFbw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.13.2", + "chromium-bidi": "14.0.0", + "debug": "^4.4.3", + "devtools-protocol": "0.0.1608973", + "typed-query-selector": "^2.12.2", + "webdriver-bidi-protocol": "0.4.1", + "ws": "^8.20.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/pure-rand": { "version": "6.1.0", "dev": true, @@ -21158,6 +21199,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typed-query-selector": { + "version": "2.12.2", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.2.tgz", + "integrity": "sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ==", + "dev": true, + "license": "MIT" + }, "node_modules/typedarray": { "version": "0.0.6", "dev": true, @@ -21675,6 +21723,13 @@ "node": ">=18.20.0" } }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz", + "integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/webdriver/node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -22452,6 +22507,16 @@ "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index 7884df953b..d25129aa23 100644 --- a/package.json +++ b/package.json @@ -121,6 +121,7 @@ "mockdate": "^3.0.5", "npm-run-all": "^4.1.5", "prettier": "^3.6.1", + "puppeteer-core": "^24.43.1", "react": "^16.14.0", "react-dom": "^16.14.0", "react-dom18": "npm:react-dom@^18.3.1", From 30bb9dbd17e6b5a8663740c8b0a004dac24f4c2e Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 11 May 2026 22:19:28 +0200 Subject: [PATCH 08/81] Capture screenshot area or permutations --- test/visual/compare-screenshots.ts | 51 +++++++++++++++++--- test/visual/definitions/action-card.ts | 31 ++++++++++++ test/visual/definitions/alert.ts | 11 +++++ test/visual/definitions/button.ts | 15 ------ test/visual/definitions/date-range-picker.ts | 40 --------------- test/visual/definitions/index.ts | 7 +-- test/visual/definitions/table.ts | 15 ------ test/visual/types.ts | 5 ++ 8 files changed, 93 insertions(+), 82 deletions(-) create mode 100644 test/visual/definitions/action-card.ts delete mode 100644 test/visual/definitions/button.ts delete mode 100644 test/visual/definitions/date-range-picker.ts delete mode 100644 test/visual/definitions/table.ts diff --git a/test/visual/compare-screenshots.ts b/test/visual/compare-screenshots.ts index 51254363ac..8847e10d21 100644 --- a/test/visual/compare-screenshots.ts +++ b/test/visual/compare-screenshots.ts @@ -3,6 +3,7 @@ import pixelmatch from 'pixelmatch'; import { PNG } from 'pngjs'; +import { parsePng } from '@cloudscape-design/browser-test-tools/image-utils'; import { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; import useBrowser from '@cloudscape-design/browser-test-tools/use-browser'; @@ -15,19 +16,52 @@ const defaultWindowSize = { width: 1600, height: 800 }; const newHost = process.env.NEW_HOST || 'http://localhost:8080'; const oldHost = process.env.OLD_HOST || 'http://localhost:8081'; +/** + * Captures the .screenshot-area element on a focused page. + * Uses a standard ScreenshotPageObject (no forced scroll-and-merge). + */ +async function captureScreenshotArea(browser: WebdriverIO.Browser, url: string): Promise { + await browser.url(url); + const page = new ScreenshotPageObject(browser); + await page.waitForVisible(screenshotAreaSelector); + const { image } = await page.captureBySelector(screenshotAreaSelector); + return image; +} + +/** + * Captures the full page as a PNG for permutation pages. + * Uses fullPageScreenshot which handles pages taller than the viewport. + */ +async function capturePermutations(browser: WebdriverIO.Browser, url: string): Promise { + await browser.url(url); + const page = new ScreenshotPageObject(browser); + await page.waitForVisible(screenshotAreaSelector); + const base64 = await page.fullPageScreenshot(); + return parsePng(base64); +} + async function captureScreenshot( browser: WebdriverIO.Browser, url: string, + testDef: TestDefinition, setup?: (page: ScreenshotPageObject) => Promise ): Promise { - await browser.url(url); - const page = new ScreenshotPageObject(browser); - await page.waitForVisible(screenshotAreaSelector); if (setup) { + await browser.url(url); + const page = new ScreenshotPageObject(browser); + await page.waitForVisible(screenshotAreaSelector); await setup(page); + if (testDef.screenshotType === 'permutations') { + const base64 = await page.fullPageScreenshot(); + return parsePng(base64); + } + const { image } = await page.captureBySelector(screenshotAreaSelector); + return image; } - const { image } = await page.captureBySelector(screenshotAreaSelector); - return image; + if (testDef.screenshotType === 'permutations') { + return capturePermutations(browser, url); + } + return captureScreenshotArea(browser, url); } function buildUrl(host: string, path: string, queryParams?: Record): string { @@ -65,12 +99,15 @@ function runSingleTest(testDef: TestDefinition) { testDef.description, useBrowser(windowSize, async browser => { const newUrl = buildUrl(newHost, testDef.path, testDef.queryParams); - const newScreenshot = await captureScreenshot(browser, newUrl, testDef.setup); + const newScreenshot = await captureScreenshot(browser, newUrl, testDef, testDef.setup); const oldUrl = buildUrl(oldHost, testDef.path, testDef.queryParams); - const oldScreenshot = await captureScreenshot(browser, oldUrl, testDef.setup); + const oldScreenshot = await captureScreenshot(browser, oldUrl, testDef, testDef.setup); const diffPixels = compareImages(newScreenshot, oldScreenshot); expect(diffPixels).toBe(0); }) ); } + +// Export the capture functions for use in custom setup callbacks if needed. +export { captureScreenshotArea, capturePermutations }; diff --git a/test/visual/definitions/action-card.ts b/test/visual/definitions/action-card.ts new file mode 100644 index 0000000000..8730ac9504 --- /dev/null +++ b/test/visual/definitions/action-card.ts @@ -0,0 +1,31 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'action-card', + tests: [ + { + description: 'permutations', + path: 'action-card/permutations', + screenshotType: 'permutations', + }, + { + description: 'variant permutations', + path: 'action-card/variant-permutations', + screenshotType: 'permutations', + }, + { + description: 'padding permutations', + path: 'action-card/padding-permutations', + screenshotType: 'permutations', + }, + { + description: 'simple', + path: 'action-card/simple', + screenshotType: 'screenshotArea', + }, + ], +}; + +export default suite; diff --git a/test/visual/definitions/alert.ts b/test/visual/definitions/alert.ts index 37f0985f24..a272e4dfee 100644 --- a/test/visual/definitions/alert.ts +++ b/test/visual/definitions/alert.ts @@ -8,6 +8,17 @@ const suite: TestSuite = { { description: 'permutations', path: 'alert/permutations', + screenshotType: 'permutations', + }, + { + description: 'simple', + path: 'alert/simple', + screenshotType: 'screenshotArea', + }, + { + description: 'custom types', + path: 'alert/style-custom-types', + screenshotType: 'screenshotArea', }, ], }; diff --git a/test/visual/definitions/button.ts b/test/visual/definitions/button.ts deleted file mode 100644 index cb7d590a59..0000000000 --- a/test/visual/definitions/button.ts +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { TestSuite } from '../types'; - -const suite: TestSuite = { - description: 'button', - tests: [ - { - description: 'permutations', - path: 'button/permutations', - }, - ], -}; - -export default suite; diff --git a/test/visual/definitions/date-range-picker.ts b/test/visual/definitions/date-range-picker.ts deleted file mode 100644 index 6800eebc90..0000000000 --- a/test/visual/definitions/date-range-picker.ts +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { TestSuite } from '../types'; - -const suite: TestSuite = { - description: 'date-range-picker', - tests: [ - { - description: 'with value', - path: 'date-range-picker/with-value', - }, - { - description: 'range calendar', - path: 'date-range-picker/range-calendar', - }, - { - description: 'month calendar permutations', - path: 'date-range-picker/month-calendar-permutations', - }, - { - description: 'year calendar permutations', - path: 'date-range-picker/year-calendar-permutations', - }, - { - description: 'in small viewport', - path: 'date-range-picker/small-viewport', - configuration: { width: 400 }, - }, - { - description: 'with dropdown open', - path: 'date-range-picker/with-value', - setup: async page => { - await page.click('[data-testid="date-range-picker-trigger"]'); - await page.waitForVisible('.awsui-context-content-header'); - }, - }, - ], -}; - -export default suite; diff --git a/test/visual/definitions/index.ts b/test/visual/definitions/index.ts index 27f5c9e7ca..55c1374550 100644 --- a/test/visual/definitions/index.ts +++ b/test/visual/definitions/index.ts @@ -4,9 +4,6 @@ // Each component has its own test definition file. // Import them here manually to form the full test suite. import { TestSuite } from '../types'; -import alert from './alert'; -import button from './button'; -import dateRangePicker from './date-range-picker'; -import table from './table'; +import actionCard from './action-card'; -export const allSuites: TestSuite[] = [alert, button, dateRangePicker, table]; +export const allSuites: TestSuite[] = [actionCard]; diff --git a/test/visual/definitions/table.ts b/test/visual/definitions/table.ts deleted file mode 100644 index 6529046d7b..0000000000 --- a/test/visual/definitions/table.ts +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { TestSuite } from '../types'; - -const suite: TestSuite = { - description: 'table', - tests: [ - { - description: 'permutations', - path: 'table/permutations', - }, - ], -}; - -export default suite; diff --git a/test/visual/types.ts b/test/visual/types.ts index 7c4c90809a..f0fa665863 100644 --- a/test/visual/types.ts +++ b/test/visual/types.ts @@ -9,9 +9,14 @@ export interface ScreenshotTestConfiguration { export type TestCallback = (page: ScreenshotPageObject) => Promise; +// 'screenshotArea' — captures the .screenshot-area element on a focused page. +// 'permutations' — captures the entire page and crops permutations out of it. +export type ScreenshotType = 'screenshotArea' | 'permutations'; + export interface TestDefinition { description: string; path: string; + screenshotType: ScreenshotType; queryParams?: Record; configuration?: ScreenshotTestConfiguration; setup?: TestCallback; From 3b11fe35ca5c7d04f9ba3321170c2d018586de02 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 13 May 2026 11:54:13 +0200 Subject: [PATCH 09/81] Reuse build --- .github/workflows/deploy.yml | 9 +++++ .github/workflows/visual-regression.yml | 45 ++++++++++++------------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5593984949..01bb5a45d2 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -65,3 +65,12 @@ jobs: with: artifact-name: dev-pages-react${{ matrix.react }} deployment-path: pages/lib/static-default + + visual: + name: Visual regression + needs: quick-build + if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + uses: ./.github/workflows/visual-regression.yml + secrets: inherit + with: + pr-artifact-name: dev-pages-react18 \ No newline at end of file diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 7ee7d42669..f358626055 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -1,9 +1,12 @@ name: Visual Regression Tests on: - pull_request: - branches: - - main + workflow_call: + inputs: + pr-artifact-name: + description: Name of the GitHub Actions artifact containing the PR's built dev pages + required: true + type: string defaults: run: @@ -33,27 +36,21 @@ jobs: with: chrome-version: stable - # ── Build PR (test) pages ────────────────────────────────────────────── - # Install the PR's dependencies and build its pages. - - name: Install PR dependencies + - name: Install dependencies run: npm i - - name: Build PR pages - run: npx gulp quick-build - env: - NODE_ENV: production - - - name: Bundle PR pages - run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-default - env: - NODE_ENV: production - - # ── Build baseline (main) pages ──────────────────────────────────────── - # Use a git worktree so the baseline has its own directory and its own - # node_modules. This means a PR that changes package-lock.json will still - # produce a correct baseline: the baseline installs from main's lockfile - # and the PR build installs from the PR's lockfile, so both sides use the - # dependency versions that are correct for their respective source trees. + # ── PR pages: reuse the artifact built by the build job ─────────────── + - name: Download PR pages artifact + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.pr-artifact-name }} + path: pages/lib/static-default + + # ── Baseline (main) pages: build from origin/main ───────────────────── + # GitHub Actions artifacts are scoped to a workflow run, so there is no + # built-in way to reuse a previous main-branch artifact without the API. + # We build main locally instead — it shares node_modules with the PR + # checkout, so both sides resolve the same dependency versions. - name: Create baseline worktree from origin/main run: git worktree add /tmp/baseline origin/main @@ -73,8 +70,8 @@ jobs: env: NODE_ENV: production - # ── Run tests ───────────────────────────────────────────────────────── - - name: Start test server (port 8080) + # ── Start both servers and run tests ────────────────────────────────── + - name: Start PR server (port 8080) run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8080 --static pages/lib/static-default --no-hot & env: NODE_ENV: development From d59cf2ed587c2401ce346c99c945e6f49000585e Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 13 May 2026 12:16:27 +0200 Subject: [PATCH 10/81] Fix wofklow deps --- .github/workflows/visual-regression.yml | 98 ------------------------- 1 file changed, 98 deletions(-) delete mode 100644 .github/workflows/visual-regression.yml diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml deleted file mode 100644 index f358626055..0000000000 --- a/.github/workflows/visual-regression.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: Visual Regression Tests - -on: - workflow_call: - inputs: - pr-artifact-name: - description: Name of the GitHub Actions artifact containing the PR's built dev pages - required: true - type: string - -defaults: - run: - shell: bash - -permissions: - id-token: write - contents: read - -jobs: - visual: - name: Visual regression - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: npm - - - name: Setup Chrome and ChromeDriver - uses: browser-actions/setup-chrome@v1 - with: - chrome-version: stable - - - name: Install dependencies - run: npm i - - # ── PR pages: reuse the artifact built by the build job ─────────────── - - name: Download PR pages artifact - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.pr-artifact-name }} - path: pages/lib/static-default - - # ── Baseline (main) pages: build from origin/main ───────────────────── - # GitHub Actions artifacts are scoped to a workflow run, so there is no - # built-in way to reuse a previous main-branch artifact without the API. - # We build main locally instead — it shares node_modules with the PR - # checkout, so both sides resolve the same dependency versions. - - name: Create baseline worktree from origin/main - run: git worktree add /tmp/baseline origin/main - - - name: Install baseline dependencies - run: npm i - working-directory: /tmp/baseline - - - name: Build baseline pages - run: npx gulp quick-build - working-directory: /tmp/baseline - env: - NODE_ENV: production - - - name: Bundle baseline pages - run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path ${{ github.workspace }}/pages/lib/static-visual-baseline - working-directory: /tmp/baseline - env: - NODE_ENV: production - - # ── Start both servers and run tests ────────────────────────────────── - - name: Start PR server (port 8080) - run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8080 --static pages/lib/static-default --no-hot & - env: - NODE_ENV: development - - - name: Start baseline server (port 8081) - run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8081 --static pages/lib/static-visual-baseline --no-hot & - env: - NODE_ENV: development - - - name: Wait for servers to be ready - run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081 - - - name: Run visual regression tests - run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js - env: - TZ: UTC - - - name: Upload diff artifacts - if: failure() - uses: actions/upload-artifact@v4 - with: - name: visual-regression-diffs - path: visual-regression-output/ - retention-days: 14 From f54fbc516deae8c85a2c79f84b5bb91384f5c1d4 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 13 May 2026 12:36:12 +0200 Subject: [PATCH 11/81] Again --- .github/workflows/visual-regression.yml | 120 ++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 .github/workflows/visual-regression.yml diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml new file mode 100644 index 0000000000..bbe372b7d6 --- /dev/null +++ b/.github/workflows/visual-regression.yml @@ -0,0 +1,120 @@ +name: Visual Regression Tests + +on: + workflow_run: + workflows: + - Build, lint and test + types: + - completed + +defaults: + run: + shell: bash + +permissions: + id-token: write + contents: read + actions: read + +jobs: + visual: + name: Visual regression + # Only run on PRs from the same repo (not forks), and only when the build succeeded. + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_repository.full_name == github.repository + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + # Check out the PR head commit, not the merge commit that workflow_run uses. + ref: ${{ github.event.workflow_run.head_sha }} + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Setup Chrome and ChromeDriver + uses: browser-actions/setup-chrome@v1 + with: + chrome-version: stable + + - name: Install dependencies + run: npm ci + + # ── PR pages: download the artifact produced by the build workflow ───── + - name: Download PR pages artifact + uses: actions/github-script@v7 + with: + script: | + const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + const artifact = artifacts.data.artifacts.find(a => a.name === 'dev-pages-react18'); + if (!artifact) throw new Error('dev-pages-react18 artifact not found'); + const download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact.id, + archive_format: 'zip', + }); + const fs = require('fs'); + fs.writeFileSync('/tmp/dev-pages.zip', Buffer.from(download.data)); + + - name: Extract PR pages artifact + run: | + mkdir -p pages/lib/static-default + unzip /tmp/dev-pages.zip -d pages/lib/static-default + + # ── Baseline (main) pages: build from origin/main ───────────────────── + - name: Create baseline worktree from origin/main + run: git worktree add /tmp/baseline origin/main + + - name: Install baseline dependencies + run: npm ci + working-directory: /tmp/baseline + + - name: Build baseline pages + run: npx gulp quick-build + working-directory: /tmp/baseline + env: + NODE_ENV: production + + - name: Bundle baseline pages + run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path ${{ github.workspace }}/pages/lib/static-visual-baseline + working-directory: /tmp/baseline + env: + NODE_ENV: production + + # ── Start both servers and run tests ────────────────────────────────── + - name: Start PR server (port 8080) + run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8080 --static pages/lib/static-default --no-hot & + env: + NODE_ENV: development + + - name: Start baseline server (port 8081) + run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8081 --static pages/lib/static-visual-baseline --no-hot & + env: + NODE_ENV: development + + - name: Wait for servers to be ready + run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081 + + - name: Run visual regression tests + run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js + env: + TZ: UTC + + - name: Upload diff artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: visual-regression-diffs + path: visual-regression-output/ + retention-days: 14 From 286c91394fb4b2eb8e622d4b087b23d38494b8e1 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 13 May 2026 12:50:50 +0200 Subject: [PATCH 12/81] Fix npm install command --- .github/workflows/visual-regression.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index bbe372b7d6..858a35a01e 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -44,7 +44,7 @@ jobs: chrome-version: stable - name: Install dependencies - run: npm ci + run: npm i # ── PR pages: download the artifact produced by the build workflow ───── - name: Download PR pages artifact @@ -77,7 +77,7 @@ jobs: run: git worktree add /tmp/baseline origin/main - name: Install baseline dependencies - run: npm ci + run: npm i working-directory: /tmp/baseline - name: Build baseline pages From 61d7f035bb4c8f668bad7c480dbda8ed8b05da67 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 13 May 2026 13:09:44 +0200 Subject: [PATCH 13/81] Wait only for the build --- .github/workflows/build-lint-test.yml | 14 ++++------- .github/workflows/build.yml | 31 +++++++++++++++++++++++++ .github/workflows/visual-regression.yml | 2 +- 3 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build-lint-test.yml b/.github/workflows/build-lint-test.yml index d9e7487d3d..553dbc4d02 100644 --- a/.github/workflows/build-lint-test.yml +++ b/.github/workflows/build-lint-test.yml @@ -1,15 +1,11 @@ name: Build, lint and test on: - pull_request: - branches: - - main - merge_group: - branches: - - main - push: - branches: - - main + workflow_run: + workflows: + - Build + types: + - completed permissions: id-token: write diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..9ddca205f7 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,31 @@ +name: Build + +on: + pull_request: + branches: + - main + merge_group: + branches: + - main + push: + branches: + - main + +permissions: + id-token: write + actions: read + contents: read + security-events: write + +jobs: + build: + name: build${{ matrix.react != 16 && format(' (React {0})', matrix.react) || '' }} + strategy: + matrix: + react: [16, 18] + uses: cloudscape-design/actions/.github/workflows/build-lint-test.yml@main + secrets: inherit + with: + artifact-path: pages/lib/static-default + artifact-name: dev-pages-react${{ matrix.react }} + react-version: ${{ matrix.react }} diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 858a35a01e..224e7bb94a 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -3,7 +3,7 @@ name: Visual Regression Tests on: workflow_run: workflows: - - Build, lint and test + - Build types: - completed From 9f7c5c5817ce507d9a4958841b14de2c052ed0db Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 13 May 2026 14:05:37 +0200 Subject: [PATCH 14/81] Revert "Wait only for the build" This reverts commit 23a62c420b23f83e90190060649e3b6895e84e50. --- .github/workflows/build-lint-test.yml | 14 +++++++---- .github/workflows/build.yml | 31 ------------------------- .github/workflows/visual-regression.yml | 2 +- 3 files changed, 10 insertions(+), 37 deletions(-) delete mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build-lint-test.yml b/.github/workflows/build-lint-test.yml index 553dbc4d02..d9e7487d3d 100644 --- a/.github/workflows/build-lint-test.yml +++ b/.github/workflows/build-lint-test.yml @@ -1,11 +1,15 @@ name: Build, lint and test on: - workflow_run: - workflows: - - Build - types: - - completed + pull_request: + branches: + - main + merge_group: + branches: + - main + push: + branches: + - main permissions: id-token: write diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 9ddca205f7..0000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Build - -on: - pull_request: - branches: - - main - merge_group: - branches: - - main - push: - branches: - - main - -permissions: - id-token: write - actions: read - contents: read - security-events: write - -jobs: - build: - name: build${{ matrix.react != 16 && format(' (React {0})', matrix.react) || '' }} - strategy: - matrix: - react: [16, 18] - uses: cloudscape-design/actions/.github/workflows/build-lint-test.yml@main - secrets: inherit - with: - artifact-path: pages/lib/static-default - artifact-name: dev-pages-react${{ matrix.react }} - react-version: ${{ matrix.react }} diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 224e7bb94a..858a35a01e 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -3,7 +3,7 @@ name: Visual Regression Tests on: workflow_run: workflows: - - Build + - Build, lint and test types: - completed From 043bf4eb1046e1a0a8d1bcb221c8e768361fcbfb Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 13 May 2026 14:05:44 +0200 Subject: [PATCH 15/81] Revert "Fix npm install command" This reverts commit ffb88c1feb2fa9076c3463d4800c921f05fa66d7. --- .github/workflows/visual-regression.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 858a35a01e..bbe372b7d6 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -44,7 +44,7 @@ jobs: chrome-version: stable - name: Install dependencies - run: npm i + run: npm ci # ── PR pages: download the artifact produced by the build workflow ───── - name: Download PR pages artifact @@ -77,7 +77,7 @@ jobs: run: git worktree add /tmp/baseline origin/main - name: Install baseline dependencies - run: npm i + run: npm ci working-directory: /tmp/baseline - name: Build baseline pages From d1c1a8b3c9304bda5182ba9cca1af12b2be135e6 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 13 May 2026 14:05:46 +0200 Subject: [PATCH 16/81] Revert "Again" This reverts commit 263a253c442ce54ea385252ef8291cce91839172. --- .github/workflows/visual-regression.yml | 120 ------------------------ 1 file changed, 120 deletions(-) delete mode 100644 .github/workflows/visual-regression.yml diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml deleted file mode 100644 index bbe372b7d6..0000000000 --- a/.github/workflows/visual-regression.yml +++ /dev/null @@ -1,120 +0,0 @@ -name: Visual Regression Tests - -on: - workflow_run: - workflows: - - Build, lint and test - types: - - completed - -defaults: - run: - shell: bash - -permissions: - id-token: write - contents: read - actions: read - -jobs: - visual: - name: Visual regression - # Only run on PRs from the same repo (not forks), and only when the build succeeded. - if: > - github.event.workflow_run.event == 'pull_request' && - github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.head_repository.full_name == github.repository - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - # Check out the PR head commit, not the merge commit that workflow_run uses. - ref: ${{ github.event.workflow_run.head_sha }} - fetch-depth: 0 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: npm - - - name: Setup Chrome and ChromeDriver - uses: browser-actions/setup-chrome@v1 - with: - chrome-version: stable - - - name: Install dependencies - run: npm ci - - # ── PR pages: download the artifact produced by the build workflow ───── - - name: Download PR pages artifact - uses: actions/github-script@v7 - with: - script: | - const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{ github.event.workflow_run.id }}, - }); - const artifact = artifacts.data.artifacts.find(a => a.name === 'dev-pages-react18'); - if (!artifact) throw new Error('dev-pages-react18 artifact not found'); - const download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: artifact.id, - archive_format: 'zip', - }); - const fs = require('fs'); - fs.writeFileSync('/tmp/dev-pages.zip', Buffer.from(download.data)); - - - name: Extract PR pages artifact - run: | - mkdir -p pages/lib/static-default - unzip /tmp/dev-pages.zip -d pages/lib/static-default - - # ── Baseline (main) pages: build from origin/main ───────────────────── - - name: Create baseline worktree from origin/main - run: git worktree add /tmp/baseline origin/main - - - name: Install baseline dependencies - run: npm ci - working-directory: /tmp/baseline - - - name: Build baseline pages - run: npx gulp quick-build - working-directory: /tmp/baseline - env: - NODE_ENV: production - - - name: Bundle baseline pages - run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path ${{ github.workspace }}/pages/lib/static-visual-baseline - working-directory: /tmp/baseline - env: - NODE_ENV: production - - # ── Start both servers and run tests ────────────────────────────────── - - name: Start PR server (port 8080) - run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8080 --static pages/lib/static-default --no-hot & - env: - NODE_ENV: development - - - name: Start baseline server (port 8081) - run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8081 --static pages/lib/static-visual-baseline --no-hot & - env: - NODE_ENV: development - - - name: Wait for servers to be ready - run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081 - - - name: Run visual regression tests - run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js - env: - TZ: UTC - - - name: Upload diff artifacts - if: failure() - uses: actions/upload-artifact@v4 - with: - name: visual-regression-diffs - path: visual-regression-output/ - retention-days: 14 From a6ef608a0364490d39245e303e0722343a649000 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 13 May 2026 14:05:52 +0200 Subject: [PATCH 17/81] Revert "Fix wofklow deps" This reverts commit b1eee0b5ba3596aa6e90c6be8e917edf7af898ba. --- .github/workflows/visual-regression.yml | 98 +++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 .github/workflows/visual-regression.yml diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml new file mode 100644 index 0000000000..f358626055 --- /dev/null +++ b/.github/workflows/visual-regression.yml @@ -0,0 +1,98 @@ +name: Visual Regression Tests + +on: + workflow_call: + inputs: + pr-artifact-name: + description: Name of the GitHub Actions artifact containing the PR's built dev pages + required: true + type: string + +defaults: + run: + shell: bash + +permissions: + id-token: write + contents: read + +jobs: + visual: + name: Visual regression + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Setup Chrome and ChromeDriver + uses: browser-actions/setup-chrome@v1 + with: + chrome-version: stable + + - name: Install dependencies + run: npm i + + # ── PR pages: reuse the artifact built by the build job ─────────────── + - name: Download PR pages artifact + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.pr-artifact-name }} + path: pages/lib/static-default + + # ── Baseline (main) pages: build from origin/main ───────────────────── + # GitHub Actions artifacts are scoped to a workflow run, so there is no + # built-in way to reuse a previous main-branch artifact without the API. + # We build main locally instead — it shares node_modules with the PR + # checkout, so both sides resolve the same dependency versions. + - name: Create baseline worktree from origin/main + run: git worktree add /tmp/baseline origin/main + + - name: Install baseline dependencies + run: npm i + working-directory: /tmp/baseline + + - name: Build baseline pages + run: npx gulp quick-build + working-directory: /tmp/baseline + env: + NODE_ENV: production + + - name: Bundle baseline pages + run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path ${{ github.workspace }}/pages/lib/static-visual-baseline + working-directory: /tmp/baseline + env: + NODE_ENV: production + + # ── Start both servers and run tests ────────────────────────────────── + - name: Start PR server (port 8080) + run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8080 --static pages/lib/static-default --no-hot & + env: + NODE_ENV: development + + - name: Start baseline server (port 8081) + run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8081 --static pages/lib/static-visual-baseline --no-hot & + env: + NODE_ENV: development + + - name: Wait for servers to be ready + run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081 + + - name: Run visual regression tests + run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js + env: + TZ: UTC + + - name: Upload diff artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: visual-regression-diffs + path: visual-regression-output/ + retention-days: 14 From f6661e74a6a86e1778ba11a5fe59e7c52e5bb7c1 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 13 May 2026 14:05:53 +0200 Subject: [PATCH 18/81] Revert "Reuse build" This reverts commit 1f71f10a69ef3b99627b3f8bec675dc7a2e4802f. --- .github/workflows/visual-regression.yml | 45 +++++++++++++------------ 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index f358626055..7ee7d42669 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -1,12 +1,9 @@ name: Visual Regression Tests on: - workflow_call: - inputs: - pr-artifact-name: - description: Name of the GitHub Actions artifact containing the PR's built dev pages - required: true - type: string + pull_request: + branches: + - main defaults: run: @@ -36,21 +33,27 @@ jobs: with: chrome-version: stable - - name: Install dependencies + # ── Build PR (test) pages ────────────────────────────────────────────── + # Install the PR's dependencies and build its pages. + - name: Install PR dependencies run: npm i - # ── PR pages: reuse the artifact built by the build job ─────────────── - - name: Download PR pages artifact - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.pr-artifact-name }} - path: pages/lib/static-default - - # ── Baseline (main) pages: build from origin/main ───────────────────── - # GitHub Actions artifacts are scoped to a workflow run, so there is no - # built-in way to reuse a previous main-branch artifact without the API. - # We build main locally instead — it shares node_modules with the PR - # checkout, so both sides resolve the same dependency versions. + - name: Build PR pages + run: npx gulp quick-build + env: + NODE_ENV: production + + - name: Bundle PR pages + run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-default + env: + NODE_ENV: production + + # ── Build baseline (main) pages ──────────────────────────────────────── + # Use a git worktree so the baseline has its own directory and its own + # node_modules. This means a PR that changes package-lock.json will still + # produce a correct baseline: the baseline installs from main's lockfile + # and the PR build installs from the PR's lockfile, so both sides use the + # dependency versions that are correct for their respective source trees. - name: Create baseline worktree from origin/main run: git worktree add /tmp/baseline origin/main @@ -70,8 +73,8 @@ jobs: env: NODE_ENV: production - # ── Start both servers and run tests ────────────────────────────────── - - name: Start PR server (port 8080) + # ── Run tests ───────────────────────────────────────────────────────── + - name: Start test server (port 8080) run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8080 --static pages/lib/static-default --no-hot & env: NODE_ENV: development From 824aeb36ac11ed55619f5cb3076a41b6c3c1f797 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 13 May 2026 15:20:37 +0200 Subject: [PATCH 19/81] Include alert tests --- test/visual/definitions/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/visual/definitions/index.ts b/test/visual/definitions/index.ts index 55c1374550..318ce7c68b 100644 --- a/test/visual/definitions/index.ts +++ b/test/visual/definitions/index.ts @@ -5,5 +5,6 @@ // Import them here manually to form the full test suite. import { TestSuite } from '../types'; import actionCard from './action-card'; +import alert from './alert'; -export const allSuites: TestSuite[] = [actionCard]; +export const allSuites: TestSuite[] = [actionCard, alert]; From 533cade9c3a4a8035f4e0660d3561b229f292478 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 13 May 2026 15:21:01 +0200 Subject: [PATCH 20/81] Add more alert tests --- test/visual/definitions/alert.ts | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/test/visual/definitions/alert.ts b/test/visual/definitions/alert.ts index a272e4dfee..30792c0d2d 100644 --- a/test/visual/definitions/alert.ts +++ b/test/visual/definitions/alert.ts @@ -1,25 +1,36 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 + import { TestSuite } from '../types'; const suite: TestSuite = { description: 'alert', tests: [ - { - description: 'permutations', - path: 'alert/permutations', - screenshotType: 'permutations', - }, { description: 'simple', path: 'alert/simple', screenshotType: 'screenshotArea', }, { - description: 'custom types', + description: 'style custom page', path: 'alert/style-custom-types', screenshotType: 'screenshotArea', }, + ...[600, 1280].map(width => ({ + description: `width ${width}px`, + tests: [ + { + description: 'permutations', + path: 'alert/permutations', + screenshotType: 'permutations' as const, + }, + { + description: 'custom types', + path: 'alert/style-custom-types', + screenshotType: 'screenshotArea' as const, + }, + ], + })), ], }; From 57086c62443b5111ad0de46f48042c60fb816ce0 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 13 May 2026 15:46:26 +0200 Subject: [PATCH 21/81] chore: Export visual test definitions --- build-tools/tasks/index.js | 5 +++++ build-tools/tasks/visual-definitions.js | 8 ++++++++ gulpfile.js | 2 ++ test/visual/types.ts | 2 +- tsconfig.visual-definitions.json | 19 +++++++++++++++++++ 5 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 build-tools/tasks/visual-definitions.js create mode 100644 tsconfig.visual-definitions.json diff --git a/build-tools/tasks/index.js b/build-tools/tasks/index.js index d7db3c0784..413695be14 100644 --- a/build-tools/tasks/index.js +++ b/build-tools/tasks/index.js @@ -21,5 +21,10 @@ module.exports = { themeableSource: require('./themeable-source'), bundleVendorFiles: require('./bundle-vendor-files'), sizeLimit: require('./size-limit'), +<<<<<<< HEAD testDefinitions: require('./test-definitions'), +======= + visual: require('./visual'), + visualDefinitions: require('./visual-definitions'), +>>>>>>> 4213557f5 (chore: Export visual test definitions) }; diff --git a/build-tools/tasks/visual-definitions.js b/build-tools/tasks/visual-definitions.js new file mode 100644 index 0000000000..34dfe16adf --- /dev/null +++ b/build-tools/tasks/visual-definitions.js @@ -0,0 +1,8 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +const execa = require('execa'); +const { task } = require('../utils/gulp-utils'); + +module.exports = task('visual-definitions', () => + execa('tsc', ['-p', 'tsconfig.visual-definitions.json'], { stdio: 'inherit' }) +); diff --git a/gulpfile.js b/gulpfile.js index 221c065982..09e0833868 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -20,6 +20,7 @@ const { integ, motion, visual, + visualDefinitions, copyFiles, themeableSource, bundleVendorFiles, @@ -45,6 +46,7 @@ exports['test:integ'] = integ; exports['test:a11y'] = a11y; exports['test:motion'] = motion; exports['test:visual'] = visual; +exports['build:visual-definitions'] = visualDefinitions; exports.watch = () => { watch( diff --git a/test/visual/types.ts b/test/visual/types.ts index f0fa665863..c4c23b622f 100644 --- a/test/visual/types.ts +++ b/test/visual/types.ts @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; +import type { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; export interface ScreenshotTestConfiguration { width?: number; diff --git a/tsconfig.visual-definitions.json b/tsconfig.visual-definitions.json new file mode 100644 index 0000000000..103f77ee47 --- /dev/null +++ b/tsconfig.visual-definitions.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "lib": ["ES2021"], + "target": "ES2019", + "types": [], + "module": "CommonJS", + "moduleResolution": "node", + "esModuleInterop": true, + "strict": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "inlineSources": true, + "rootDir": "test/visual", + "outDir": "lib/visual-test-definitions" + }, + "include": ["test/visual/definitions", "test/visual/types.ts"], + "exclude": [] +} From aa01ece186645382f509a751030cdc312bc6b044 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 18 May 2026 15:17:49 +0200 Subject: [PATCH 22/81] Use the quick build --- .github/workflows/visual-regression.yml | 31 ++++++++++++++++--------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 7ee7d42669..35ae55ba9b 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -13,6 +13,11 @@ permissions: id-token: write contents: read +inputs: + pr-artifact-name: + description: 'Name of the artifact containing PR pages (built by quick-build job). If not provided, pages will be built locally.' + required: false + jobs: visual: name: Visual regression @@ -33,20 +38,24 @@ jobs: with: chrome-version: stable - # ── Build PR (test) pages ────────────────────────────────────────────── - # Install the PR's dependencies and build its pages. - - name: Install PR dependencies - run: npm i - - - name: Build PR pages - run: npx gulp quick-build + # ── Build or download PR (test) pages ───────────────────────────────── + # When called from deploy.yml with pr-artifact-name, download the artifact. + # When run standalone (no pr-artifact-name provided), build the pages locally. + - name: Build PR pages locally + if: ${{ !inputs.pr-artifact-name }} + run: | + npm i + npx gulp quick-build + node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-default env: NODE_ENV: production - - name: Bundle PR pages - run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-default - env: - NODE_ENV: production + - name: Download PR pages artifact + if: ${{ inputs.pr-artifact-name }} + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.pr-artifact-name }} + path: pages/lib/static-default # ── Build baseline (main) pages ──────────────────────────────────────── # Use a git worktree so the baseline has its own directory and its own From b6767f634d0b9e9c677b5d180fb4e4ad47f57cc1 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 18 May 2026 18:16:20 +0200 Subject: [PATCH 23/81] Fix tsconfig --- tsconfig.visual-definitions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.visual-definitions.json b/tsconfig.visual-definitions.json index 103f77ee47..a31ccd02d3 100644 --- a/tsconfig.visual-definitions.json +++ b/tsconfig.visual-definitions.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "lib": ["ES2021"], + "lib": ["ES2021", "DOM"], "target": "ES2019", "types": [], "module": "CommonJS", From d7182b239c28fe031ad9cec15e3888bad64e02ae Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 18 May 2026 18:20:07 +0200 Subject: [PATCH 24/81] Fix workflow --- .github/workflows/visual-regression.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 35ae55ba9b..c6d1bf031a 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -4,6 +4,12 @@ on: pull_request: branches: - main + workflow_call: + inputs: + pr-artifact-name: + description: 'Name of the artifact containing PR pages (built by quick-build job). If not provided, pages will be built locally.' + required: false + type: string defaults: run: @@ -13,11 +19,6 @@ permissions: id-token: write contents: read -inputs: - pr-artifact-name: - description: 'Name of the artifact containing PR pages (built by quick-build job). If not provided, pages will be built locally.' - required: false - jobs: visual: name: Visual regression From 6324d9a5d2e7461d49fdc56b583d53ea1a711652 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 18 May 2026 18:41:58 +0200 Subject: [PATCH 25/81] Fix workflow --- .github/workflows/visual-regression.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index c6d1bf031a..d3b36a4ac9 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -39,13 +39,15 @@ jobs: with: chrome-version: stable + - name: Install dependencies + run: npm i + # ── Build or download PR (test) pages ───────────────────────────────── # When called from deploy.yml with pr-artifact-name, download the artifact. # When run standalone (no pr-artifact-name provided), build the pages locally. - name: Build PR pages locally if: ${{ !inputs.pr-artifact-name }} run: | - npm i npx gulp quick-build node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-default env: From 49325de9893dcc14f14d2af3effc3f42fdcba207 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 18 May 2026 19:01:10 +0200 Subject: [PATCH 26/81] Use serve --- .github/workflows/visual-regression.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index d3b36a4ac9..742cc72f68 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -87,12 +87,12 @@ jobs: # ── Run tests ───────────────────────────────────────────────────────── - name: Start test server (port 8080) - run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8080 --static pages/lib/static-default --no-hot & + run: npx serve --no-clipboard --listen 8080 pages/lib/static-default & env: NODE_ENV: development - name: Start baseline server (port 8081) - run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8081 --static pages/lib/static-visual-baseline --no-hot & + run: npx serve --no-clipboard --listen 8081 pages/lib/static-visual-baseline & env: NODE_ENV: development From d5b8d6cec8eb010da58988f6c832091d0182f664 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 18 May 2026 19:10:56 +0200 Subject: [PATCH 27/81] Run tests on Safari --- .github/workflows/visual-regression.yml | 27 ++++++++++++++++--------- build-tools/visual/global-setup.js | 20 ++++++++++++++++-- build-tools/visual/global-teardown.js | 12 +++++++++-- build-tools/visual/setup.js | 6 ++++-- 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 742cc72f68..39cea38599 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -21,8 +21,16 @@ permissions: jobs: visual: - name: Visual regression - runs-on: ubuntu-latest + name: Visual regression (${{ matrix.browser }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - browser: chrome + os: ubuntu-latest + - browser: safari + os: macos-latest steps: - uses: actions/checkout@v4 with: @@ -35,16 +43,19 @@ jobs: cache: npm - name: Setup Chrome and ChromeDriver + if: matrix.browser == 'chrome' uses: browser-actions/setup-chrome@v1 with: chrome-version: stable + - name: Enable SafariDriver + if: matrix.browser == 'safari' + run: sudo safaridriver --enable + - name: Install dependencies run: npm i # ── Build or download PR (test) pages ───────────────────────────────── - # When called from deploy.yml with pr-artifact-name, download the artifact. - # When run standalone (no pr-artifact-name provided), build the pages locally. - name: Build PR pages locally if: ${{ !inputs.pr-artifact-name }} run: | @@ -61,11 +72,6 @@ jobs: path: pages/lib/static-default # ── Build baseline (main) pages ──────────────────────────────────────── - # Use a git worktree so the baseline has its own directory and its own - # node_modules. This means a PR that changes package-lock.json will still - # produce a correct baseline: the baseline installs from main's lockfile - # and the PR build installs from the PR's lockfile, so both sides use the - # dependency versions that are correct for their respective source trees. - name: Create baseline worktree from origin/main run: git worktree add /tmp/baseline origin/main @@ -103,11 +109,12 @@ jobs: run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js env: TZ: UTC + BROWSER: ${{ matrix.browser }} - name: Upload diff artifacts if: failure() uses: actions/upload-artifact@v4 with: - name: visual-regression-diffs + name: visual-regression-diffs-${{ matrix.browser }} path: visual-regression-output/ retention-days: 14 diff --git a/build-tools/visual/global-setup.js b/build-tools/visual/global-setup.js index 075ee6e398..f27f7d04cb 100644 --- a/build-tools/visual/global-setup.js +++ b/build-tools/visual/global-setup.js @@ -1,4 +1,20 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -const { startWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); -module.exports = () => startWebdriver(); +const { spawn } = require('child_process'); +const waitOn = require('wait-on'); + +let driverProcess; + +module.exports = async () => { + if (process.env.BROWSER === 'safari') { + driverProcess = spawn('safaridriver', ['--port', '4444']); + driverProcess.on('error', err => { + throw err; + }); + await waitOn({ resources: ['http-get://localhost:4444/status'], timeout: 10000 }); + } else { + const { startWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); + await startWebdriver(); + } + global.__DRIVER_PROCESS__ = driverProcess; +}; diff --git a/build-tools/visual/global-teardown.js b/build-tools/visual/global-teardown.js index 57ad21b454..366c3f7660 100644 --- a/build-tools/visual/global-teardown.js +++ b/build-tools/visual/global-teardown.js @@ -1,4 +1,12 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -const { shutdownWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); -module.exports = () => shutdownWebdriver(); +module.exports = () => { + if (process.env.BROWSER === 'safari') { + if (global.__DRIVER_PROCESS__) { + global.__DRIVER_PROCESS__.kill(); + } + } else { + const { shutdownWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); + shutdownWebdriver(); + } +}; diff --git a/build-tools/visual/setup.js b/build-tools/visual/setup.js index 2625a43809..79050cab6a 100644 --- a/build-tools/visual/setup.js +++ b/build-tools/visual/setup.js @@ -3,12 +3,14 @@ /* global jest */ const { configure } = require('@cloudscape-design/browser-test-tools/use-browser'); +const isSafari = process.env.BROWSER === 'safari'; + // The PR build (the code under test) is served on port 8080. // The baseline build (main branch, same node_modules) is served on port 8081. configure({ - browserName: 'ChromeHeadlessIntegration', + browserName: isSafari ? 'Safari' : 'ChromeHeadlessIntegration', browserCreatorOptions: { - seleniumUrl: 'http://localhost:9515', + seleniumUrl: isSafari ? 'http://localhost:4444' : 'http://localhost:9515', }, webdriverOptions: { baseUrl: 'http://localhost:8080', From c158a1d6ecd700e489c636837c832651aab3bb2f Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 18 May 2026 19:27:05 +0200 Subject: [PATCH 28/81] Prevent visual regression workflow from running twice --- .github/workflows/visual-regression.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 39cea38599..09685d9bf5 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -1,9 +1,6 @@ name: Visual Regression Tests on: - pull_request: - branches: - - main workflow_call: inputs: pr-artifact-name: From 45e188f4fdc73073241d508941e298e75e542713 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 18 May 2026 19:45:39 +0200 Subject: [PATCH 29/81] Reuse baseline build across browsers --- .github/workflows/visual-regression.yml | 99 ++++++++++++++++++------- 1 file changed, 73 insertions(+), 26 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 09685d9bf5..2c9e633b05 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -17,17 +17,10 @@ permissions: contents: read jobs: - visual: - name: Visual regression (${{ matrix.browser }}) - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - browser: chrome - os: ubuntu-latest - - browser: safari - os: macos-latest + # Build the baseline (main branch) pages once and share them across all browser jobs. + build-baseline: + name: Build baseline pages + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: @@ -39,20 +32,12 @@ jobs: node-version: 20 cache: npm - - name: Setup Chrome and ChromeDriver - if: matrix.browser == 'chrome' - uses: browser-actions/setup-chrome@v1 - with: - chrome-version: stable - - - name: Enable SafariDriver - if: matrix.browser == 'safari' - run: sudo safaridriver --enable - - name: Install dependencies run: npm i # ── Build or download PR (test) pages ───────────────────────────────── + # When called from deploy.yml with pr-artifact-name, download the artifact. + # When run standalone (no pr-artifact-name provided), build the pages locally. - name: Build PR pages locally if: ${{ !inputs.pr-artifact-name }} run: | @@ -69,6 +54,11 @@ jobs: path: pages/lib/static-default # ── Build baseline (main) pages ──────────────────────────────────────── + # Use a git worktree so the baseline has its own directory and its own + # node_modules. This means a PR that changes package-lock.json will still + # produce a correct baseline: the baseline installs from main's lockfile + # and the PR build installs from the PR's lockfile, so both sides use the + # dependency versions that are correct for their respective source trees. - name: Create baseline worktree from origin/main run: git worktree add /tmp/baseline origin/main @@ -83,21 +73,78 @@ jobs: NODE_ENV: production - name: Bundle baseline pages - run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path ${{ github.workspace }}/pages/lib/static-visual-baseline + run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-visual-baseline working-directory: /tmp/baseline env: NODE_ENV: production + - name: Upload baseline artifact + uses: actions/upload-artifact@v4 + with: + name: visual-baseline-pages + path: pages/lib/static-visual-baseline + retention-days: 1 + + - name: Upload PR pages artifact + if: ${{ !inputs.pr-artifact-name }} + uses: actions/upload-artifact@v4 + with: + name: visual-pr-pages + path: pages/lib/static-default + retention-days: 1 + + visual: + name: Visual regression (${{ matrix.browser }}) + needs: build-baseline + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - browser: chrome + os: ubuntu-latest + - browser: safari + os: macos-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Setup Chrome and ChromeDriver + if: matrix.browser == 'chrome' + uses: browser-actions/setup-chrome@v1 + with: + chrome-version: stable + + - name: Enable SafariDriver + if: matrix.browser == 'safari' + run: sudo safaridriver --enable + + - name: Install dependencies + run: npm i + + - name: Download PR pages artifact + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.pr-artifact-name || 'visual-pr-pages' }} + path: pages/lib/static-default + + - name: Download baseline artifact + uses: actions/download-artifact@v4 + with: + name: visual-baseline-pages + path: pages/lib/static-visual-baseline + # ── Run tests ───────────────────────────────────────────────────────── - name: Start test server (port 8080) run: npx serve --no-clipboard --listen 8080 pages/lib/static-default & - env: - NODE_ENV: development - name: Start baseline server (port 8081) run: npx serve --no-clipboard --listen 8081 pages/lib/static-visual-baseline & - env: - NODE_ENV: development - name: Wait for servers to be ready run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081 From 5c052621272d10b97166e9698460667b37d421c7 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 18 May 2026 19:46:53 +0200 Subject: [PATCH 30/81] Limit Safari concurrency --- jest.visual.config.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jest.visual.config.js b/jest.visual.config.js index 7f1d020880..0d3abc58fd 100644 --- a/jest.visual.config.js +++ b/jest.visual.config.js @@ -16,7 +16,9 @@ module.exports = { }, reporters: ['default', 'github-actions'], testTimeout: 120_000, // 2min — pages can be tall and slow to capture - maxWorkers: os.cpus().length * (process.env.GITHUB_ACTION ? 3 : 1), + // Safari's WebDriver only supports one concurrent session, so tests must run serially. + // Chrome can run multiple workers to speed things up. + maxWorkers: process.env.BROWSER === 'safari' ? 1 : os.cpus().length * (process.env.GITHUB_ACTION ? 3 : 1), globalSetup: '/build-tools/visual/global-setup.js', globalTeardown: '/build-tools/visual/global-teardown.js', setupFilesAfterEnv: [path.join(__dirname, 'build-tools', 'visual', 'setup.js')], From d1103c6a648f2532f755f33f5a447a5a1c7c87dc Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 18 May 2026 20:05:39 +0200 Subject: [PATCH 31/81] Fix workflow --- .github/workflows/deploy.yml | 3 ++- .github/workflows/visual-regression.yml | 34 +++++++++++++++---------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 01bb5a45d2..8705c33048 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -73,4 +73,5 @@ jobs: uses: ./.github/workflows/visual-regression.yml secrets: inherit with: - pr-artifact-name: dev-pages-react18 \ No newline at end of file + pr-artifact-name: dev-pages-react18 + caller-run-id: ${{ github.run_id }} \ No newline at end of file diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 2c9e633b05..bedbb5435e 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -7,6 +7,10 @@ on: description: 'Name of the artifact containing PR pages (built by quick-build job). If not provided, pages will be built locally.' required: false type: string + caller-run-id: + description: 'The run ID of the calling workflow, used to download artifacts it uploaded.' + required: false + type: string defaults: run: @@ -15,9 +19,11 @@ defaults: permissions: id-token: write contents: read + actions: read jobs: # Build the baseline (main branch) pages once and share them across all browser jobs. + # Also stages the PR pages artifact within this run so all jobs use the same run context. build-baseline: name: Build baseline pages runs-on: ubuntu-latest @@ -35,9 +41,10 @@ jobs: - name: Install dependencies run: npm i - # ── Build or download PR (test) pages ───────────────────────────────── - # When called from deploy.yml with pr-artifact-name, download the artifact. - # When run standalone (no pr-artifact-name provided), build the pages locally. + # ── Stage PR (test) pages ────────────────────────────────────────────── + # Download from the caller's run (deploy.yml) or build locally. + # Either way, re-upload as 'visual-pr-pages' within this run so the + # matrix jobs can download it without needing cross-run artifact access. - name: Build PR pages locally if: ${{ !inputs.pr-artifact-name }} run: | @@ -46,12 +53,21 @@ jobs: env: NODE_ENV: production - - name: Download PR pages artifact + - name: Download PR pages artifact from caller run if: ${{ inputs.pr-artifact-name }} uses: actions/download-artifact@v4 with: name: ${{ inputs.pr-artifact-name }} path: pages/lib/static-default + github-token: ${{ github.token }} + run-id: ${{ inputs.caller-run-id }} + + - name: Upload PR pages artifact (for matrix jobs) + uses: actions/upload-artifact@v4 + with: + name: visual-pr-pages + path: pages/lib/static-default + retention-days: 1 # ── Build baseline (main) pages ──────────────────────────────────────── # Use a git worktree so the baseline has its own directory and its own @@ -85,14 +101,6 @@ jobs: path: pages/lib/static-visual-baseline retention-days: 1 - - name: Upload PR pages artifact - if: ${{ !inputs.pr-artifact-name }} - uses: actions/upload-artifact@v4 - with: - name: visual-pr-pages - path: pages/lib/static-default - retention-days: 1 - visual: name: Visual regression (${{ matrix.browser }}) needs: build-baseline @@ -130,7 +138,7 @@ jobs: - name: Download PR pages artifact uses: actions/download-artifact@v4 with: - name: ${{ inputs.pr-artifact-name || 'visual-pr-pages' }} + name: visual-pr-pages path: pages/lib/static-default - name: Download baseline artifact From 7ba5aca6e7adff76578d1cc1f078f6ea1bf6cc44 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 18 May 2026 20:27:19 +0200 Subject: [PATCH 32/81] Fix workflows --- .github/workflows/deploy.yml | 57 +++++++++++++++++++++++- .github/workflows/visual-regression.yml | 59 ++++++++++--------------- 2 files changed, 78 insertions(+), 38 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8705c33048..8a3b4459b7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,6 +53,58 @@ jobs: name: dev-pages-react${{ matrix.react }} path: pages/lib/static-default + # Build the baseline (main branch) pages in parallel with quick-build. + # The result is passed to the visual regression workflow so it doesn't + # need to rebuild the baseline for each browser. + build-baseline: + name: Build baseline pages + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Install dependencies + run: npm i + + # Use a git worktree so the baseline has its own directory and its own + # node_modules. This means a PR that changes package-lock.json will still + # produce a correct baseline: the baseline installs from main's lockfile + # and the PR build installs from the PR's lockfile, so both sides use the + # dependency versions that are correct for their respective source trees. + - name: Create baseline worktree from origin/main + run: git worktree add /tmp/baseline origin/main + + - name: Install baseline dependencies + run: npm i + working-directory: /tmp/baseline + + - name: Build baseline pages + run: npx gulp quick-build + working-directory: /tmp/baseline + env: + NODE_ENV: production + + - name: Bundle baseline pages + run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-visual-baseline + working-directory: /tmp/baseline + env: + NODE_ENV: production + + - name: Upload baseline artifact + uses: actions/upload-artifact@v4 + with: + name: visual-baseline-pages + path: pages/lib/static-visual-baseline + retention-days: 1 + deploy: needs: quick-build name: deploy${{ matrix.react != 16 && format(' (React {0})', matrix.react) || '' }} @@ -68,10 +120,11 @@ jobs: visual: name: Visual regression - needs: quick-build + needs: [quick-build, build-baseline] if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} uses: ./.github/workflows/visual-regression.yml secrets: inherit with: pr-artifact-name: dev-pages-react18 - caller-run-id: ${{ github.run_id }} \ No newline at end of file + baseline-artifact-name: visual-baseline-pages + caller-run-id: ${{ github.run_id }} diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index bedbb5435e..5fab2453e2 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -7,6 +7,10 @@ on: description: 'Name of the artifact containing PR pages (built by quick-build job). If not provided, pages will be built locally.' required: false type: string + baseline-artifact-name: + description: 'Name of the artifact containing baseline pages (built by build-baseline job in the caller workflow).' + required: true + type: string caller-run-id: description: 'The run ID of the calling workflow, used to download artifacts it uploaded.' required: false @@ -22,15 +26,13 @@ permissions: actions: read jobs: - # Build the baseline (main branch) pages once and share them across all browser jobs. - # Also stages the PR pages artifact within this run so all jobs use the same run context. - build-baseline: - name: Build baseline pages + # Stage the PR pages within this run so matrix jobs can download them without + # needing cross-run artifact access. Runs in parallel with stage-baseline. + stage-pr-pages: + name: Stage PR pages runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - name: Setup Node.js uses: actions/setup-node@v4 @@ -41,10 +43,6 @@ jobs: - name: Install dependencies run: npm i - # ── Stage PR (test) pages ────────────────────────────────────────────── - # Download from the caller's run (deploy.yml) or build locally. - # Either way, re-upload as 'visual-pr-pages' within this run so the - # matrix jobs can download it without needing cross-run artifact access. - name: Build PR pages locally if: ${{ !inputs.pr-artifact-name }} run: | @@ -69,32 +67,21 @@ jobs: path: pages/lib/static-default retention-days: 1 - # ── Build baseline (main) pages ──────────────────────────────────────── - # Use a git worktree so the baseline has its own directory and its own - # node_modules. This means a PR that changes package-lock.json will still - # produce a correct baseline: the baseline installs from main's lockfile - # and the PR build installs from the PR's lockfile, so both sides use the - # dependency versions that are correct for their respective source trees. - - name: Create baseline worktree from origin/main - run: git worktree add /tmp/baseline origin/main - - - name: Install baseline dependencies - run: npm i - working-directory: /tmp/baseline - - - name: Build baseline pages - run: npx gulp quick-build - working-directory: /tmp/baseline - env: - NODE_ENV: production - - - name: Bundle baseline pages - run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-visual-baseline - working-directory: /tmp/baseline - env: - NODE_ENV: production + # Stage the baseline pages within this run so matrix jobs can download them + # without needing cross-run artifact access. Runs in parallel with stage-pr-pages. + stage-baseline: + name: Stage baseline pages + runs-on: ubuntu-latest + steps: + - name: Download baseline artifact from caller run + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.baseline-artifact-name }} + path: pages/lib/static-visual-baseline + github-token: ${{ github.token }} + run-id: ${{ inputs.caller-run-id }} - - name: Upload baseline artifact + - name: Upload baseline artifact (for matrix jobs) uses: actions/upload-artifact@v4 with: name: visual-baseline-pages @@ -103,7 +90,7 @@ jobs: visual: name: Visual regression (${{ matrix.browser }}) - needs: build-baseline + needs: [stage-pr-pages, stage-baseline] runs-on: ${{ matrix.os }} strategy: fail-fast: false From f3744e8d5fee33b055a16816265595cb8a1ce632 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 18 May 2026 21:22:00 +0200 Subject: [PATCH 33/81] Fix workflow --- .github/workflows/visual-regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 5fab2453e2..9b68239196 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -14,7 +14,7 @@ on: caller-run-id: description: 'The run ID of the calling workflow, used to download artifacts it uploaded.' required: false - type: string + type: number defaults: run: From ea3ec1a0ee86284d5fa615b1839cc49cc37f4caa Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 18 May 2026 21:41:50 +0200 Subject: [PATCH 34/81] Fix workflow --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8a3b4459b7..45f7ac8fbb 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -121,7 +121,7 @@ jobs: visual: name: Visual regression needs: [quick-build, build-baseline] - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} uses: ./.github/workflows/visual-regression.yml secrets: inherit with: From 034dc8785e5b37c9d9d6a5d35560eaf28d9ac9cf Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Mon, 18 May 2026 21:47:43 +0200 Subject: [PATCH 35/81] Fix workflow --- .github/workflows/visual-regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 9b68239196..5fab2453e2 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -14,7 +14,7 @@ on: caller-run-id: description: 'The run ID of the calling workflow, used to download artifacts it uploaded.' required: false - type: number + type: string defaults: run: From cdf087dfc61c39b16809152377e6ee9342b0e129 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Tue, 19 May 2026 01:56:14 +0200 Subject: [PATCH 36/81] Fix workflow --- .github/workflows/deploy.yml | 55 +---------------------- .github/workflows/visual-regression.yml | 59 ++++++++++++++++++------- 2 files changed, 43 insertions(+), 71 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 45f7ac8fbb..2add3494cd 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,58 +53,6 @@ jobs: name: dev-pages-react${{ matrix.react }} path: pages/lib/static-default - # Build the baseline (main branch) pages in parallel with quick-build. - # The result is passed to the visual regression workflow so it doesn't - # need to rebuild the baseline for each browser. - build-baseline: - name: Build baseline pages - if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: npm - - - name: Install dependencies - run: npm i - - # Use a git worktree so the baseline has its own directory and its own - # node_modules. This means a PR that changes package-lock.json will still - # produce a correct baseline: the baseline installs from main's lockfile - # and the PR build installs from the PR's lockfile, so both sides use the - # dependency versions that are correct for their respective source trees. - - name: Create baseline worktree from origin/main - run: git worktree add /tmp/baseline origin/main - - - name: Install baseline dependencies - run: npm i - working-directory: /tmp/baseline - - - name: Build baseline pages - run: npx gulp quick-build - working-directory: /tmp/baseline - env: - NODE_ENV: production - - - name: Bundle baseline pages - run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-visual-baseline - working-directory: /tmp/baseline - env: - NODE_ENV: production - - - name: Upload baseline artifact - uses: actions/upload-artifact@v4 - with: - name: visual-baseline-pages - path: pages/lib/static-visual-baseline - retention-days: 1 - deploy: needs: quick-build name: deploy${{ matrix.react != 16 && format(' (React {0})', matrix.react) || '' }} @@ -120,11 +68,10 @@ jobs: visual: name: Visual regression - needs: [quick-build, build-baseline] + needs: quick-build if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} uses: ./.github/workflows/visual-regression.yml secrets: inherit with: pr-artifact-name: dev-pages-react18 - baseline-artifact-name: visual-baseline-pages caller-run-id: ${{ github.run_id }} diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 5fab2453e2..8c3c44c0ff 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -7,10 +7,6 @@ on: description: 'Name of the artifact containing PR pages (built by quick-build job). If not provided, pages will be built locally.' required: false type: string - baseline-artifact-name: - description: 'Name of the artifact containing baseline pages (built by build-baseline job in the caller workflow).' - required: true - type: string caller-run-id: description: 'The run ID of the calling workflow, used to download artifacts it uploaded.' required: false @@ -27,7 +23,7 @@ permissions: jobs: # Stage the PR pages within this run so matrix jobs can download them without - # needing cross-run artifact access. Runs in parallel with stage-baseline. + # needing cross-run artifact access. Runs in parallel with build-baseline. stage-pr-pages: name: Stage PR pages runs-on: ubuntu-latest @@ -67,21 +63,50 @@ jobs: path: pages/lib/static-default retention-days: 1 - # Stage the baseline pages within this run so matrix jobs can download them - # without needing cross-run artifact access. Runs in parallel with stage-pr-pages. - stage-baseline: - name: Stage baseline pages + # Build the baseline (main branch) pages once and share them across all browser jobs. + # Runs in parallel with stage-pr-pages. + build-baseline: + name: Build baseline pages runs-on: ubuntu-latest steps: - - name: Download baseline artifact from caller run - uses: actions/download-artifact@v4 + - uses: actions/checkout@v4 with: - name: ${{ inputs.baseline-artifact-name }} - path: pages/lib/static-visual-baseline - github-token: ${{ github.token }} - run-id: ${{ inputs.caller-run-id }} + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Install dependencies + run: npm i + + # Use a git worktree so the baseline has its own directory and its own + # node_modules. This means a PR that changes package-lock.json will still + # produce a correct baseline: the baseline installs from main's lockfile + # and the PR build installs from the PR's lockfile, so both sides use the + # dependency versions that are correct for their respective source trees. + - name: Create baseline worktree from origin/main + run: git worktree add /tmp/baseline origin/main + + - name: Install baseline dependencies + run: npm i + working-directory: /tmp/baseline + + - name: Build baseline pages + run: npx gulp quick-build + working-directory: /tmp/baseline + env: + NODE_ENV: production + + - name: Bundle baseline pages + run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-visual-baseline + working-directory: /tmp/baseline + env: + NODE_ENV: production - - name: Upload baseline artifact (for matrix jobs) + - name: Upload baseline artifact uses: actions/upload-artifact@v4 with: name: visual-baseline-pages @@ -90,7 +115,7 @@ jobs: visual: name: Visual regression (${{ matrix.browser }}) - needs: [stage-pr-pages, stage-baseline] + needs: [stage-pr-pages, build-baseline] runs-on: ${{ matrix.os }} strategy: fail-fast: false From 5230a6d41d055b5d4bf876b5401d6d09b8f1a638 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Tue, 19 May 2026 02:07:06 +0200 Subject: [PATCH 37/81] Fix workflow --- .github/workflows/visual-regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 8c3c44c0ff..4f62e0fa16 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -101,7 +101,7 @@ jobs: NODE_ENV: production - name: Bundle baseline pages - run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-visual-baseline + run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path ${{ github.workspace }}/pages/lib/static-visual-baseline working-directory: /tmp/baseline env: NODE_ENV: production From 11ec0be0bd21d32886732125820358117310222c Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Tue, 19 May 2026 12:01:18 +0200 Subject: [PATCH 38/81] Add delay between retries in Safari --- build-tools/visual/setup.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/build-tools/visual/setup.js b/build-tools/visual/setup.js index 79050cab6a..a9013a3d61 100644 --- a/build-tools/visual/setup.js +++ b/build-tools/visual/setup.js @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -/* global jest */ +/* global jest, afterEach */ const { configure } = require('@cloudscape-design/browser-test-tools/use-browser'); const isSafari = process.env.BROWSER === 'safari'; @@ -18,3 +18,11 @@ configure({ }); jest.retryTimes(2, { logErrorsBeforeRetry: true }); + +// Safari's WebDriver needs a moment to fully release a session before a new one +// can be created. Without this delay, retried tests hit "already paired" errors. +if (isSafari) { + afterEach(async () => { + await new Promise(resolve => setTimeout(resolve, 1000)); + }); +} From 0f6f0e79de3499379062b3bb3d1f3503fdde1337 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Tue, 19 May 2026 12:21:48 +0200 Subject: [PATCH 39/81] Fine tune Safari delay --- build-tools/visual/setup.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-tools/visual/setup.js b/build-tools/visual/setup.js index a9013a3d61..e6d166d7f3 100644 --- a/build-tools/visual/setup.js +++ b/build-tools/visual/setup.js @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -/* global jest, afterEach */ +/* global jest, beforeEach */ const { configure } = require('@cloudscape-design/browser-test-tools/use-browser'); const isSafari = process.env.BROWSER === 'safari'; @@ -20,9 +20,9 @@ configure({ jest.retryTimes(2, { logErrorsBeforeRetry: true }); // Safari's WebDriver needs a moment to fully release a session before a new one -// can be created. Without this delay, retried tests hit "already paired" errors. +// can be created. Without this delay, the next test hits "already paired" errors. if (isSafari) { - afterEach(async () => { - await new Promise(resolve => setTimeout(resolve, 1000)); + beforeEach(async () => { + await new Promise(resolve => setTimeout(resolve, 5000)); }); } From f44e03d75a4b558aab329ce7efac24b42401e1db Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Tue, 19 May 2026 14:00:13 +0200 Subject: [PATCH 40/81] Release Safari session between tests --- build-tools/visual/global-setup.js | 6 ++---- build-tools/visual/setup.js | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/build-tools/visual/global-setup.js b/build-tools/visual/global-setup.js index f27f7d04cb..b4acf683c9 100644 --- a/build-tools/visual/global-setup.js +++ b/build-tools/visual/global-setup.js @@ -3,18 +3,16 @@ const { spawn } = require('child_process'); const waitOn = require('wait-on'); -let driverProcess; - module.exports = async () => { if (process.env.BROWSER === 'safari') { - driverProcess = spawn('safaridriver', ['--port', '4444']); + const driverProcess = spawn('safaridriver', ['--port', '4444']); driverProcess.on('error', err => { throw err; }); await waitOn({ resources: ['http-get://localhost:4444/status'], timeout: 10000 }); + global.__DRIVER_PROCESS__ = driverProcess; } else { const { startWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); await startWebdriver(); } - global.__DRIVER_PROCESS__ = driverProcess; }; diff --git a/build-tools/visual/setup.js b/build-tools/visual/setup.js index e6d166d7f3..d8e7ab5779 100644 --- a/build-tools/visual/setup.js +++ b/build-tools/visual/setup.js @@ -1,6 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 /* global jest, beforeEach */ +const { spawn } = require('child_process'); +const waitOn = require('wait-on'); const { configure } = require('@cloudscape-design/browser-test-tools/use-browser'); const isSafari = process.env.BROWSER === 'safari'; @@ -19,10 +21,18 @@ configure({ jest.retryTimes(2, { logErrorsBeforeRetry: true }); -// Safari's WebDriver needs a moment to fully release a session before a new one -// can be created. Without this delay, the next test hits "already paired" errors. +// Local safaridriver only supports one session at a time and doesn't reliably +// release the session lock between tests. Restarting the process before each +// test guarantees a clean state. This is not needed with BrowserStack. if (isSafari) { + let safariDriverProcess; + beforeEach(async () => { - await new Promise(resolve => setTimeout(resolve, 5000)); + if (safariDriverProcess) { + safariDriverProcess.kill(); + await new Promise(resolve => setTimeout(resolve, 500)); + } + safariDriverProcess = spawn('safaridriver', ['--port', '4444']); + await waitOn({ resources: ['http-get://localhost:4444/status'], timeout: 10000 }); }); } From 16dd3dd99cf393f56cddb9cd589947e994111cc3 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Tue, 19 May 2026 17:07:46 +0200 Subject: [PATCH 41/81] Do not retry with Safari --- build-tools/visual/setup.js | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/build-tools/visual/setup.js b/build-tools/visual/setup.js index d8e7ab5779..c63a18416f 100644 --- a/build-tools/visual/setup.js +++ b/build-tools/visual/setup.js @@ -1,8 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -/* global jest, beforeEach */ -const { spawn } = require('child_process'); -const waitOn = require('wait-on'); +/* global jest */ const { configure } = require('@cloudscape-design/browser-test-tools/use-browser'); const isSafari = process.env.BROWSER === 'safari'; @@ -19,20 +17,9 @@ configure({ }, }); -jest.retryTimes(2, { logErrorsBeforeRetry: true }); - -// Local safaridriver only supports one session at a time and doesn't reliably -// release the session lock between tests. Restarting the process before each -// test guarantees a clean state. This is not needed with BrowserStack. -if (isSafari) { - let safariDriverProcess; - - beforeEach(async () => { - if (safariDriverProcess) { - safariDriverProcess.kill(); - await new Promise(resolve => setTimeout(resolve, 500)); - } - safariDriverProcess = spawn('safaridriver', ['--port', '4444']); - await waitOn({ resources: ['http-get://localhost:4444/status'], timeout: 10000 }); - }); +// Retries help with flaky tests, but Safari's single-session constraint means +// a retry can hit "already paired" if the previous attempt's session hasn't +// fully released. Disable retries for Safari. +if (!isSafari) { + jest.retryTimes(2, { logErrorsBeforeRetry: true }); } From 09c8134f0c555bfa7e4116bb63e1514731070462 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Tue, 19 May 2026 18:43:41 +0200 Subject: [PATCH 42/81] Change target directory --- build-tools/tasks/index.js | 5 -- build-tools/tasks/visual-definitions.js | 8 -- build-tools/tasks/visual.js | 107 ------------------------ tsconfig.visual-definitions.json | 2 +- 4 files changed, 1 insertion(+), 121 deletions(-) delete mode 100644 build-tools/tasks/visual-definitions.js delete mode 100644 build-tools/tasks/visual.js diff --git a/build-tools/tasks/index.js b/build-tools/tasks/index.js index 413695be14..d7db3c0784 100644 --- a/build-tools/tasks/index.js +++ b/build-tools/tasks/index.js @@ -21,10 +21,5 @@ module.exports = { themeableSource: require('./themeable-source'), bundleVendorFiles: require('./bundle-vendor-files'), sizeLimit: require('./size-limit'), -<<<<<<< HEAD testDefinitions: require('./test-definitions'), -======= - visual: require('./visual'), - visualDefinitions: require('./visual-definitions'), ->>>>>>> 4213557f5 (chore: Export visual test definitions) }; diff --git a/build-tools/tasks/visual-definitions.js b/build-tools/tasks/visual-definitions.js deleted file mode 100644 index 34dfe16adf..0000000000 --- a/build-tools/tasks/visual-definitions.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -const execa = require('execa'); -const { task } = require('../utils/gulp-utils'); - -module.exports = task('visual-definitions', () => - execa('tsc', ['-p', 'tsconfig.visual-definitions.json'], { stdio: 'inherit' }) -); diff --git a/build-tools/tasks/visual.js b/build-tools/tasks/visual.js deleted file mode 100644 index 0864790afb..0000000000 --- a/build-tools/tasks/visual.js +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -const execa = require('execa'); -const path = require('path'); -const fs = require('fs'); -const waitOn = require('wait-on'); -const { task } = require('../utils/gulp-utils.js'); -const { parseArgs } = require('node:util'); - -const BASELINE_WORKTREE = '/tmp/visual-baseline'; -const BASELINE_OUTPUT = path.resolve('pages/lib/static-visual-baseline'); -const TEST_OUTPUT = path.resolve('pages/lib/static-default'); - -// Port assignments: -// 8080 — test build (PR / local changes) -// 8081 — baseline build (main branch) -const TEST_PORT = 8080; -const BASELINE_PORT = 8081; - -/** - * Serves a pre-built static directory using webpack-dev-server in static mode. - */ -function serveStatic(dir, port) { - return execa( - 'node_modules/.bin/webpack', - ['serve', '--config', 'pages/webpack.config.integ.cjs', '--port', String(port), '--static', dir, '--no-hot'], - { env: { ...process.env, NODE_ENV: 'development' } } - ); -} - -/** - * Builds the dev pages from the source tree at `cwd` into `outputPath`. - * Uses the node_modules present in `cwd`. - */ -async function buildPages(cwd, outputPath) { - await execa('npx', ['gulp', 'quick-build'], { - stdio: 'inherit', - cwd, - env: { ...process.env, NODE_ENV: 'production' }, - }); - await execa( - path.join(cwd, 'node_modules/.bin/webpack'), - ['--config', 'pages/webpack.config.integ.cjs', '--output-path', outputPath], - { stdio: 'inherit', cwd, env: { ...process.env, NODE_ENV: 'production' } } - ); -} - -module.exports = task('test:visual', async () => { - const options = { - shard: { type: 'string' }, - // Pass --skip-build to skip the build steps when artifacts are already present. - skipBuild: { type: 'boolean' }, - }; - const { shard, skipBuild } = parseArgs({ options, strict: false }).values; - - const cwd = process.cwd(); - - if (!skipBuild) { - // ── 1. Build the test (PR) pages ──────────────────────────────────────── - console.log('Building test pages (current branch)…'); - await buildPages(cwd, TEST_OUTPUT); - - // ── 2. Build the baseline (main) pages ────────────────────────────────── - // Create a worktree for origin/main so it gets its own node_modules. - // This correctly handles PRs that change package-lock.json: each side - // installs from its own lockfile. - console.log('Setting up baseline worktree from origin/main…'); - if (fs.existsSync(BASELINE_WORKTREE)) { - await execa('git', ['worktree', 'remove', '--force', BASELINE_WORKTREE]); - } - await execa('git', ['worktree', 'add', BASELINE_WORKTREE, 'origin/main']); - - try { - console.log('Installing baseline dependencies…'); - await execa('npm', ['ci'], { stdio: 'inherit', cwd: BASELINE_WORKTREE }); - - console.log('Building baseline pages (origin/main)…'); - await buildPages(BASELINE_WORKTREE, BASELINE_OUTPUT); - } finally { - await execa('git', ['worktree', 'remove', '--force', BASELINE_WORKTREE]); - } - } - - // ── 3. Start both static servers ────────────────────────────────────────── - console.log(`Starting test server on :${TEST_PORT} (${TEST_OUTPUT})…`); - const testServer = serveStatic(TEST_OUTPUT, TEST_PORT); - - console.log(`Starting baseline server on :${BASELINE_PORT} (${BASELINE_OUTPUT})…`); - const baselineServer = serveStatic(BASELINE_OUTPUT, BASELINE_PORT); - - try { - await waitOn({ resources: [`http://localhost:${TEST_PORT}`, `http://localhost:${BASELINE_PORT}`] }); - - // ── 4. Run visual tests ────────────────────────────────────────────────── - const jestArgs = ['-c', 'jest.visual.config.js']; - if (shard) { - jestArgs.push(`--shard=${shard}`); - } - await execa('jest', jestArgs, { - stdio: 'inherit', - env: { ...process.env, NODE_OPTIONS: '--experimental-vm-modules' }, - }); - } finally { - testServer.cancel(); - baselineServer.cancel(); - } -}); diff --git a/tsconfig.visual-definitions.json b/tsconfig.visual-definitions.json index a31ccd02d3..fbc322c92a 100644 --- a/tsconfig.visual-definitions.json +++ b/tsconfig.visual-definitions.json @@ -12,7 +12,7 @@ "sourceMap": true, "inlineSources": true, "rootDir": "test/visual", - "outDir": "lib/visual-test-definitions" + "outDir": "lib/test-definitions" }, "include": ["test/visual/definitions", "test/visual/types.ts"], "exclude": [] From 83af893e958e74ff6a3a5aabe7f0f05d58f0c021 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Thu, 21 May 2026 16:35:14 +0200 Subject: [PATCH 43/81] Remove local testing setup --- gulpfile.js | 4 ---- tsconfig.visual-definitions.json | 19 ------------------- 2 files changed, 23 deletions(-) delete mode 100644 tsconfig.visual-definitions.json diff --git a/gulpfile.js b/gulpfile.js index 09e0833868..6e3f389082 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -19,8 +19,6 @@ const { generateI18nMessages, integ, motion, - visual, - visualDefinitions, copyFiles, themeableSource, bundleVendorFiles, @@ -45,8 +43,6 @@ exports['test:unit'] = unit; exports['test:integ'] = integ; exports['test:a11y'] = a11y; exports['test:motion'] = motion; -exports['test:visual'] = visual; -exports['build:visual-definitions'] = visualDefinitions; exports.watch = () => { watch( diff --git a/tsconfig.visual-definitions.json b/tsconfig.visual-definitions.json deleted file mode 100644 index fbc322c92a..0000000000 --- a/tsconfig.visual-definitions.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "compilerOptions": { - "lib": ["ES2021", "DOM"], - "target": "ES2019", - "types": [], - "module": "CommonJS", - "moduleResolution": "node", - "esModuleInterop": true, - "strict": true, - "declaration": true, - "declarationMap": true, - "sourceMap": true, - "inlineSources": true, - "rootDir": "test/visual", - "outDir": "lib/test-definitions" - }, - "include": ["test/visual/definitions", "test/visual/types.ts"], - "exclude": [] -} From 574d1a587a45bbc96b3c09cda9502ecdc9798868 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Thu, 21 May 2026 16:44:41 +0200 Subject: [PATCH 44/81] Refactor --- docs/RUNNING_TESTS.md | 4 +- .../visual/compare-screenshots.ts | 4 +- test/{visual => }/visual.test.ts | 2 +- test/visual/definitions/action-card.ts | 31 ---------------- test/visual/definitions/alert.ts | 37 ------------------- test/visual/definitions/index.ts | 10 ----- test/visual/types.ts | 28 -------------- 7 files changed, 6 insertions(+), 110 deletions(-) rename test/{ => definitions}/visual/compare-screenshots.ts (96%) rename test/{visual => }/visual.test.ts (70%) delete mode 100644 test/visual/definitions/action-card.ts delete mode 100644 test/visual/definitions/alert.ts delete mode 100644 test/visual/definitions/index.ts delete mode 100644 test/visual/types.ts diff --git a/docs/RUNNING_TESTS.md b/docs/RUNNING_TESTS.md index c5c483e981..6e0eb84503 100644 --- a/docs/RUNNING_TESTS.md +++ b/docs/RUNNING_TESTS.md @@ -86,7 +86,7 @@ NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.con ### Adding tests for a new component -Create `test/visual/definitions/.ts`: +Create `test/definitions/visual/.ts`: ```ts import { TestSuite } from '../types'; @@ -104,7 +104,7 @@ const suite: TestSuite = { export default suite; ``` -Then import and add it to `test/visual/definitions/index.ts`: +Then import and add it to `test/definitions/visual/index.ts`: ```ts import myComponent from './my-component'; diff --git a/test/visual/compare-screenshots.ts b/test/definitions/visual/compare-screenshots.ts similarity index 96% rename from test/visual/compare-screenshots.ts rename to test/definitions/visual/compare-screenshots.ts index 8847e10d21..f81b8afc56 100644 --- a/test/visual/compare-screenshots.ts +++ b/test/definitions/visual/compare-screenshots.ts @@ -7,7 +7,7 @@ import { parsePng } from '@cloudscape-design/browser-test-tools/image-utils'; import { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; import useBrowser from '@cloudscape-design/browser-test-tools/use-browser'; -import { TestDefinition, TestSuite } from './types'; +import { TestDefinition, TestSuite } from '../types'; const screenshotAreaSelector = '.screenshot-area'; const defaultWindowSize = { width: 1600, height: 800 }; @@ -97,6 +97,8 @@ function runSingleTest(testDef: TestDefinition) { test( testDef.description, + // useBrowser is not a React hook, despite the name + // eslint-disable-next-line react-hooks/rules-of-hooks useBrowser(windowSize, async browser => { const newUrl = buildUrl(newHost, testDef.path, testDef.queryParams); const newScreenshot = await captureScreenshot(browser, newUrl, testDef, testDef.setup); diff --git a/test/visual/visual.test.ts b/test/visual.test.ts similarity index 70% rename from test/visual/visual.test.ts rename to test/visual.test.ts index 06ef0fa8a4..bc7369d6a9 100644 --- a/test/visual/visual.test.ts +++ b/test/visual.test.ts @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from './compare-screenshots'; import { allSuites } from './definitions'; +import { runTestSuites } from './definitions/visual/compare-screenshots'; runTestSuites(allSuites); diff --git a/test/visual/definitions/action-card.ts b/test/visual/definitions/action-card.ts deleted file mode 100644 index 8730ac9504..0000000000 --- a/test/visual/definitions/action-card.ts +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { TestSuite } from '../types'; - -const suite: TestSuite = { - description: 'action-card', - tests: [ - { - description: 'permutations', - path: 'action-card/permutations', - screenshotType: 'permutations', - }, - { - description: 'variant permutations', - path: 'action-card/variant-permutations', - screenshotType: 'permutations', - }, - { - description: 'padding permutations', - path: 'action-card/padding-permutations', - screenshotType: 'permutations', - }, - { - description: 'simple', - path: 'action-card/simple', - screenshotType: 'screenshotArea', - }, - ], -}; - -export default suite; diff --git a/test/visual/definitions/alert.ts b/test/visual/definitions/alert.ts deleted file mode 100644 index 30792c0d2d..0000000000 --- a/test/visual/definitions/alert.ts +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { TestSuite } from '../types'; - -const suite: TestSuite = { - description: 'alert', - tests: [ - { - description: 'simple', - path: 'alert/simple', - screenshotType: 'screenshotArea', - }, - { - description: 'style custom page', - path: 'alert/style-custom-types', - screenshotType: 'screenshotArea', - }, - ...[600, 1280].map(width => ({ - description: `width ${width}px`, - tests: [ - { - description: 'permutations', - path: 'alert/permutations', - screenshotType: 'permutations' as const, - }, - { - description: 'custom types', - path: 'alert/style-custom-types', - screenshotType: 'screenshotArea' as const, - }, - ], - })), - ], -}; - -export default suite; diff --git a/test/visual/definitions/index.ts b/test/visual/definitions/index.ts deleted file mode 100644 index 318ce7c68b..0000000000 --- a/test/visual/definitions/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Each component has its own test definition file. -// Import them here manually to form the full test suite. -import { TestSuite } from '../types'; -import actionCard from './action-card'; -import alert from './alert'; - -export const allSuites: TestSuite[] = [actionCard, alert]; diff --git a/test/visual/types.ts b/test/visual/types.ts deleted file mode 100644 index c4c23b622f..0000000000 --- a/test/visual/types.ts +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import type { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; - -export interface ScreenshotTestConfiguration { - width?: number; - height?: number; -} - -export type TestCallback = (page: ScreenshotPageObject) => Promise; - -// 'screenshotArea' — captures the .screenshot-area element on a focused page. -// 'permutations' — captures the entire page and crops permutations out of it. -export type ScreenshotType = 'screenshotArea' | 'permutations'; - -export interface TestDefinition { - description: string; - path: string; - screenshotType: ScreenshotType; - queryParams?: Record; - configuration?: ScreenshotTestConfiguration; - setup?: TestCallback; -} - -export interface TestSuite { - description: string; - tests: Array; -} From 735e123fe57567544a3e7902f3b4dd91b457a124 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 27 May 2026 10:34:20 +0200 Subject: [PATCH 45/81] Refactor --- test/definitions/utils.ts | 72 +++++++++++ .../definitions/visual/compare-screenshots.ts | 115 ------------------ test/visual.test.ts | 2 +- 3 files changed, 73 insertions(+), 116 deletions(-) create mode 100644 test/definitions/utils.ts delete mode 100644 test/definitions/visual/compare-screenshots.ts diff --git a/test/definitions/utils.ts b/test/definitions/utils.ts new file mode 100644 index 0000000000..2a650f79ed --- /dev/null +++ b/test/definitions/utils.ts @@ -0,0 +1,72 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { cropAndCompare } from '@cloudscape-design/browser-test-tools/image-utils'; +import { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; +import useBrowser from '@cloudscape-design/browser-test-tools/use-browser'; + +import { TestDefinition, TestSuite } from './types'; + +const screenshotAreaSelector = '.screenshot-area'; +const defaultWindowSize = { width: 1600, height: 800 }; + +// NEW_HOST serves the PR's pages, OLD_HOST serves the baseline (main) pages. +const newHost = process.env.NEW_HOST || 'http://localhost:8080'; +const oldHost = process.env.OLD_HOST || 'http://localhost:8081'; + +function buildUrl(host: string, path: string, queryParams?: Record): string { + const params = new URLSearchParams(queryParams); + const qs = params.toString(); + return `${host}/#/${path}${qs ? `?${qs}` : ''}`; +} + +function isTestDefinition(item: TestDefinition | TestSuite): item is TestDefinition { + return (item as TestDefinition).path !== undefined; +} + +export function runTestSuites(suites: Array) { + for (const item of suites) { + if (isTestDefinition(item)) { + runSingleTest(item); + } else { + describe(item.description, () => { + runTestSuites(item.tests); + }); + } + } +} + +function runSingleTest(testDef: TestDefinition) { + const windowSize = { ...defaultWindowSize, ...testDef.configuration }; + + test( + testDef.description, + // useBrowser is not a React hook, despite the name + // eslint-disable-next-line react-hooks/rules-of-hooks + useBrowser(windowSize, async browser => { + const page = new ScreenshotPageObject(browser); + + const capture = async (host: string) => { + await browser.url(buildUrl(host, testDef.path, testDef.queryParams)); + await page.waitForVisible(screenshotAreaSelector); + if (testDef.setup) { + await testDef.setup(page); + } + return testDef.screenshotType === 'permutations' + ? page.capturePermutations() + : page.captureBySelector(screenshotAreaSelector); + }; + + const newScreenshots = await capture(newHost); + const oldScreenshots = await capture(oldHost); + + const newArr = Array.isArray(newScreenshots) ? newScreenshots : [newScreenshots]; + const oldArr = Array.isArray(oldScreenshots) ? oldScreenshots : [oldScreenshots]; + + expect(newArr.length).toBe(oldArr.length); + for (let i = 0; i < newArr.length; i++) { + const { diffPixels } = await cropAndCompare(newArr[i], oldArr[i]); + expect(diffPixels).toBe(0); + } + }) + ); +} diff --git a/test/definitions/visual/compare-screenshots.ts b/test/definitions/visual/compare-screenshots.ts deleted file mode 100644 index f81b8afc56..0000000000 --- a/test/definitions/visual/compare-screenshots.ts +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import pixelmatch from 'pixelmatch'; -import { PNG } from 'pngjs'; - -import { parsePng } from '@cloudscape-design/browser-test-tools/image-utils'; -import { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; -import useBrowser from '@cloudscape-design/browser-test-tools/use-browser'; - -import { TestDefinition, TestSuite } from '../types'; - -const screenshotAreaSelector = '.screenshot-area'; -const defaultWindowSize = { width: 1600, height: 800 }; - -// NEW_HOST serves the PR's pages, OLD_HOST serves the baseline (main) pages. -const newHost = process.env.NEW_HOST || 'http://localhost:8080'; -const oldHost = process.env.OLD_HOST || 'http://localhost:8081'; - -/** - * Captures the .screenshot-area element on a focused page. - * Uses a standard ScreenshotPageObject (no forced scroll-and-merge). - */ -async function captureScreenshotArea(browser: WebdriverIO.Browser, url: string): Promise { - await browser.url(url); - const page = new ScreenshotPageObject(browser); - await page.waitForVisible(screenshotAreaSelector); - const { image } = await page.captureBySelector(screenshotAreaSelector); - return image; -} - -/** - * Captures the full page as a PNG for permutation pages. - * Uses fullPageScreenshot which handles pages taller than the viewport. - */ -async function capturePermutations(browser: WebdriverIO.Browser, url: string): Promise { - await browser.url(url); - const page = new ScreenshotPageObject(browser); - await page.waitForVisible(screenshotAreaSelector); - const base64 = await page.fullPageScreenshot(); - return parsePng(base64); -} - -async function captureScreenshot( - browser: WebdriverIO.Browser, - url: string, - testDef: TestDefinition, - setup?: (page: ScreenshotPageObject) => Promise -): Promise { - if (setup) { - await browser.url(url); - const page = new ScreenshotPageObject(browser); - await page.waitForVisible(screenshotAreaSelector); - await setup(page); - if (testDef.screenshotType === 'permutations') { - const base64 = await page.fullPageScreenshot(); - return parsePng(base64); - } - const { image } = await page.captureBySelector(screenshotAreaSelector); - return image; - } - if (testDef.screenshotType === 'permutations') { - return capturePermutations(browser, url); - } - return captureScreenshotArea(browser, url); -} - -function buildUrl(host: string, path: string, queryParams?: Record): string { - const params = new URLSearchParams(queryParams); - const qs = params.toString(); - return `${host}/#/${path}${qs ? `?${qs}` : ''}`; -} - -function compareImages(newImage: PNG, oldImage: PNG): number { - const { width, height } = newImage; - const diff = new PNG({ width, height }); - return pixelmatch(newImage.data, oldImage.data, diff.data, width, height, { threshold: 0.1 }); -} - -function isTestDefinition(item: TestDefinition | TestSuite): item is TestDefinition { - return (item as TestDefinition).path !== undefined; -} - -export function runTestSuites(suites: Array) { - for (const item of suites) { - if (isTestDefinition(item)) { - runSingleTest(item); - } else { - describe(item.description, () => { - runTestSuites(item.tests); - }); - } - } -} - -function runSingleTest(testDef: TestDefinition) { - const windowSize = { ...defaultWindowSize, ...testDef.configuration }; - - test( - testDef.description, - // useBrowser is not a React hook, despite the name - // eslint-disable-next-line react-hooks/rules-of-hooks - useBrowser(windowSize, async browser => { - const newUrl = buildUrl(newHost, testDef.path, testDef.queryParams); - const newScreenshot = await captureScreenshot(browser, newUrl, testDef, testDef.setup); - - const oldUrl = buildUrl(oldHost, testDef.path, testDef.queryParams); - const oldScreenshot = await captureScreenshot(browser, oldUrl, testDef, testDef.setup); - const diffPixels = compareImages(newScreenshot, oldScreenshot); - expect(diffPixels).toBe(0); - }) - ); -} - -// Export the capture functions for use in custom setup callbacks if needed. -export { captureScreenshotArea, capturePermutations }; diff --git a/test/visual.test.ts b/test/visual.test.ts index bc7369d6a9..4e6e992f4f 100644 --- a/test/visual.test.ts +++ b/test/visual.test.ts @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 import { allSuites } from './definitions'; -import { runTestSuites } from './definitions/visual/compare-screenshots'; +import { runTestSuites } from './definitions/utils'; runTestSuites(allSuites); From a2639989eda4539595f12cef8bdb476cb49c67be Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 27 May 2026 10:38:40 +0200 Subject: [PATCH 46/81] Fix setup --- tsconfig.integ.json | 2 +- tsconfig.test-definitions.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tsconfig.integ.json b/tsconfig.integ.json index e816f749e9..784a3d2f1f 100644 --- a/tsconfig.integ.json +++ b/tsconfig.integ.json @@ -11,5 +11,5 @@ "resolveJsonModule": true, "moduleResolution": "node" }, - "include": ["**/__integ__/**/*.ts", "**/__a11y__/**/*.ts", "**/__motion__/**/*.ts", "types"] + "include": ["**/__integ__/**/*.ts", "**/__a11y__/**/*.ts", "**/__motion__/**/*.ts", "test/definitions/utils.ts", "test/visual.test.ts", "types"] } diff --git a/tsconfig.test-definitions.json b/tsconfig.test-definitions.json index 30e82b4047..3f5961a78b 100644 --- a/tsconfig.test-definitions.json +++ b/tsconfig.test-definitions.json @@ -16,5 +16,5 @@ "skipLibCheck": true }, "include": ["test/definitions", "test/types.ts"], - "exclude": [] + "exclude": ["test/definitions/utils.ts"] } From 77e9470a839d3bd1c2760576a7ffeaca71077c30 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 27 May 2026 10:49:40 +0200 Subject: [PATCH 47/81] Fix setup --- docs/RUNNING_TESTS.md | 2 +- eslint.config.mjs | 2 +- jest.visual.config.js | 2 +- test/definitions/utils.ts | 2 -- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/RUNNING_TESTS.md b/docs/RUNNING_TESTS.md index 6e0eb84503..c7832dca72 100644 --- a/docs/RUNNING_TESTS.md +++ b/docs/RUNNING_TESTS.md @@ -68,7 +68,7 @@ They compare permutation pages between the PR build and a baseline build of `mai 1. The PR pages are built and served on port 8080. 2. A git worktree of `origin/main` is created, its dependencies installed, and its pages built and served on port 8081. -3. The single test runner (`test/visual/visual.test.ts`) iterates over all test definitions, captures the `.screenshot-area` element from both servers for each test, and fails if any pixels differ. +3. The single test runner (`test/visual.test.ts`) iterates over all test definitions, captures the `.screenshot-area` element from both servers for each test, and fails if any pixels differ. ### Running locally diff --git a/eslint.config.mjs b/eslint.config.mjs index f03eb9ce60..b92a169696 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -225,7 +225,7 @@ export default tsEslint.config( }, }, { - files: ['**/__integ__/**', '**/__motion__/**', '**/__a11y__/**', 'test/visual/**'], + files: ['**/__integ__/**', '**/__motion__/**', '**/__a11y__/**', 'test/definitions/**'], rules: { // useBrowser is not a hook 'react-hooks/rules-of-hooks': 'off', diff --git a/jest.visual.config.js b/jest.visual.config.js index 0d3abc58fd..097054bb69 100644 --- a/jest.visual.config.js +++ b/jest.visual.config.js @@ -23,5 +23,5 @@ module.exports = { globalTeardown: '/build-tools/visual/global-teardown.js', setupFilesAfterEnv: [path.join(__dirname, 'build-tools', 'visual', 'setup.js')], moduleFileExtensions: ['js', 'ts'], - testMatch: ['/test/visual/visual.test.ts'], + testMatch: ['/test/visual.test.ts'], }; diff --git a/test/definitions/utils.ts b/test/definitions/utils.ts index 2a650f79ed..b29d3a8543 100644 --- a/test/definitions/utils.ts +++ b/test/definitions/utils.ts @@ -40,8 +40,6 @@ function runSingleTest(testDef: TestDefinition) { test( testDef.description, - // useBrowser is not a React hook, despite the name - // eslint-disable-next-line react-hooks/rules-of-hooks useBrowser(windowSize, async browser => { const page = new ScreenshotPageObject(browser); From d022c5bf748cfcbd8787ff3f0077ed22c432f2f0 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 27 May 2026 11:59:05 +0200 Subject: [PATCH 48/81] Create one single browser session for all tests --- test/definitions/utils.ts | 88 ++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/test/definitions/utils.ts b/test/definitions/utils.ts index b29d3a8543..253d5c008b 100644 --- a/test/definitions/utils.ts +++ b/test/definitions/utils.ts @@ -1,8 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 import { cropAndCompare } from '@cloudscape-design/browser-test-tools/image-utils'; -import { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; -import useBrowser from '@cloudscape-design/browser-test-tools/use-browser'; +import { ScreenshotPageObject, ScreenshotWithOffset } from '@cloudscape-design/browser-test-tools/page-objects'; import { TestDefinition, TestSuite } from './types'; @@ -23,48 +22,71 @@ function isTestDefinition(item: TestDefinition | TestSuite): item is TestDefinit return (item as TestDefinition).path !== undefined; } +/** + * Registers all test suites. Creates a single shared browser session for the + * entire suite rather than spinning up a new session per test, which is the + * main source of overhead compared to AWS-UI-IntegrationTests. + */ export function runTestSuites(suites: Array) { + let browser: WebdriverIO.Browser; + + beforeAll(async () => { + const { default: getBrowserCreator } = await import('@cloudscape-design/browser-test-tools/browser'); + const isSafari = process.env.BROWSER === 'safari'; + const browserName = isSafari ? 'Safari' : 'ChromeHeadlessIntegration'; + const seleniumUrl = isSafari ? 'http://localhost:4444' : 'http://localhost:9515'; + const creator = getBrowserCreator(browserName, 'local', { seleniumUrl }); + browser = await creator.getBrowser({ width: defaultWindowSize.width, height: defaultWindowSize.height }); + }); + + afterAll(async () => { + await browser?.deleteSession(); + }); + + registerSuites(suites, () => browser); +} + +function registerSuites(suites: Array, getBrowser: () => WebdriverIO.Browser) { for (const item of suites) { if (isTestDefinition(item)) { - runSingleTest(item); + registerTest(item, getBrowser); } else { describe(item.description, () => { - runTestSuites(item.tests); + registerSuites(item.tests, getBrowser); }); } } } -function runSingleTest(testDef: TestDefinition) { +function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Browser) { const windowSize = { ...defaultWindowSize, ...testDef.configuration }; - test( - testDef.description, - useBrowser(windowSize, async browser => { - const page = new ScreenshotPageObject(browser); - - const capture = async (host: string) => { - await browser.url(buildUrl(host, testDef.path, testDef.queryParams)); - await page.waitForVisible(screenshotAreaSelector); - if (testDef.setup) { - await testDef.setup(page); - } - return testDef.screenshotType === 'permutations' - ? page.capturePermutations() - : page.captureBySelector(screenshotAreaSelector); - }; - - const newScreenshots = await capture(newHost); - const oldScreenshots = await capture(oldHost); - - const newArr = Array.isArray(newScreenshots) ? newScreenshots : [newScreenshots]; - const oldArr = Array.isArray(oldScreenshots) ? oldScreenshots : [oldScreenshots]; - - expect(newArr.length).toBe(oldArr.length); - for (let i = 0; i < newArr.length; i++) { - const { diffPixels } = await cropAndCompare(newArr[i], oldArr[i]); - expect(diffPixels).toBe(0); + test(testDef.description, async () => { + const browser = getBrowser(); + await browser.setWindowSize(windowSize.width, windowSize.height); + const page = new ScreenshotPageObject(browser); + + const capture = async (host: string) => { + await browser.url(buildUrl(host, testDef.path, testDef.queryParams)); + await page.waitForVisible(screenshotAreaSelector); + if (testDef.setup) { + await testDef.setup(page); } - }) - ); + return testDef.screenshotType === 'permutations' + ? page.capturePermutations() + : page.captureBySelector(screenshotAreaSelector); + }; + + const newScreenshots = await capture(newHost); + const oldScreenshots = await capture(oldHost); + + const newArr: ScreenshotWithOffset[] = Array.isArray(newScreenshots) ? newScreenshots : [newScreenshots]; + const oldArr: ScreenshotWithOffset[] = Array.isArray(oldScreenshots) ? oldScreenshots : [oldScreenshots]; + + expect(newArr.length).toBe(oldArr.length); + for (let i = 0; i < newArr.length; i++) { + const { diffPixels } = await cropAndCompare(newArr[i], oldArr[i]); + expect(diffPixels).toBe(0); + } + }); } From c6c3e5d56fbc759288b3da5a73c3c573108b61a4 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 27 May 2026 12:35:06 +0200 Subject: [PATCH 49/81] Try to fix Safari --- test/definitions/utils.ts | 115 +++++++++++++++++++++++--------------- 1 file changed, 70 insertions(+), 45 deletions(-) diff --git a/test/definitions/utils.ts b/test/definitions/utils.ts index 253d5c008b..369026bd02 100644 --- a/test/definitions/utils.ts +++ b/test/definitions/utils.ts @@ -12,41 +12,55 @@ const defaultWindowSize = { width: 1600, height: 800 }; const newHost = process.env.NEW_HOST || 'http://localhost:8080'; const oldHost = process.env.OLD_HOST || 'http://localhost:8081'; +const isSafari = process.env.BROWSER === 'safari'; + function buildUrl(host: string, path: string, queryParams?: Record): string { const params = new URLSearchParams(queryParams); const qs = params.toString(); return `${host}/#/${path}${qs ? `?${qs}` : ''}`; } +async function createBrowser(windowSize = defaultWindowSize): Promise { + const { default: getBrowserCreator } = await import('@cloudscape-design/browser-test-tools/browser'); + const browserName = isSafari ? 'Safari' : 'ChromeHeadlessIntegration'; + const seleniumUrl = isSafari ? 'http://localhost:4444' : 'http://localhost:9515'; + const creator = getBrowserCreator(browserName, 'local', { seleniumUrl }); + return creator.getBrowser({ width: windowSize.width, height: windowSize.height }); +} + function isTestDefinition(item: TestDefinition | TestSuite): item is TestDefinition { return (item as TestDefinition).path !== undefined; } /** - * Registers all test suites. Creates a single shared browser session for the - * entire suite rather than spinning up a new session per test, which is the - * main source of overhead compared to AWS-UI-IntegrationTests. + * Registers all test suites. For Chrome, creates a single shared browser + * session for the entire suite to avoid per-test session overhead. + * + * Safari only supports one WebDriver session at a time and is sensitive to + * session teardown timing, so for Safari we create a fresh session per test. */ export function runTestSuites(suites: Array) { - let browser: WebdriverIO.Browser; - - beforeAll(async () => { - const { default: getBrowserCreator } = await import('@cloudscape-design/browser-test-tools/browser'); - const isSafari = process.env.BROWSER === 'safari'; - const browserName = isSafari ? 'Safari' : 'ChromeHeadlessIntegration'; - const seleniumUrl = isSafari ? 'http://localhost:4444' : 'http://localhost:9515'; - const creator = getBrowserCreator(browserName, 'local', { seleniumUrl }); - browser = await creator.getBrowser({ width: defaultWindowSize.width, height: defaultWindowSize.height }); - }); - - afterAll(async () => { - await browser?.deleteSession(); - }); - - registerSuites(suites, () => browser); + if (isSafari) { + // Per-test sessions: safe for Safari's single-session constraint. + registerSuites(suites, null); + } else { + // Shared session: one browser for all tests in this worker. + let browser: WebdriverIO.Browser; + + beforeAll(async () => { + browser = await createBrowser(); + }); + + afterAll(async () => { + await browser?.deleteSession(); + }); + + registerSuites(suites, () => browser); + } } -function registerSuites(suites: Array, getBrowser: () => WebdriverIO.Browser) { +// getBrowser === null means "create a fresh session per test" (Safari mode). +function registerSuites(suites: Array, getBrowser: (() => WebdriverIO.Browser) | null) { for (const item of suites) { if (isTestDefinition(item)) { registerTest(item, getBrowser); @@ -58,35 +72,46 @@ function registerSuites(suites: Array, getBrowser: ( } } -function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Browser) { +function registerTest(testDef: TestDefinition, getBrowser: (() => WebdriverIO.Browser) | null) { const windowSize = { ...defaultWindowSize, ...testDef.configuration }; test(testDef.description, async () => { - const browser = getBrowser(); - await browser.setWindowSize(windowSize.width, windowSize.height); - const page = new ScreenshotPageObject(browser); - - const capture = async (host: string) => { - await browser.url(buildUrl(host, testDef.path, testDef.queryParams)); - await page.waitForVisible(screenshotAreaSelector); - if (testDef.setup) { - await testDef.setup(page); - } - return testDef.screenshotType === 'permutations' - ? page.capturePermutations() - : page.captureBySelector(screenshotAreaSelector); - }; - - const newScreenshots = await capture(newHost); - const oldScreenshots = await capture(oldHost); - - const newArr: ScreenshotWithOffset[] = Array.isArray(newScreenshots) ? newScreenshots : [newScreenshots]; - const oldArr: ScreenshotWithOffset[] = Array.isArray(oldScreenshots) ? oldScreenshots : [oldScreenshots]; + // Safari: create and destroy a session per test. + // Chrome: reuse the shared session, just resize the window. + const browser = getBrowser ? getBrowser() : await createBrowser(windowSize); + if (getBrowser) { + await browser.setWindowSize(windowSize.width, windowSize.height); + } - expect(newArr.length).toBe(oldArr.length); - for (let i = 0; i < newArr.length; i++) { - const { diffPixels } = await cropAndCompare(newArr[i], oldArr[i]); - expect(diffPixels).toBe(0); + try { + const page = new ScreenshotPageObject(browser); + + const capture = async (host: string) => { + await browser.url(buildUrl(host, testDef.path, testDef.queryParams)); + await page.waitForVisible(screenshotAreaSelector); + if (testDef.setup) { + await testDef.setup(page); + } + return testDef.screenshotType === 'permutations' + ? page.capturePermutations() + : page.captureBySelector(screenshotAreaSelector); + }; + + const newScreenshots = await capture(newHost); + const oldScreenshots = await capture(oldHost); + + const newArr: ScreenshotWithOffset[] = Array.isArray(newScreenshots) ? newScreenshots : [newScreenshots]; + const oldArr: ScreenshotWithOffset[] = Array.isArray(oldScreenshots) ? oldScreenshots : [oldScreenshots]; + + expect(newArr.length).toBe(oldArr.length); + for (let i = 0; i < newArr.length; i++) { + const { diffPixels } = await cropAndCompare(newArr[i], oldArr[i]); + expect(diffPixels).toBe(0); + } + } finally { + if (!getBrowser) { + await browser.deleteSession(); + } } }); } From de6525b40c7d28cb28101562a40411b0cfce670b Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 27 May 2026 12:56:37 +0200 Subject: [PATCH 50/81] Try again --- build-tools/visual/global-setup.js | 11 +++ test/definitions/utils.ts | 112 ++++++++++++----------------- 2 files changed, 55 insertions(+), 68 deletions(-) diff --git a/build-tools/visual/global-setup.js b/build-tools/visual/global-setup.js index b4acf683c9..e4082c183d 100644 --- a/build-tools/visual/global-setup.js +++ b/build-tools/visual/global-setup.js @@ -5,6 +5,17 @@ const waitOn = require('wait-on'); module.exports = async () => { if (process.env.BROWSER === 'safari') { + // Kill any lingering safaridriver process from a previous run to ensure + // no stale sessions exist (Safari only supports one session at a time). + const { execSync } = require('child_process'); + try { + execSync('pkill -f safaridriver', { stdio: 'ignore' }); + // Give the OS a moment to release the port. + await new Promise(resolve => setTimeout(resolve, 1000)); + } catch { + // No existing process — that's fine. + } + const driverProcess = spawn('safaridriver', ['--port', '4444']); driverProcess.on('error', err => { throw err; diff --git a/test/definitions/utils.ts b/test/definitions/utils.ts index 369026bd02..7597a465e2 100644 --- a/test/definitions/utils.ts +++ b/test/definitions/utils.ts @@ -20,47 +20,34 @@ function buildUrl(host: string, path: string, queryParams?: Record { - const { default: getBrowserCreator } = await import('@cloudscape-design/browser-test-tools/browser'); - const browserName = isSafari ? 'Safari' : 'ChromeHeadlessIntegration'; - const seleniumUrl = isSafari ? 'http://localhost:4444' : 'http://localhost:9515'; - const creator = getBrowserCreator(browserName, 'local', { seleniumUrl }); - return creator.getBrowser({ width: windowSize.width, height: windowSize.height }); -} - function isTestDefinition(item: TestDefinition | TestSuite): item is TestDefinition { return (item as TestDefinition).path !== undefined; } /** - * Registers all test suites. For Chrome, creates a single shared browser - * session for the entire suite to avoid per-test session overhead. - * - * Safari only supports one WebDriver session at a time and is sensitive to - * session teardown timing, so for Safari we create a fresh session per test. + * Registers all test suites with a single shared browser session per worker. + * This avoids the per-test session creation overhead that made tests slow. + * Safari runs with maxWorkers: 1, so there's only ever one session. */ export function runTestSuites(suites: Array) { - if (isSafari) { - // Per-test sessions: safe for Safari's single-session constraint. - registerSuites(suites, null); - } else { - // Shared session: one browser for all tests in this worker. - let browser: WebdriverIO.Browser; - - beforeAll(async () => { - browser = await createBrowser(); - }); - - afterAll(async () => { - await browser?.deleteSession(); - }); - - registerSuites(suites, () => browser); - } + let browser: WebdriverIO.Browser; + + beforeAll(async () => { + const { default: getBrowserCreator } = await import('@cloudscape-design/browser-test-tools/browser'); + const browserName = isSafari ? 'Safari' : 'ChromeHeadlessIntegration'; + const seleniumUrl = isSafari ? 'http://localhost:4444' : 'http://localhost:9515'; + const creator = getBrowserCreator(browserName, 'local', { seleniumUrl }); + browser = await creator.getBrowser({ width: defaultWindowSize.width, height: defaultWindowSize.height }); + }); + + afterAll(async () => { + await browser?.deleteSession(); + }); + + registerSuites(suites, () => browser); } -// getBrowser === null means "create a fresh session per test" (Safari mode). -function registerSuites(suites: Array, getBrowser: (() => WebdriverIO.Browser) | null) { +function registerSuites(suites: Array, getBrowser: () => WebdriverIO.Browser) { for (const item of suites) { if (isTestDefinition(item)) { registerTest(item, getBrowser); @@ -72,46 +59,35 @@ function registerSuites(suites: Array, getBrowser: ( } } -function registerTest(testDef: TestDefinition, getBrowser: (() => WebdriverIO.Browser) | null) { +function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Browser) { const windowSize = { ...defaultWindowSize, ...testDef.configuration }; test(testDef.description, async () => { - // Safari: create and destroy a session per test. - // Chrome: reuse the shared session, just resize the window. - const browser = getBrowser ? getBrowser() : await createBrowser(windowSize); - if (getBrowser) { - await browser.setWindowSize(windowSize.width, windowSize.height); - } - - try { - const page = new ScreenshotPageObject(browser); - - const capture = async (host: string) => { - await browser.url(buildUrl(host, testDef.path, testDef.queryParams)); - await page.waitForVisible(screenshotAreaSelector); - if (testDef.setup) { - await testDef.setup(page); - } - return testDef.screenshotType === 'permutations' - ? page.capturePermutations() - : page.captureBySelector(screenshotAreaSelector); - }; - - const newScreenshots = await capture(newHost); - const oldScreenshots = await capture(oldHost); - - const newArr: ScreenshotWithOffset[] = Array.isArray(newScreenshots) ? newScreenshots : [newScreenshots]; - const oldArr: ScreenshotWithOffset[] = Array.isArray(oldScreenshots) ? oldScreenshots : [oldScreenshots]; - - expect(newArr.length).toBe(oldArr.length); - for (let i = 0; i < newArr.length; i++) { - const { diffPixels } = await cropAndCompare(newArr[i], oldArr[i]); - expect(diffPixels).toBe(0); - } - } finally { - if (!getBrowser) { - await browser.deleteSession(); + const browser = getBrowser(); + await browser.setWindowSize(windowSize.width, windowSize.height); + const page = new ScreenshotPageObject(browser); + + const capture = async (host: string) => { + await browser.url(buildUrl(host, testDef.path, testDef.queryParams)); + await page.waitForVisible(screenshotAreaSelector); + if (testDef.setup) { + await testDef.setup(page); } + return testDef.screenshotType === 'permutations' + ? page.capturePermutations() + : page.captureBySelector(screenshotAreaSelector); + }; + + const newScreenshots = await capture(newHost); + const oldScreenshots = await capture(oldHost); + + const newArr: ScreenshotWithOffset[] = Array.isArray(newScreenshots) ? newScreenshots : [newScreenshots]; + const oldArr: ScreenshotWithOffset[] = Array.isArray(oldScreenshots) ? oldScreenshots : [oldScreenshots]; + + expect(newArr.length).toBe(oldArr.length); + for (let i = 0; i < newArr.length; i++) { + const { diffPixels } = await cropAndCompare(newArr[i], oldArr[i]); + expect(diffPixels).toBe(0); } }); } From 38a09d43db4224886a66c2e1e3d0c0dbad8ef8c9 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 27 May 2026 15:52:13 +0200 Subject: [PATCH 51/81] Start safaridriver from workflow --- .github/workflows/visual-regression.yml | 4 +++- build-tools/visual/global-setup.js | 18 +----------------- build-tools/visual/global-teardown.js | 7 ++----- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 4f62e0fa16..a763c7868a 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -142,7 +142,9 @@ jobs: - name: Enable SafariDriver if: matrix.browser == 'safari' - run: sudo safaridriver --enable + run: | + sudo safaridriver --enable + safaridriver --port 4444 & - name: Install dependencies run: npm i diff --git a/build-tools/visual/global-setup.js b/build-tools/visual/global-setup.js index e4082c183d..a1c19b7022 100644 --- a/build-tools/visual/global-setup.js +++ b/build-tools/visual/global-setup.js @@ -1,27 +1,11 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -const { spawn } = require('child_process'); const waitOn = require('wait-on'); module.exports = async () => { if (process.env.BROWSER === 'safari') { - // Kill any lingering safaridriver process from a previous run to ensure - // no stale sessions exist (Safari only supports one session at a time). - const { execSync } = require('child_process'); - try { - execSync('pkill -f safaridriver', { stdio: 'ignore' }); - // Give the OS a moment to release the port. - await new Promise(resolve => setTimeout(resolve, 1000)); - } catch { - // No existing process — that's fine. - } - - const driverProcess = spawn('safaridriver', ['--port', '4444']); - driverProcess.on('error', err => { - throw err; - }); + // safaridriver is started by the CI workflow on port 4444. await waitOn({ resources: ['http-get://localhost:4444/status'], timeout: 10000 }); - global.__DRIVER_PROCESS__ = driverProcess; } else { const { startWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); await startWebdriver(); diff --git a/build-tools/visual/global-teardown.js b/build-tools/visual/global-teardown.js index 366c3f7660..61f45caa02 100644 --- a/build-tools/visual/global-teardown.js +++ b/build-tools/visual/global-teardown.js @@ -1,12 +1,9 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 module.exports = () => { - if (process.env.BROWSER === 'safari') { - if (global.__DRIVER_PROCESS__) { - global.__DRIVER_PROCESS__.kill(); - } - } else { + if (process.env.BROWSER !== 'safari') { const { shutdownWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); shutdownWebdriver(); } + // Safari: safaridriver is managed by the CI workflow, nothing to tear down. }; From b607c4da56256df4ef886875ad0d3d47477c61b5 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Wed, 27 May 2026 17:05:43 +0200 Subject: [PATCH 52/81] Move Safaridriver initialization back to local test setup --- .github/workflows/visual-regression.yml | 8 +++----- build-tools/visual/global-setup.js | 7 ++++++- build-tools/visual/global-teardown.js | 7 +++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index a763c7868a..50891f2558 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -142,9 +142,7 @@ jobs: - name: Enable SafariDriver if: matrix.browser == 'safari' - run: | - sudo safaridriver --enable - safaridriver --port 4444 & + run: sudo safaridriver --enable - name: Install dependencies run: npm i @@ -163,10 +161,10 @@ jobs: # ── Run tests ───────────────────────────────────────────────────────── - name: Start test server (port 8080) - run: npx serve --no-clipboard --listen 8080 pages/lib/static-default & + run: npx --yes serve --no-clipboard --listen 8080 pages/lib/static-default & - name: Start baseline server (port 8081) - run: npx serve --no-clipboard --listen 8081 pages/lib/static-visual-baseline & + run: npx --yes serve --no-clipboard --listen 8081 pages/lib/static-visual-baseline & - name: Wait for servers to be ready run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081 diff --git a/build-tools/visual/global-setup.js b/build-tools/visual/global-setup.js index a1c19b7022..b4acf683c9 100644 --- a/build-tools/visual/global-setup.js +++ b/build-tools/visual/global-setup.js @@ -1,11 +1,16 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +const { spawn } = require('child_process'); const waitOn = require('wait-on'); module.exports = async () => { if (process.env.BROWSER === 'safari') { - // safaridriver is started by the CI workflow on port 4444. + const driverProcess = spawn('safaridriver', ['--port', '4444']); + driverProcess.on('error', err => { + throw err; + }); await waitOn({ resources: ['http-get://localhost:4444/status'], timeout: 10000 }); + global.__DRIVER_PROCESS__ = driverProcess; } else { const { startWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); await startWebdriver(); diff --git a/build-tools/visual/global-teardown.js b/build-tools/visual/global-teardown.js index 61f45caa02..366c3f7660 100644 --- a/build-tools/visual/global-teardown.js +++ b/build-tools/visual/global-teardown.js @@ -1,9 +1,12 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 module.exports = () => { - if (process.env.BROWSER !== 'safari') { + if (process.env.BROWSER === 'safari') { + if (global.__DRIVER_PROCESS__) { + global.__DRIVER_PROCESS__.kill(); + } + } else { const { shutdownWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); shutdownWebdriver(); } - // Safari: safaridriver is managed by the CI workflow, nothing to tear down. }; From dedf6ecbe56844ceca58fe753b6a28476ae62b58 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Thu, 28 May 2026 10:23:36 +0200 Subject: [PATCH 53/81] Remove outdated npm script --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index d25129aa23..8110f028b0 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,6 @@ "test:a11y": "gulp test:a11y", "test:integ": "gulp test:integ", "test:motion": "gulp test:motion", - "test:visual": "gulp test:visual", "lint": "npm-run-all --parallel lint:*", "lint:eslint": "eslint .", "lint:stylelint": "stylelint --ignore-path .gitignore '{src,pages}/**/*.{css,scss}'", From 3781167eefc43472650ae436221abc49cbaf7b32 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 29 May 2026 09:02:08 +0200 Subject: [PATCH 54/81] Remove Safari setup --- .github/workflows/visual-regression.yml | 7 ---- build-tools/visual/global-setup.js | 15 ++------- build-tools/visual/global-teardown.js | 11 ++----- build-tools/visual/setup.js | 15 ++------- jest.visual.config.js | 4 +-- test/definitions/utils.ts | 43 +++++++++++-------------- 6 files changed, 28 insertions(+), 67 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 50891f2558..40493f7e9e 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -123,8 +123,6 @@ jobs: include: - browser: chrome os: ubuntu-latest - - browser: safari - os: macos-latest steps: - uses: actions/checkout@v4 @@ -135,15 +133,10 @@ jobs: cache: npm - name: Setup Chrome and ChromeDriver - if: matrix.browser == 'chrome' uses: browser-actions/setup-chrome@v1 with: chrome-version: stable - - name: Enable SafariDriver - if: matrix.browser == 'safari' - run: sudo safaridriver --enable - - name: Install dependencies run: npm i diff --git a/build-tools/visual/global-setup.js b/build-tools/visual/global-setup.js index b4acf683c9..52ce2f271c 100644 --- a/build-tools/visual/global-setup.js +++ b/build-tools/visual/global-setup.js @@ -1,18 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -const { spawn } = require('child_process'); -const waitOn = require('wait-on'); module.exports = async () => { - if (process.env.BROWSER === 'safari') { - const driverProcess = spawn('safaridriver', ['--port', '4444']); - driverProcess.on('error', err => { - throw err; - }); - await waitOn({ resources: ['http-get://localhost:4444/status'], timeout: 10000 }); - global.__DRIVER_PROCESS__ = driverProcess; - } else { - const { startWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); - await startWebdriver(); - } + const { startWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); + await startWebdriver(); }; diff --git a/build-tools/visual/global-teardown.js b/build-tools/visual/global-teardown.js index 366c3f7660..0fa05eebfe 100644 --- a/build-tools/visual/global-teardown.js +++ b/build-tools/visual/global-teardown.js @@ -1,12 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 + module.exports = () => { - if (process.env.BROWSER === 'safari') { - if (global.__DRIVER_PROCESS__) { - global.__DRIVER_PROCESS__.kill(); - } - } else { - const { shutdownWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); - shutdownWebdriver(); - } + const { shutdownWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); + shutdownWebdriver(); }; diff --git a/build-tools/visual/setup.js b/build-tools/visual/setup.js index c63a18416f..d52cd606fb 100644 --- a/build-tools/visual/setup.js +++ b/build-tools/visual/setup.js @@ -3,23 +3,14 @@ /* global jest */ const { configure } = require('@cloudscape-design/browser-test-tools/use-browser'); -const isSafari = process.env.BROWSER === 'safari'; - -// The PR build (the code under test) is served on port 8080. -// The baseline build (main branch, same node_modules) is served on port 8081. configure({ - browserName: isSafari ? 'Safari' : 'ChromeHeadlessIntegration', + browserName: 'ChromeHeadlessIntegration', browserCreatorOptions: { - seleniumUrl: isSafari ? 'http://localhost:4444' : 'http://localhost:9515', + seleniumUrl: 'http://localhost:9515', }, webdriverOptions: { baseUrl: 'http://localhost:8080', }, }); -// Retries help with flaky tests, but Safari's single-session constraint means -// a retry can hit "already paired" if the previous attempt's session hasn't -// fully released. Disable retries for Safari. -if (!isSafari) { - jest.retryTimes(2, { logErrorsBeforeRetry: true }); -} +jest.retryTimes(2, { logErrorsBeforeRetry: true }); diff --git a/jest.visual.config.js b/jest.visual.config.js index 097054bb69..a4d06d46ae 100644 --- a/jest.visual.config.js +++ b/jest.visual.config.js @@ -16,9 +16,7 @@ module.exports = { }, reporters: ['default', 'github-actions'], testTimeout: 120_000, // 2min — pages can be tall and slow to capture - // Safari's WebDriver only supports one concurrent session, so tests must run serially. - // Chrome can run multiple workers to speed things up. - maxWorkers: process.env.BROWSER === 'safari' ? 1 : os.cpus().length * (process.env.GITHUB_ACTION ? 3 : 1), + maxWorkers: os.cpus().length * (process.env.GITHUB_ACTION ? 3 : 1), globalSetup: '/build-tools/visual/global-setup.js', globalTeardown: '/build-tools/visual/global-teardown.js', setupFilesAfterEnv: [path.join(__dirname, 'build-tools', 'visual', 'setup.js')], diff --git a/test/definitions/utils.ts b/test/definitions/utils.ts index 7597a465e2..2467c3801b 100644 --- a/test/definitions/utils.ts +++ b/test/definitions/utils.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 import { cropAndCompare } from '@cloudscape-design/browser-test-tools/image-utils'; -import { ScreenshotPageObject, ScreenshotWithOffset } from '@cloudscape-design/browser-test-tools/page-objects'; +import { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; import { TestDefinition, TestSuite } from './types'; @@ -12,8 +12,6 @@ const defaultWindowSize = { width: 1600, height: 800 }; const newHost = process.env.NEW_HOST || 'http://localhost:8080'; const oldHost = process.env.OLD_HOST || 'http://localhost:8081'; -const isSafari = process.env.BROWSER === 'safari'; - function buildUrl(host: string, path: string, queryParams?: Record): string { const params = new URLSearchParams(queryParams); const qs = params.toString(); @@ -26,17 +24,16 @@ function isTestDefinition(item: TestDefinition | TestSuite): item is TestDefinit /** * Registers all test suites with a single shared browser session per worker. - * This avoids the per-test session creation overhead that made tests slow. - * Safari runs with maxWorkers: 1, so there's only ever one session. + * This avoids the per-test session creation overhead. */ export function runTestSuites(suites: Array) { let browser: WebdriverIO.Browser; beforeAll(async () => { const { default: getBrowserCreator } = await import('@cloudscape-design/browser-test-tools/browser'); - const browserName = isSafari ? 'Safari' : 'ChromeHeadlessIntegration'; - const seleniumUrl = isSafari ? 'http://localhost:4444' : 'http://localhost:9515'; - const creator = getBrowserCreator(browserName, 'local', { seleniumUrl }); + const creator = getBrowserCreator('ChromeHeadlessIntegration', 'local', { + seleniumUrl: 'http://localhost:9515', + }); browser = await creator.getBrowser({ width: defaultWindowSize.width, height: defaultWindowSize.height }); }); @@ -60,11 +57,13 @@ function registerSuites(suites: Array, getBrowser: ( } function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Browser) { - const windowSize = { ...defaultWindowSize, ...testDef.configuration }; - test(testDef.description, async () => { const browser = getBrowser(); - await browser.setWindowSize(windowSize.width, windowSize.height); + // Only resize if the test needs a non-default window size. + if (testDef.configuration) { + const windowSize = { ...defaultWindowSize, ...testDef.configuration }; + await browser.setWindowSize(windowSize.width, windowSize.height); + } const page = new ScreenshotPageObject(browser); const capture = async (host: string) => { @@ -73,21 +72,17 @@ function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Bro if (testDef.setup) { await testDef.setup(page); } - return testDef.screenshotType === 'permutations' - ? page.capturePermutations() - : page.captureBySelector(screenshotAreaSelector); + // For screenshotArea pages the element fits in the viewport, so we use + // viewportOnly to avoid the expensive scroll-and-merge full-page capture. + // Permutations pages can be taller than the viewport and need the full strategy. + const viewportOnly = testDef.screenshotType !== 'permutations'; + return page.captureBySelector(screenshotAreaSelector, { viewportOnly }); }; - const newScreenshots = await capture(newHost); - const oldScreenshots = await capture(oldHost); + const newScreenshot = await capture(newHost); + const oldScreenshot = await capture(oldHost); - const newArr: ScreenshotWithOffset[] = Array.isArray(newScreenshots) ? newScreenshots : [newScreenshots]; - const oldArr: ScreenshotWithOffset[] = Array.isArray(oldScreenshots) ? oldScreenshots : [oldScreenshots]; - - expect(newArr.length).toBe(oldArr.length); - for (let i = 0; i < newArr.length; i++) { - const { diffPixels } = await cropAndCompare(newArr[i], oldArr[i]); - expect(diffPixels).toBe(0); - } + const { diffPixels } = await cropAndCompare(newScreenshot, oldScreenshot); + expect(diffPixels).toBe(0); }); } From 5fdb3af4da450bf42d967f30385c87b416f567ec Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 29 May 2026 09:13:40 +0200 Subject: [PATCH 55/81] use captureScreenshotArea for permutation comparison unless there is a failure --- test/definitions/utils.ts | 65 +++++++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/test/definitions/utils.ts b/test/definitions/utils.ts index 2467c3801b..a1e8a3bd1f 100644 --- a/test/definitions/utils.ts +++ b/test/definitions/utils.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 import { cropAndCompare } from '@cloudscape-design/browser-test-tools/image-utils'; -import { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; +import { ScreenshotPageObject, ScreenshotWithOffset } from '@cloudscape-design/browser-test-tools/page-objects'; import { TestDefinition, TestSuite } from './types'; @@ -56,33 +56,72 @@ function registerSuites(suites: Array, getBrowser: ( } } +/** + * Captures the .screenshot-area element using a viewport-only screenshot (fast). + */ +async function captureScreenshotArea( + browser: WebdriverIO.Browser, + page: ScreenshotPageObject, + url: string, + setup?: (page: ScreenshotPageObject) => Promise +): Promise { + await browser.url(url); + await page.waitForVisible(screenshotAreaSelector); + if (setup) { + await setup(page); + } + return page.captureBySelector(screenshotAreaSelector, { viewportOnly: true }); +} + function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Browser) { test(testDef.description, async () => { const browser = getBrowser(); - // Only resize if the test needs a non-default window size. if (testDef.configuration) { const windowSize = { ...defaultWindowSize, ...testDef.configuration }; await browser.setWindowSize(windowSize.width, windowSize.height); } const page = new ScreenshotPageObject(browser); - const capture = async (host: string) => { - await browser.url(buildUrl(host, testDef.path, testDef.queryParams)); + const newUrl = buildUrl(newHost, testDef.path, testDef.queryParams); + const oldUrl = buildUrl(oldHost, testDef.path, testDef.queryParams); + + // Fast path: compare the screenshot area (viewport-only, no scroll-and-merge). + const newScreenshot = await captureScreenshotArea(browser, page, newUrl, testDef.setup); + const oldScreenshot = await captureScreenshotArea(browser, page, oldUrl, testDef.setup); + const { diffPixels } = await cropAndCompare(newScreenshot, oldScreenshot); + + if (diffPixels === 0) { + return; + } + + // For permutations pages, a screenshot-area diff might be a false positive + // caused by content extending beyond the viewport. Re-capture using the + // full capturePermutations strategy which resizes the window to fit all + // content and returns individual permutation crops for precise comparison. + if (testDef.screenshotType === 'permutations') { + await browser.url(newUrl); await page.waitForVisible(screenshotAreaSelector); if (testDef.setup) { await testDef.setup(page); } - // For screenshotArea pages the element fits in the viewport, so we use - // viewportOnly to avoid the expensive scroll-and-merge full-page capture. - // Permutations pages can be taller than the viewport and need the full strategy. - const viewportOnly = testDef.screenshotType !== 'permutations'; - return page.captureBySelector(screenshotAreaSelector, { viewportOnly }); - }; + const newPermutations = await page.capturePermutations(); - const newScreenshot = await capture(newHost); - const oldScreenshot = await capture(oldHost); + await browser.url(oldUrl); + await page.waitForVisible(screenshotAreaSelector); + if (testDef.setup) { + await testDef.setup(page); + } + const oldPermutations = await page.capturePermutations(); - const { diffPixels } = await cropAndCompare(newScreenshot, oldScreenshot); + expect(newPermutations.length).toBe(oldPermutations.length); + for (let i = 0; i < newPermutations.length; i++) { + const { diffPixels: permDiff } = await cropAndCompare(newPermutations[i], oldPermutations[i]); + expect(permDiff).toBe(0); + } + return; + } + + // For screenshotArea type, the diff is a real failure. expect(diffPixels).toBe(0); }); } From f95ef17c66b3a8ca5e49fd04bf0be7898701ba85 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 29 May 2026 09:38:28 +0200 Subject: [PATCH 56/81] Add more test definitions --- test/definitions/index.ts | 3 +- test/definitions/visual/app-layout.ts | 586 ++++++++++++++++++++++++++ 2 files changed, 588 insertions(+), 1 deletion(-) create mode 100644 test/definitions/visual/app-layout.ts diff --git a/test/definitions/index.ts b/test/definitions/index.ts index 91e90a89f7..6ff236cfa6 100644 --- a/test/definitions/index.ts +++ b/test/definitions/index.ts @@ -6,5 +6,6 @@ import { TestSuite } from './types'; import actionCard from './visual/action-card'; import alert from './visual/alert'; +import appLayout from './visual/app-layout'; -export const allSuites: TestSuite[] = [actionCard, alert]; +export const allSuites: TestSuite[] = [actionCard, alert, appLayout]; diff --git a/test/definitions/visual/app-layout.ts b/test/definitions/visual/app-layout.ts new file mode 100644 index 0000000000..936f07152f --- /dev/null +++ b/test/definitions/visual/app-layout.ts @@ -0,0 +1,586 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestDefinition, TestSuite } from '../types'; + +function responsiveTests(width: number): TestSuite { + return { + description: `width ${width}px`, + componentName: 'app-layout', + tests: [ + { + description: 'default', + path: 'app-layout/default', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'navigation drawer is open', + path: 'app-layout/with-wizard', + screenshotType: 'screenshotArea', + configuration: { width }, + setup: async page => { + await page.click('[aria-label="Open navigation"]'); + }, + }, + { + description: 'wizard', + path: 'app-layout/with-wizard', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'with wizard and table', + path: 'app-layout/with-wizard-and-table', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'with wizard, table, and breadcrumbs', + path: 'app-layout/with-wizard-and-table', + screenshotType: 'screenshotArea', + configuration: { width }, + queryParams: { hasBreadcrumbs: 'true' }, + }, + { + description: 'notifications', + path: 'app-layout/with-notifications', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'breadcrumbs', + path: 'app-layout/with-breadcrumbs', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'notifications and breadcrumbs', + path: 'app-layout/with-breadcrumbs-notifications', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'dashboard content type', + path: 'app-layout/dashboard-content-type', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'fixed header and footer', + path: 'app-layout/with-fixed-header-footer', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'disableBodyScroll - empty', + path: 'app-layout/legacy-nav-empty', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'disableBodyScroll - with content', + path: 'app-layout/legacy-nav-scrollable', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'disableBodyScroll - with split panel', + path: 'app-layout/legacy-nav-scrollable-with-split-panel', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'disable paddings', + path: 'app-layout/disable-paddings', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'disable paddings with breadcrumbs', + path: 'app-layout/disable-paddings-breadcrumbs', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'sticky notifications', + path: 'app-layout/with-sticky-notifications', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'sticky notifications scrolled down', + path: 'app-layout/with-sticky-notifications', + screenshotType: 'screenshotArea', + configuration: { width }, + setup: async page => { + await page.windowScrollTo({ top: 2000 }); + }, + }, + { + description: 'layout without panels', + path: 'app-layout/no-panels', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'layout without panels but with notifications', + path: 'app-layout/no-panels-with-notifications', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'with drawers', + path: 'app-layout/with-drawers', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'with empty drawers', + path: 'app-layout/with-drawers-empty', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'with open drawer', + path: 'app-layout/with-drawers', + screenshotType: 'screenshotArea', + configuration: { width }, + setup: async page => { + await page.click('[aria-label="Security trigger button"]'); + }, + }, + ], + }; +} + +const suite: TestSuite = { + description: 'AppLayout', + componentName: 'app-layout', + tests: [ + // ── Responsive tests at multiple breakpoints ────────────────────────── + responsiveTests(600), + responsiveTests(1280), + responsiveTests(1400), + responsiveTests(1920), + responsiveTests(2540), + + // ── General tests ───────────────────────────────────────────────────── + { + description: 'no scrollbars at 320px', + path: 'app-layout/default', + screenshotType: 'screenshotArea', + configuration: { width: 320 }, + }, + { + description: 'drawer buttons alignment', + path: 'app-layout/default', + screenshotType: 'screenshotArea', + configuration: { width: 800 }, + setup: async page => { + await page.click('[aria-label="Open tools"]'); + }, + }, + { + description: 'disable paddings - navigation closed', + path: 'app-layout/disable-paddings', + screenshotType: 'screenshotArea', + configuration: { width: 1280 }, + setup: async page => { + await page.click('[aria-label="Close navigation"]'); + }, + }, + { + description: 'panels stacking on mobile', + path: 'app-layout/all-panels-open', + screenshotType: 'screenshotArea', + configuration: { width: 600 }, + }, + { + description: 'wrapping long words', + path: 'app-layout/text-wrap', + screenshotType: 'screenshotArea', + }, + { + description: 'fill content area', + path: 'app-layout/fill-content-area', + screenshotType: 'screenshotArea', + }, + { + description: 'with tools and drawers', + path: 'app-layout/with-drawers', + screenshotType: 'screenshotArea', + queryParams: { hasTools: 'true' }, + }, + { + description: 'with open drawer and open side split panel', + path: 'app-layout/with-drawers', + screenshotType: 'screenshotArea', + configuration: { width: 1400 }, + queryParams: { splitPanelPosition: 'side' }, + setup: async page => { + await page.click('[aria-label="Security trigger button"]'); + await page.click('[aria-label="Open panel"]'); + }, + }, + + // ── Content paddings ────────────────────────────────────────────────── + { + description: 'Content paddings', + tests: [ + ...(['true', 'false'] as const).flatMap(toolsEnabled => + (['true', 'false'] as const).flatMap(splitPanelEnabled => + (['bottom', 'side'] as const).map(splitPanelPosition => ({ + description: `toolsEnabled=${toolsEnabled} splitPanelEnabled=${splitPanelEnabled} splitPanelPosition=${splitPanelPosition}`, + path: 'app-layout/with-split-panel', + screenshotType: 'screenshotArea' as const, + queryParams: { toolsEnabled, splitPanelEnabled, splitPanelPosition }, + })) + ) + ), + ...[1500, 600].map(width => ({ + description: `with split panel and disabled content paddings - width=${width}`, + path: 'app-layout/disable-paddings-with-split-panel', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + queryParams: { splitPanelOpen: 'true', splitPanelPosition: 'side' }, + })), + ], + }, + + // ── Drawers ─────────────────────────────────────────────────────────── + { + description: 'Drawers', + tests: [ + { + description: 'with split panel', + path: 'app-layout/with-drawers', + screenshotType: 'screenshotArea', + setup: async page => { + await page.click('[aria-label="Pro help trigger button"]'); + }, + }, + { + description: 'with tooltip on hover', + path: 'app-layout/with-drawers', + screenshotType: 'screenshotArea', + setup: async page => { + await page.hoverElement('[aria-label="Pro help trigger button"]'); + }, + }, + { + description: 'with custom scrollable drawer content', + path: 'app-layout/with-drawers-scrollable', + screenshotType: 'screenshotArea', + queryParams: { sideNavFill: 'false' }, + setup: async page => { + await page.click('[aria-label="Chat trigger button"]'); + }, + }, + ], + }, + + // ── Headers ─────────────────────────────────────────────────────────── + { + description: 'Headers', + tests: [600, 1280].flatMap(width => [ + { + description: `alignment with full-page table (${width}px)`, + path: 'app-layout/with-table', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + }, + { + description: `alignment with full-page table in sticky state (${width}px)`, + path: 'app-layout/with-table', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + setup: async page => { + await page.windowScrollTo({ top: 200 }); + }, + }, + { + description: `alignment with full-page table in sticky state with sticky notifications (${width}px)`, + path: 'app-layout/with-table', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + queryParams: { stickyNotifications: 'true' }, + setup: async page => { + await page.windowScrollTo({ top: 200 }); + }, + }, + { + description: `high contrast header variant in landing page (${width}px)`, + path: 'app-layout/landing-page', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + }, + ]), + }, + + // ── High contrast header variant ────────────────────────────────────── + { + description: 'High contrast header variant', + tests: [ + ...[1400, 600].flatMap(width => [ + { + description: `with breadcrumbs and notifications at ${width}px`, + path: 'app-layout/high-contrast-header-variant', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + queryParams: { hasBreadcrumbs: 'true', hasNotifications: 'true', hasContainer: 'true' }, + }, + { + description: `without overlap at ${width}px`, + path: 'app-layout/high-contrast-header-variant', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + queryParams: { disableOverlap: 'true' }, + }, + { + description: `with content layout at ${width}px`, + path: 'app-layout/high-contrast-header-variant', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + queryParams: { + hasBreadcrumbs: 'true', + hasNotifications: 'true', + hasContainer: 'true', + hasContentLayout: 'true', + }, + }, + ]), + ], + }, + + // ── Multiple instances ───────────────────────────────────────────────── + { + description: 'Multiple instances', + tests: [600, 1280].flatMap(width => [ + { + description: `simple (${width}px)`, + path: 'app-layout/multi-layout-simple', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + }, + { + description: `iframe (${width}px)`, + path: 'app-layout/multi-layout-iframe', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + }, + ]), + }, + + // ── Z-index (absolute components) ───────────────────────────────────── + { + description: 'Z-index', + tests: [ + ...[600, 1280].flatMap(width => [ + { + description: `button dropdown (${width}px)`, + path: 'app-layout/with-absolute-components', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + setup: async page => { + await page.click('button=Button dropdown'); + await page.click('[data-testid="2"]'); + await page.windowScrollTo({ top: 300 }); + }, + } as TestDefinition, + { + description: `select (${width}px)`, + path: 'app-layout/with-absolute-components', + screenshotType: 'screenshotArea' as const, + configuration: { width, height: 800 }, + setup: async page => { + await page.click('[data-testid="select-demo"] button'); + await page.windowScrollTo({ top: 300 }); + }, + } as TestDefinition, + { + description: `split-panel and full-page table (${width}px)`, + path: 'app-layout/with-full-page-table-and-split-panel', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + }, + ]), + { + description: 'split-panel and full-page with open navigation (600px)', + path: 'app-layout/with-full-page-table-and-split-panel', + screenshotType: 'screenshotArea' as const, + configuration: { width: 600 }, + setup: async page => { + await page.click('button[aria-label="Open navigation"]'); + }, + }, + { + description: 'split-panel and full-page with open tools (600px)', + path: 'app-layout/with-full-page-table-and-split-panel', + screenshotType: 'screenshotArea' as const, + configuration: { width: 600 }, + setup: async page => { + await page.click('button[aria-label="Open tools"]'); + }, + }, + ], + }, + + // ── Toolbar ─────────────────────────────────────────────────────────── + { + description: 'Toolbar', + tests: [ + { + description: 'multiple nested instances (no breadcrumbs dedup)', + path: 'app-layout-toolbar/multi-layout-with-hidden-instances', + screenshotType: 'screenshotArea', + }, + { + description: 'no toolbar', + path: 'app-layout-toolbar/without-toolbar', + screenshotType: 'screenshotArea', + }, + ], + }, + + // ── Max content width ───────────────────────────────────────────────── + { + description: 'Max content width', + tests: [ + { + description: 'maxContentWidth set to Number.MAX_VALUE', + path: 'app-layout/refresh-content-width', + screenshotType: 'screenshotArea', + configuration: { width: 1280, height: 700 }, + setup: async page => { + await page.click('[data-test-id="button_width-number-max_value"]'); + }, + }, + ], + }, + + // ── Sticky table header with split panel ────────────────────────────── + { + description: 'Sticky header with split panel', + tests: [ + { + description: 'scrolling to bottom with closed split panel (1 table row)', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'screenshotArea', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-1"]'); + await page.scrollToBottom('html'); + }, + }, + { + description: 'scrolling to bottom with closed split panel (30 table rows)', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'screenshotArea', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-30"]'); + await page.scrollToBottom('html'); + }, + }, + { + description: 'header stays sticky with open split panel (1 table row)', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'screenshotArea', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-1"]'); + await page.click('aria/Open panel'); + await page.scrollToBottom('html'); + }, + }, + { + description: 'header stays sticky with open split panel (30 table rows)', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'screenshotArea', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-30"]'); + await page.click('aria/Open panel'); + await page.scrollToBottom('html'); + }, + }, + { + description: 'header stays sticky when mounting and unmounting a second table', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'screenshotArea', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-30"]'); + await page.click('aria/Open panel'); + await page.windowScrollTo({ top: 0 }); + await page.click('aria/Close panel'); + await page.scrollToBottom('html'); + }, + }, + ], + }, + + // ── Flashbar ────────────────────────────────────────────────────────── + { + description: 'Flashbar', + tests: [true, false].flatMap(disableContentPaddings => + [true, false].flatMap(stickyNotifications => + [true, false].flatMap(stickyTableHeader => + [true, false].map(stackNotifications => ({ + description: `disableContentPaddings: ${disableContentPaddings}, stickyNotifications: ${stickyNotifications}, stickyTableHeader: ${stickyTableHeader}, stackNotifications: ${stackNotifications}`, + path: 'app-layout/with-stacked-notifications-and-table', + screenshotType: 'screenshotArea' as const, + configuration: { width: 1280, height: 900 }, + setup: async ( + page: import('@cloudscape-design/browser-test-tools/page-objects').ScreenshotPageObject + ) => { + if (!disableContentPaddings) { + await page.click('[data-id="toggle-content-paddings"]'); + } + if (stickyNotifications) { + await page.click('[data-id="toggle-sticky-notifications"]'); + } + if (!stickyTableHeader) { + await page.click('[data-id="toggle-sticky-table-header"]'); + } + if (!stackNotifications) { + await page.click('[data-id="toggle-stack-items"]'); + } + await page.click('[data-id="add-notification"]'); + await page.click('[data-id="add-notification"]'); + }, + })) + ) + ) + ), + }, + + // ── Transitions ─────────────────────────────────────────────────────── + { + description: 'Transitions', + tests: [ + { + description: 'transition from 400px to 1800px', + path: 'app-layout/default', + screenshotType: 'screenshotArea', + configuration: { width: 400, height: 400 }, + setup: async page => { + await page.setWindowSize({ width: 1800, height: 400 }); + }, + }, + { + description: 'transition from 1800px to 400px', + path: 'app-layout/default', + screenshotType: 'screenshotArea', + configuration: { width: 1800, height: 400 }, + setup: async page => { + await page.setWindowSize({ width: 400, height: 400 }); + }, + }, + ], + }, + ], +}; + +export default suite; From 354af7d0e7ef44ba4554317d6fad98257faa64ef Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 29 May 2026 09:47:07 +0200 Subject: [PATCH 57/81] Refine config --- tsconfig.integ.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tsconfig.integ.json b/tsconfig.integ.json index 784a3d2f1f..e23c746e71 100644 --- a/tsconfig.integ.json +++ b/tsconfig.integ.json @@ -5,6 +5,7 @@ "types": ["jest"], "noEmit": true, "strict": true, + "isolatedModules": true, "sourceMap": true, "allowSyntheticDefaultImports": true, "esModuleInterop": true, From e3937e7a4763a10451410b96c12231d9b70a37de Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 29 May 2026 10:46:16 +0200 Subject: [PATCH 58/81] Shard tests --- .github/workflows/visual-regression.yml | 13 +++++-------- jest.visual.config.js | 2 +- test/visual.test.ts | 6 ------ test/visual/action-card.test.ts | 6 ++++++ test/visual/alert.test.ts | 6 ++++++ test/visual/app-layout.test.ts | 6 ++++++ tsconfig.integ.json | 2 +- 7 files changed, 25 insertions(+), 16 deletions(-) delete mode 100644 test/visual.test.ts create mode 100644 test/visual/action-card.test.ts create mode 100644 test/visual/alert.test.ts create mode 100644 test/visual/app-layout.test.ts diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 40493f7e9e..86f0fdc7ea 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -114,15 +114,13 @@ jobs: retention-days: 1 visual: - name: Visual regression (${{ matrix.browser }}) + name: Visual regression (shard ${{ matrix.shard }}) needs: [stage-pr-pages, build-baseline] - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - include: - - browser: chrome - os: ubuntu-latest + shard: [1, 2, 3, 4] steps: - uses: actions/checkout@v4 @@ -163,15 +161,14 @@ jobs: run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081 - name: Run visual regression tests - run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js + run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js --shard=${{ matrix.shard }}/4 env: TZ: UTC - BROWSER: ${{ matrix.browser }} - name: Upload diff artifacts if: failure() uses: actions/upload-artifact@v4 with: - name: visual-regression-diffs-${{ matrix.browser }} + name: visual-regression-diffs-shard-${{ matrix.shard }} path: visual-regression-output/ retention-days: 14 diff --git a/jest.visual.config.js b/jest.visual.config.js index a4d06d46ae..69a5b71f8b 100644 --- a/jest.visual.config.js +++ b/jest.visual.config.js @@ -21,5 +21,5 @@ module.exports = { globalTeardown: '/build-tools/visual/global-teardown.js', setupFilesAfterEnv: [path.join(__dirname, 'build-tools', 'visual', 'setup.js')], moduleFileExtensions: ['js', 'ts'], - testMatch: ['/test/visual.test.ts'], + testMatch: ['/test/visual/**/*.test.ts'], }; diff --git a/test/visual.test.ts b/test/visual.test.ts deleted file mode 100644 index 4e6e992f4f..0000000000 --- a/test/visual.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { allSuites } from './definitions'; -import { runTestSuites } from './definitions/utils'; - -runTestSuites(allSuites); diff --git a/test/visual/action-card.test.ts b/test/visual/action-card.test.ts new file mode 100644 index 0000000000..8505dad23b --- /dev/null +++ b/test/visual/action-card.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import actionCard from '../definitions/visual/action-card'; + +runTestSuites([actionCard]); diff --git a/test/visual/alert.test.ts b/test/visual/alert.test.ts new file mode 100644 index 0000000000..431e4907e4 --- /dev/null +++ b/test/visual/alert.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import alert from '../definitions/visual/alert'; + +runTestSuites([alert]); diff --git a/test/visual/app-layout.test.ts b/test/visual/app-layout.test.ts new file mode 100644 index 0000000000..93500354cc --- /dev/null +++ b/test/visual/app-layout.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import appLayout from '../definitions/visual/app-layout'; + +runTestSuites([appLayout]); diff --git a/tsconfig.integ.json b/tsconfig.integ.json index e23c746e71..b3f5080284 100644 --- a/tsconfig.integ.json +++ b/tsconfig.integ.json @@ -12,5 +12,5 @@ "resolveJsonModule": true, "moduleResolution": "node" }, - "include": ["**/__integ__/**/*.ts", "**/__a11y__/**/*.ts", "**/__motion__/**/*.ts", "test/definitions/utils.ts", "test/visual.test.ts", "types"] + "include": ["**/__integ__/**/*.ts", "**/__a11y__/**/*.ts", "**/__motion__/**/*.ts", "test/definitions/utils.ts", "test/visual/**/*.test.ts", "types"] } From c33545b6366aaf56795c40428512ee9012869982 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 29 May 2026 11:55:07 +0200 Subject: [PATCH 59/81] Fix config --- jest.visual.config.js | 2 +- tsconfig.integ.json | 3 +-- tsconfig.visual.json | 7 +++++++ 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 tsconfig.visual.json diff --git a/jest.visual.config.js b/jest.visual.config.js index 69a5b71f8b..bd04fe5d73 100644 --- a/jest.visual.config.js +++ b/jest.visual.config.js @@ -10,7 +10,7 @@ module.exports = { '^.+\\.tsx?$': [ 'ts-jest', { - tsconfig: 'tsconfig.integ.json', + tsconfig: 'tsconfig.visual.json', }, ], }, diff --git a/tsconfig.integ.json b/tsconfig.integ.json index b3f5080284..e816f749e9 100644 --- a/tsconfig.integ.json +++ b/tsconfig.integ.json @@ -5,12 +5,11 @@ "types": ["jest"], "noEmit": true, "strict": true, - "isolatedModules": true, "sourceMap": true, "allowSyntheticDefaultImports": true, "esModuleInterop": true, "resolveJsonModule": true, "moduleResolution": "node" }, - "include": ["**/__integ__/**/*.ts", "**/__a11y__/**/*.ts", "**/__motion__/**/*.ts", "test/definitions/utils.ts", "test/visual/**/*.test.ts", "types"] + "include": ["**/__integ__/**/*.ts", "**/__a11y__/**/*.ts", "**/__motion__/**/*.ts", "types"] } diff --git a/tsconfig.visual.json b/tsconfig.visual.json new file mode 100644 index 0000000000..41f554c24f --- /dev/null +++ b/tsconfig.visual.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.integ.json", + "compilerOptions": { + "isolatedModules": true + }, + "include": ["test/definitions/utils.ts", "test/visual/**/*.test.ts", "types"] +} From 790b506632a1b27b23975a8702227d645dd2c934 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 29 May 2026 12:03:21 +0200 Subject: [PATCH 60/81] Fix config --- tsconfig.visual.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/tsconfig.visual.json b/tsconfig.visual.json index 41f554c24f..be61d962ef 100644 --- a/tsconfig.visual.json +++ b/tsconfig.visual.json @@ -1,7 +1,4 @@ { "extends": "./tsconfig.integ.json", - "compilerOptions": { - "isolatedModules": true - }, "include": ["test/definitions/utils.ts", "test/visual/**/*.test.ts", "types"] } From eaa391101386b5468bccae3f058c4c3d3968c1de Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 29 May 2026 15:15:22 +0200 Subject: [PATCH 61/81] Increase timeout --- jest.visual.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jest.visual.config.js b/jest.visual.config.js index bd04fe5d73..19418a86dd 100644 --- a/jest.visual.config.js +++ b/jest.visual.config.js @@ -15,7 +15,7 @@ module.exports = { ], }, reporters: ['default', 'github-actions'], - testTimeout: 120_000, // 2min — pages can be tall and slow to capture + testTimeout: 240_000, // 4min — pages can be tall and slow to capture maxWorkers: os.cpus().length * (process.env.GITHUB_ACTION ? 3 : 1), globalSetup: '/build-tools/visual/global-setup.js', globalTeardown: '/build-tools/visual/global-teardown.js', From ef58b90452229fd1dcf12c4dddcb883f05cf0f2b Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 29 May 2026 15:32:46 +0200 Subject: [PATCH 62/81] Use 3 shards --- .github/workflows/visual-regression.yml | 4 ++-- test/definitions/visual/app-layout.ts | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 86f0fdc7ea..e004f354d7 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -120,7 +120,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4] + shard: [1, 2, 3] steps: - uses: actions/checkout@v4 @@ -161,7 +161,7 @@ jobs: run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081 - name: Run visual regression tests - run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js --shard=${{ matrix.shard }}/4 + run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js --shard=${{ matrix.shard }}/3 env: TZ: UTC diff --git a/test/definitions/visual/app-layout.ts b/test/definitions/visual/app-layout.ts index 936f07152f..12c76d9138 100644 --- a/test/definitions/visual/app-layout.ts +++ b/test/definitions/visual/app-layout.ts @@ -1,8 +1,11 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import createWrapper from '../../../lib/components/test-utils/selectors'; import { TestDefinition, TestSuite } from '../types'; +const wrapper = createWrapper(); + function responsiveTests(width: number): TestSuite { return { description: `width ${width}px`, @@ -257,7 +260,7 @@ const suite: TestSuite = { path: 'app-layout/with-drawers', screenshotType: 'screenshotArea', setup: async page => { - await page.click('[aria-label="Pro help trigger button"]'); + await page.click(wrapper.findAppLayout().findDrawerTriggerById('pro-help').toSelector()); }, }, { @@ -265,7 +268,7 @@ const suite: TestSuite = { path: 'app-layout/with-drawers', screenshotType: 'screenshotArea', setup: async page => { - await page.hoverElement('[aria-label="Pro help trigger button"]'); + await page.hoverElement(wrapper.findAppLayout().findDrawerTriggerById('pro-help').toSelector()); }, }, { @@ -274,7 +277,7 @@ const suite: TestSuite = { screenshotType: 'screenshotArea', queryParams: { sideNavFill: 'false' }, setup: async page => { - await page.click('[aria-label="Chat trigger button"]'); + await page.click(wrapper.findAppLayout().findDrawerTriggerById('chat').toSelector()); }, }, ], From ab2ffefc4d34ffb71b4c5079394b1b5dc7532c7c Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 29 May 2026 15:32:58 +0200 Subject: [PATCH 63/81] Fix tests --- test/definitions/utils.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/definitions/utils.ts b/test/definitions/utils.ts index a1e8a3bd1f..c5b237dc73 100644 --- a/test/definitions/utils.ts +++ b/test/definitions/utils.ts @@ -63,8 +63,12 @@ async function captureScreenshotArea( browser: WebdriverIO.Browser, page: ScreenshotPageObject, url: string, + windowSize: { width: number; height: number } | undefined, setup?: (page: ScreenshotPageObject) => Promise ): Promise { + if (windowSize) { + await browser.setWindowSize(windowSize.width, windowSize.height); + } await browser.url(url); await page.waitForVisible(screenshotAreaSelector); if (setup) { @@ -76,18 +80,15 @@ async function captureScreenshotArea( function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Browser) { test(testDef.description, async () => { const browser = getBrowser(); - if (testDef.configuration) { - const windowSize = { ...defaultWindowSize, ...testDef.configuration }; - await browser.setWindowSize(windowSize.width, windowSize.height); - } + const windowSize = testDef.configuration ? { ...defaultWindowSize, ...testDef.configuration } : undefined; const page = new ScreenshotPageObject(browser); const newUrl = buildUrl(newHost, testDef.path, testDef.queryParams); const oldUrl = buildUrl(oldHost, testDef.path, testDef.queryParams); // Fast path: compare the screenshot area (viewport-only, no scroll-and-merge). - const newScreenshot = await captureScreenshotArea(browser, page, newUrl, testDef.setup); - const oldScreenshot = await captureScreenshotArea(browser, page, oldUrl, testDef.setup); + const newScreenshot = await captureScreenshotArea(browser, page, newUrl, windowSize, testDef.setup); + const oldScreenshot = await captureScreenshotArea(browser, page, oldUrl, windowSize, testDef.setup); const { diffPixels } = await cropAndCompare(newScreenshot, oldScreenshot); if (diffPixels === 0) { From 2462deb3461fdfd38a9ed99355c0d829e7c89f44 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 29 May 2026 23:56:13 +0200 Subject: [PATCH 64/81] Build the test selectors before running the tests --- .github/workflows/visual-regression.yml | 38 ++++++++++++++++--------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index e004f354d7..ed5739b0e3 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -4,12 +4,12 @@ on: workflow_call: inputs: pr-artifact-name: - description: 'Name of the artifact containing PR pages (built by quick-build job). If not provided, pages will be built locally.' - required: false + description: 'Name of the artifact containing PR pages (built by the caller workflow).' + required: true type: string caller-run-id: description: 'The run ID of the calling workflow, used to download artifacts it uploaded.' - required: false + required: true type: string defaults: @@ -22,8 +22,8 @@ permissions: actions: read jobs: - # Stage the PR pages within this run so matrix jobs can download them without - # needing cross-run artifact access. Runs in parallel with build-baseline. + # Stage the PR pages and build test utils for the visual regression shards. + # Runs in parallel with build-baseline. stage-pr-pages: name: Stage PR pages runs-on: ubuntu-latest @@ -39,16 +39,7 @@ jobs: - name: Install dependencies run: npm i - - name: Build PR pages locally - if: ${{ !inputs.pr-artifact-name }} - run: | - npx gulp quick-build - node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-default - env: - NODE_ENV: production - - name: Download PR pages artifact from caller run - if: ${{ inputs.pr-artifact-name }} uses: actions/download-artifact@v4 with: name: ${{ inputs.pr-artifact-name }} @@ -56,6 +47,11 @@ jobs: github-token: ${{ github.token }} run-id: ${{ inputs.caller-run-id }} + - name: Build (test utils selectors) + run: npx gulp quick-build + env: + NODE_ENV: production + - name: Upload PR pages artifact (for matrix jobs) uses: actions/upload-artifact@v4 with: @@ -63,6 +59,13 @@ jobs: path: pages/lib/static-default retention-days: 1 + - name: Upload test utils artifact + uses: actions/upload-artifact@v4 + with: + name: visual-test-utils + path: lib/components/test-utils + retention-days: 1 + # Build the baseline (main branch) pages once and share them across all browser jobs. # Runs in parallel with stage-pr-pages. build-baseline: @@ -150,7 +153,14 @@ jobs: name: visual-baseline-pages path: pages/lib/static-visual-baseline + - name: Download test utils artifact + uses: actions/download-artifact@v4 + with: + name: visual-test-utils + path: lib/components/test-utils + # ── Run tests ───────────────────────────────────────────────────────── + - name: Start test server (port 8080) run: npx --yes serve --no-clipboard --listen 8080 pages/lib/static-default & From 507f57c56b4fdcceb007c491f0775892bb6dc804 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Sat, 30 May 2026 00:35:48 +0200 Subject: [PATCH 65/81] Optimizations --- .github/workflows/deploy.yml | 8 +++++++ .github/workflows/visual-regression.yml | 31 ++++++------------------- 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2add3494cd..df56701906 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,6 +53,13 @@ jobs: name: dev-pages-react${{ matrix.react }} path: pages/lib/static-default + - name: Upload test utils selectors artifact + if: matrix.react == 18 + uses: actions/upload-artifact@v4 + with: + name: test-utils-selectors + path: lib/components/test-utils + deploy: needs: quick-build name: deploy${{ matrix.react != 16 && format(' (React {0})', matrix.react) || '' }} @@ -74,4 +81,5 @@ jobs: secrets: inherit with: pr-artifact-name: dev-pages-react18 + test-utils-artifact-name: test-utils-selectors caller-run-id: ${{ github.run_id }} diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index ed5739b0e3..71349ddb28 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -7,6 +7,10 @@ on: description: 'Name of the artifact containing PR pages (built by the caller workflow).' required: true type: string + test-utils-artifact-name: + description: 'Name of the artifact containing test-utils selectors.' + required: true + type: string caller-run-id: description: 'The run ID of the calling workflow, used to download artifacts it uploaded.' required: true @@ -28,17 +32,6 @@ jobs: name: Stage PR pages runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: npm - - - name: Install dependencies - run: npm i - - name: Download PR pages artifact from caller run uses: actions/download-artifact@v4 with: @@ -47,11 +40,6 @@ jobs: github-token: ${{ github.token }} run-id: ${{ inputs.caller-run-id }} - - name: Build (test utils selectors) - run: npx gulp quick-build - env: - NODE_ENV: production - - name: Upload PR pages artifact (for matrix jobs) uses: actions/upload-artifact@v4 with: @@ -59,13 +47,6 @@ jobs: path: pages/lib/static-default retention-days: 1 - - name: Upload test utils artifact - uses: actions/upload-artifact@v4 - with: - name: visual-test-utils - path: lib/components/test-utils - retention-days: 1 - # Build the baseline (main branch) pages once and share them across all browser jobs. # Runs in parallel with stage-pr-pages. build-baseline: @@ -156,8 +137,10 @@ jobs: - name: Download test utils artifact uses: actions/download-artifact@v4 with: - name: visual-test-utils + name: ${{ inputs.test-utils-artifact-name }} path: lib/components/test-utils + github-token: ${{ github.token }} + run-id: ${{ inputs.caller-run-id }} # ── Run tests ───────────────────────────────────────────────────────── From 09157563c9ae021cffdfebaec5399ac951016b42 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Sat, 30 May 2026 00:38:30 +0200 Subject: [PATCH 66/81] Upload the entire lib/components directory --- .github/workflows/deploy.yml | 2 +- .github/workflows/visual-regression.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index df56701906..36964ddc1f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -58,7 +58,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: test-utils-selectors - path: lib/components/test-utils + path: lib/components deploy: needs: quick-build diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 71349ddb28..b1333cf1f4 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -138,7 +138,7 @@ jobs: uses: actions/download-artifact@v4 with: name: ${{ inputs.test-utils-artifact-name }} - path: lib/components/test-utils + path: lib/components github-token: ${{ github.token }} run-id: ${{ inputs.caller-run-id }} From 3909a3f9b8624f997e691347e0f781444158faa3 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Sat, 30 May 2026 00:48:38 +0200 Subject: [PATCH 67/81] Remove unnecessary step --- .github/workflows/visual-regression.yml | 28 ++++--------------------- 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index b1333cf1f4..a35b2fa877 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -26,29 +26,7 @@ permissions: actions: read jobs: - # Stage the PR pages and build test utils for the visual regression shards. - # Runs in parallel with build-baseline. - stage-pr-pages: - name: Stage PR pages - runs-on: ubuntu-latest - steps: - - name: Download PR pages artifact from caller run - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.pr-artifact-name }} - path: pages/lib/static-default - github-token: ${{ github.token }} - run-id: ${{ inputs.caller-run-id }} - - - name: Upload PR pages artifact (for matrix jobs) - uses: actions/upload-artifact@v4 - with: - name: visual-pr-pages - path: pages/lib/static-default - retention-days: 1 - # Build the baseline (main branch) pages once and share them across all browser jobs. - # Runs in parallel with stage-pr-pages. build-baseline: name: Build baseline pages runs-on: ubuntu-latest @@ -99,7 +77,7 @@ jobs: visual: name: Visual regression (shard ${{ matrix.shard }}) - needs: [stage-pr-pages, build-baseline] + needs: [build-baseline] runs-on: ubuntu-latest strategy: fail-fast: false @@ -125,8 +103,10 @@ jobs: - name: Download PR pages artifact uses: actions/download-artifact@v4 with: - name: visual-pr-pages + name: ${{ inputs.pr-artifact-name }} path: pages/lib/static-default + github-token: ${{ github.token }} + run-id: ${{ inputs.caller-run-id }} - name: Download baseline artifact uses: actions/download-artifact@v4 From e37268b445dd8e4d7c01a5fb0f536d02b2eb4a25 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Sat, 30 May 2026 01:07:41 +0200 Subject: [PATCH 68/81] Add more components tests --- .github/workflows/visual-regression.yml | 4 +- test/definitions/index.ts | 6 +- test/definitions/visual/area-chart.ts | 136 ++++++++++++++++++++ test/definitions/visual/attribute-editor.ts | 31 +++++ test/definitions/visual/autosuggest.ts | 70 ++++++++++ test/definitions/visual/badge.ts | 23 ++++ test/visual/area-chart.test.ts | 6 + test/visual/attribute-editor.test.ts | 5 + test/visual/autosuggest.test.ts | 6 + test/visual/badge.test.ts | 6 + 10 files changed, 290 insertions(+), 3 deletions(-) create mode 100644 test/definitions/visual/area-chart.ts create mode 100644 test/definitions/visual/attribute-editor.ts create mode 100644 test/definitions/visual/autosuggest.ts create mode 100644 test/definitions/visual/badge.ts create mode 100644 test/visual/area-chart.test.ts create mode 100644 test/visual/attribute-editor.test.ts create mode 100644 test/visual/autosuggest.test.ts create mode 100644 test/visual/badge.test.ts diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index a35b2fa877..adb4d3ed5b 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -82,7 +82,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3] + shard: [1, 2, 3, 4, 5, 6] steps: - uses: actions/checkout@v4 @@ -134,7 +134,7 @@ jobs: run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081 - name: Run visual regression tests - run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js --shard=${{ matrix.shard }}/3 + run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js --shard=${{ matrix.shard }}/6 env: TZ: UTC diff --git a/test/definitions/index.ts b/test/definitions/index.ts index 6ff236cfa6..5f6d790d65 100644 --- a/test/definitions/index.ts +++ b/test/definitions/index.ts @@ -7,5 +7,9 @@ import { TestSuite } from './types'; import actionCard from './visual/action-card'; import alert from './visual/alert'; import appLayout from './visual/app-layout'; +import areaChart from './visual/area-chart'; +import attributeEditor from './visual/attribute-editor'; +import autosuggest from './visual/autosuggest'; +import badge from './visual/badge'; -export const allSuites: TestSuite[] = [actionCard, alert, appLayout]; +export const allSuites: TestSuite[] = [actionCard, alert, appLayout, areaChart, attributeEditor, autosuggest, badge]; diff --git a/test/definitions/visual/area-chart.ts b/test/definitions/visual/area-chart.ts new file mode 100644 index 0000000000..10855239f6 --- /dev/null +++ b/test/definitions/visual/area-chart.ts @@ -0,0 +1,136 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const TEST_CHART_FILTER_TRIGGER = '#linear-latency-chart button'; +const TEST_CHART_TOOLTIP_HEADER = '#linear-latency-chart h2'; + +const suite: TestSuite = { + description: 'Area chart', + componentName: 'area-chart', + tests: [ + { + description: 'permutations', + path: 'area-chart/permutations', + screenshotType: 'permutations', + }, + { + description: 'fit-height', + path: 'area-chart/fit-height', + screenshotType: 'screenshotArea', + }, + { + description: 'fit-height no filter, no legend', + path: 'area-chart/fit-height', + screenshotType: 'screenshotArea', + queryParams: { hideFilter: 'true', hideLegend: 'true' }, + }, + { + description: 'fit-height, no legend', + path: 'area-chart/fit-height', + screenshotType: 'screenshotArea', + queryParams: { hideLegend: 'true' }, + }, + { + description: 'chart plot has a focus outline', + path: 'area-chart/test', + screenshotType: 'screenshotArea', + configuration: { width: 800, height: 800 }, + setup: async page => { + await page.click(TEST_CHART_FILTER_TRIGGER); + await page.keys(['Escape']); + await page.focusNextElement(); + }, + }, + { + description: 'can navigate along X axis highlighting all series with keyboard', + path: 'area-chart/test', + screenshotType: 'screenshotArea', + configuration: { width: 800, height: 800 }, + setup: async page => { + await page.click(TEST_CHART_FILTER_TRIGGER); + await page.keys(['Escape']); + await page.focusNextElement(); + await page.keys(['ArrowRight', 'ArrowRight']); + await page.waitForVisible(TEST_CHART_TOOLTIP_HEADER); + }, + }, + { + description: 'can navigate a specific series with keyboard', + path: 'area-chart/test', + screenshotType: 'screenshotArea', + configuration: { width: 800, height: 800 }, + setup: async page => { + await page.click(TEST_CHART_FILTER_TRIGGER); + await page.keys(['Escape']); + await page.focusNextElement(); + await page.keys(['ArrowRight']); + await page.keys(['ArrowDown']); + await page.keys(['ArrowRight']); + await page.waitForVisible(TEST_CHART_TOOLTIP_HEADER); + }, + }, + { + description: 'selects correct series when navigated back from legend', + path: 'area-chart/test', + screenshotType: 'screenshotArea', + configuration: { width: 800, height: 800 }, + setup: async page => { + await page.click(TEST_CHART_FILTER_TRIGGER); + await page.keys(['Escape']); + await page.keys(['Tab']); + await page.keys(['Tab']); + await page.keys(['ArrowRight']); + await page.keys(['Shift', 'Tab']); + await page.keys(['ArrowRight']); + await page.waitForVisible(TEST_CHART_TOOLTIP_HEADER); + }, + }, + { + description: 'can pin popover for all data points at a given X coordinate with keyboard', + path: 'area-chart/test', + screenshotType: 'screenshotArea', + configuration: { width: 800, height: 800 }, + setup: async page => { + await page.click(TEST_CHART_FILTER_TRIGGER); + await page.keys(['Escape']); + await page.focusNextElement(); + await page.keys(['ArrowRight']); + await page.keys(['ArrowRight']); + await page.waitForVisible(TEST_CHART_TOOLTIP_HEADER); + await page.keys(['Enter']); + await page.waitForVisible('[aria-label="Dismiss"]'); + }, + }, + { + description: 'can pin popover for a point in a specific series with keyboard', + path: 'area-chart/test', + screenshotType: 'screenshotArea', + configuration: { width: 800, height: 800 }, + setup: async page => { + await page.click(TEST_CHART_FILTER_TRIGGER); + await page.keys(['Escape']); + await page.focusNextElement(); + await page.keys(['ArrowRight']); + await page.keys(['ArrowDown']); + await page.keys(['ArrowRight']); + await page.waitForVisible(TEST_CHART_TOOLTIP_HEADER); + await page.keys(['Enter']); + await page.waitForVisible('[aria-label="Dismiss"]'); + }, + }, + { + description: 'shows popover on hover', + path: 'area-chart/test', + screenshotType: 'screenshotArea', + configuration: { width: 800, height: 800 }, + setup: async page => { + await page.hoverElement('[aria-label="Linear latency chart"]', 200, 50); + await page.waitForVisible(TEST_CHART_TOOLTIP_HEADER); + }, + }, + ], +}; + +export default suite; diff --git a/test/definitions/visual/attribute-editor.ts b/test/definitions/visual/attribute-editor.ts new file mode 100644 index 0000000000..33da3fad86 --- /dev/null +++ b/test/definitions/visual/attribute-editor.ts @@ -0,0 +1,31 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Attribute Editor', + componentName: 'attribute-editor', + tests: [360, 768, 992].flatMap(width => [ + { + description: `permutations at ${width}px`, + path: 'attribute-editor/permutations', + screenshotType: 'permutations' as const, + configuration: { width }, + }, + { + description: `customizable-footer at ${width}px`, + path: 'attribute-editor/customizable-footer', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + }, + { + description: `with long select at ${width}px`, + path: 'attribute-editor/select-with-long-value', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + }, + ]), +}; + +export default suite; diff --git a/test/definitions/visual/autosuggest.ts b/test/definitions/visual/autosuggest.ts new file mode 100644 index 0000000000..92f4d46897 --- /dev/null +++ b/test/definitions/visual/autosuggest.ts @@ -0,0 +1,70 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import createWrapper from '../../../lib/components/test-utils/selectors'; +import { TestDefinition, TestSuite } from '../types'; + +const wrapper = createWrapper(); + +const suite: TestSuite = { + description: 'Autosuggest', + componentName: 'autosuggest', + tests: [ + { + description: 'permutations', + path: 'autosuggest/permutations', + screenshotType: 'permutations', + setup: async page => { + await page.click('input'); + }, + }, + { + description: 'permutations for async properties', + path: 'autosuggest/permutations-async', + screenshotType: 'permutations', + setup: async page => { + await page.click('input'); + }, + }, + { + description: 'Displays options with groups correctly', + path: 'autosuggest/scenarios', + screenshotType: 'screenshotArea', + setup: async page => { + await page.click('input'); + }, + }, + { + description: 'Correctly displays dropdown regions', + path: 'autosuggest/regions-scenarios', + screenshotType: 'screenshotArea', + setup: async page => { + await page.click('input'); + }, + }, + { + description: 'Long virtual list - navigate to last item', + path: 'autosuggest/virtual-scroll', + screenshotType: 'screenshotArea', + setup: async page => { + await page.click(wrapper.findAutosuggest().findNativeInput().toSelector()); + await page.keys(['ArrowUp']); + }, + }, + ...[true, false].map( + virtualScroll => + ({ + description: `with custom renderOption (virtualScroll=${virtualScroll})`, + path: 'autosuggest/custom-render-option', + screenshotType: 'screenshotArea' as const, + queryParams: { virtualScroll: String(virtualScroll) }, + setup: async page => { + await page.click(wrapper.findAutosuggest().findNativeInput().toSelector()); + await page.keys(['ArrowDown']); + }, + }) as TestDefinition + ), + ], +}; + +export default suite; diff --git a/test/definitions/visual/badge.ts b/test/definitions/visual/badge.ts new file mode 100644 index 0000000000..7d405ed843 --- /dev/null +++ b/test/definitions/visual/badge.ts @@ -0,0 +1,23 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Badge', + componentName: 'badge', + tests: [ + { + description: 'permutation page', + path: 'badge/permutations', + screenshotType: 'permutations', + }, + { + description: 'style custom page', + path: 'badge/style-custom-types', + screenshotType: 'screenshotArea', + }, + ], +}; + +export default suite; diff --git a/test/visual/area-chart.test.ts b/test/visual/area-chart.test.ts new file mode 100644 index 0000000000..51d6631016 --- /dev/null +++ b/test/visual/area-chart.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import areaChart from '../definitions/visual/area-chart'; + +runTestSuites([areaChart]); diff --git a/test/visual/attribute-editor.test.ts b/test/visual/attribute-editor.test.ts new file mode 100644 index 0000000000..f57e8785b2 --- /dev/null +++ b/test/visual/attribute-editor.test.ts @@ -0,0 +1,5 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import attributeEditor from '../definitions/visual/attribute-editor'; +runTestSuites([attributeEditor]); diff --git a/test/visual/autosuggest.test.ts b/test/visual/autosuggest.test.ts new file mode 100644 index 0000000000..454c00c648 --- /dev/null +++ b/test/visual/autosuggest.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import autosuggest from '../definitions/visual/autosuggest'; + +runTestSuites([autosuggest]); diff --git a/test/visual/badge.test.ts b/test/visual/badge.test.ts new file mode 100644 index 0000000000..6b24838a42 --- /dev/null +++ b/test/visual/badge.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import badge from '../definitions/visual/badge'; + +runTestSuites([badge]); From 41c993d557b6c33832254214347cfd3ecc639542 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Sat, 30 May 2026 08:36:55 +0200 Subject: [PATCH 69/81] Split app layout test definition files --- .github/workflows/visual-regression.yml | 4 +- test/definitions/index.ts | 28 +- .../visual/app-layout-content-paddings.ts | 30 + test/definitions/visual/app-layout-drawers.ts | 41 ++ .../definitions/visual/app-layout-flashbar.ts | 39 ++ test/definitions/visual/app-layout-header.ts | 85 +++ test/definitions/visual/app-layout-multi.ts | 25 + .../visual/app-layout-responsive.ts | 169 ++++++ ...-layout-sticky-table-header-split-panel.ts | 78 +++ test/definitions/visual/app-layout-toolbar.ts | 23 + test/definitions/visual/app-layout-z-index.ts | 60 ++ test/definitions/visual/app-layout.ts | 539 +----------------- .../app-layout-content-paddings.test.ts | 6 + test/visual/app-layout-drawers.test.ts | 6 + test/visual/app-layout-flashbar.test.ts | 6 + test/visual/app-layout-header.test.ts | 6 + test/visual/app-layout-multi.test.ts | 6 + test/visual/app-layout-responsive.test.ts | 6 + ...ut-sticky-table-header-split-panel.test.ts | 6 + test/visual/app-layout-toolbar.test.ts | 6 + test/visual/app-layout-z-index.test.ts | 6 + test/visual/app-layout.test.ts | 4 +- 22 files changed, 662 insertions(+), 517 deletions(-) create mode 100644 test/definitions/visual/app-layout-content-paddings.ts create mode 100644 test/definitions/visual/app-layout-drawers.ts create mode 100644 test/definitions/visual/app-layout-flashbar.ts create mode 100644 test/definitions/visual/app-layout-header.ts create mode 100644 test/definitions/visual/app-layout-multi.ts create mode 100644 test/definitions/visual/app-layout-responsive.ts create mode 100644 test/definitions/visual/app-layout-sticky-table-header-split-panel.ts create mode 100644 test/definitions/visual/app-layout-toolbar.ts create mode 100644 test/definitions/visual/app-layout-z-index.ts create mode 100644 test/visual/app-layout-content-paddings.test.ts create mode 100644 test/visual/app-layout-drawers.test.ts create mode 100644 test/visual/app-layout-flashbar.test.ts create mode 100644 test/visual/app-layout-header.test.ts create mode 100644 test/visual/app-layout-multi.test.ts create mode 100644 test/visual/app-layout-responsive.test.ts create mode 100644 test/visual/app-layout-sticky-table-header-split-panel.test.ts create mode 100644 test/visual/app-layout-toolbar.test.ts create mode 100644 test/visual/app-layout-z-index.test.ts diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index adb4d3ed5b..8e03c9967c 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -82,7 +82,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6] + shard: [1, 2, 3, 4, 5, 6, 7, 8] steps: - uses: actions/checkout@v4 @@ -134,7 +134,7 @@ jobs: run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081 - name: Run visual regression tests - run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js --shard=${{ matrix.shard }}/6 + run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js --shard=${{ matrix.shard }}/8 env: TZ: UTC diff --git a/test/definitions/index.ts b/test/definitions/index.ts index 5f6d790d65..17dc1902a0 100644 --- a/test/definitions/index.ts +++ b/test/definitions/index.ts @@ -7,9 +7,35 @@ import { TestSuite } from './types'; import actionCard from './visual/action-card'; import alert from './visual/alert'; import appLayout from './visual/app-layout'; +import appLayoutContentPaddings from './visual/app-layout-content-paddings'; +import appLayoutDrawers from './visual/app-layout-drawers'; +import appLayoutFlashbar from './visual/app-layout-flashbar'; +import appLayoutHeader from './visual/app-layout-header'; +import appLayoutMulti from './visual/app-layout-multi'; +import appLayoutResponsive from './visual/app-layout-responsive'; +import appLayoutStickyTableHeaderSplitPanel from './visual/app-layout-sticky-table-header-split-panel'; +import appLayoutToolbar from './visual/app-layout-toolbar'; +import appLayoutZIndex from './visual/app-layout-z-index'; import areaChart from './visual/area-chart'; import attributeEditor from './visual/attribute-editor'; import autosuggest from './visual/autosuggest'; import badge from './visual/badge'; -export const allSuites: TestSuite[] = [actionCard, alert, appLayout, areaChart, attributeEditor, autosuggest, badge]; +export const allSuites: TestSuite[] = [ + actionCard, + alert, + appLayout, + appLayoutContentPaddings, + appLayoutDrawers, + appLayoutFlashbar, + appLayoutHeader, + appLayoutMulti, + appLayoutResponsive, + appLayoutStickyTableHeaderSplitPanel, + appLayoutToolbar, + appLayoutZIndex, + areaChart, + attributeEditor, + autosuggest, + badge, +]; diff --git a/test/definitions/visual/app-layout-content-paddings.ts b/test/definitions/visual/app-layout-content-paddings.ts new file mode 100644 index 0000000000..068e5dd08e --- /dev/null +++ b/test/definitions/visual/app-layout-content-paddings.ts @@ -0,0 +1,30 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Content paddings', + componentName: 'app-layout', + tests: [ + ...(['true', 'false'] as const).flatMap(toolsEnabled => + (['true', 'false'] as const).flatMap(splitPanelEnabled => + (['bottom', 'side'] as const).map(splitPanelPosition => ({ + description: `toolsEnabled=${toolsEnabled} splitPanelEnabled=${splitPanelEnabled} splitPanelPosition=${splitPanelPosition}`, + path: 'app-layout/with-split-panel', + screenshotType: 'screenshotArea' as const, + queryParams: { toolsEnabled, splitPanelEnabled, splitPanelPosition }, + })) + ) + ), + ...[1500, 600].map(width => ({ + description: `with split panel and disabled content paddings - width=${width}`, + path: 'app-layout/disable-paddings-with-split-panel', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + queryParams: { splitPanelOpen: 'true', splitPanelPosition: 'side' }, + })), + ], +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-drawers.ts b/test/definitions/visual/app-layout-drawers.ts new file mode 100644 index 0000000000..2af69c2ca1 --- /dev/null +++ b/test/definitions/visual/app-layout-drawers.ts @@ -0,0 +1,41 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import createWrapper from '../../../lib/components/test-utils/selectors'; +import { TestSuite } from '../types'; + +const wrapper = createWrapper(); + +const suite: TestSuite = { + description: 'Drawers', + componentName: 'app-layout', + tests: [ + { + description: 'with split panel', + path: 'app-layout/with-drawers', + screenshotType: 'screenshotArea', + setup: async page => { + await page.click(wrapper.findAppLayout().findDrawerTriggerById('pro-help').toSelector()); + }, + }, + { + description: 'with tooltip on hover', + path: 'app-layout/with-drawers', + screenshotType: 'screenshotArea', + setup: async page => { + await page.hoverElement(wrapper.findAppLayout().findDrawerTriggerById('pro-help').toSelector()); + }, + }, + { + description: 'with custom scrollable drawer content', + path: 'app-layout/with-drawers-scrollable', + screenshotType: 'screenshotArea', + queryParams: { sideNavFill: 'false' }, + setup: async page => { + await page.click(wrapper.findAppLayout().findDrawerTriggerById('chat').toSelector()); + }, + }, + ], +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-flashbar.ts b/test/definitions/visual/app-layout-flashbar.ts new file mode 100644 index 0000000000..5849a5cbf5 --- /dev/null +++ b/test/definitions/visual/app-layout-flashbar.ts @@ -0,0 +1,39 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Flashbar', + componentName: 'app-layout', + tests: [true, false].flatMap(disableContentPaddings => + [true, false].flatMap(stickyNotifications => + [true, false].flatMap(stickyTableHeader => + [true, false].map(stackNotifications => ({ + description: `disableContentPaddings: ${disableContentPaddings}, stickyNotifications: ${stickyNotifications}, stickyTableHeader: ${stickyTableHeader}, stackNotifications: ${stackNotifications}`, + path: 'app-layout/with-stacked-notifications-and-table', + screenshotType: 'screenshotArea' as const, + configuration: { width: 1280, height: 900 }, + setup: async (page: import('@cloudscape-design/browser-test-tools/page-objects').ScreenshotPageObject) => { + if (!disableContentPaddings) { + await page.click('[data-id="toggle-content-paddings"]'); + } + if (stickyNotifications) { + await page.click('[data-id="toggle-sticky-notifications"]'); + } + if (!stickyTableHeader) { + await page.click('[data-id="toggle-sticky-table-header"]'); + } + if (!stackNotifications) { + await page.click('[data-id="toggle-stack-items"]'); + } + await page.click('[data-id="add-notification"]'); + await page.click('[data-id="add-notification"]'); + }, + })) + ) + ) + ), +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-header.ts b/test/definitions/visual/app-layout-header.ts new file mode 100644 index 0000000000..55f674860c --- /dev/null +++ b/test/definitions/visual/app-layout-header.ts @@ -0,0 +1,85 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestDefinition, TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Headers', + componentName: 'app-layout', + tests: [ + // ── Headers ─────────────────────────────────────────────────────────── + { + description: 'Headers', + tests: [600, 1280].flatMap(width => [ + { + description: `alignment with full-page table (${width}px)`, + path: 'app-layout/with-table', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + }, + { + description: `alignment with full-page table in sticky state (${width}px)`, + path: 'app-layout/with-table', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + setup: async page => { + await page.windowScrollTo({ top: 200 }); + }, + }, + { + description: `alignment with full-page table in sticky state with sticky notifications (${width}px)`, + path: 'app-layout/with-table', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + queryParams: { stickyNotifications: 'true' }, + setup: async page => { + await page.windowScrollTo({ top: 200 }); + }, + }, + { + description: `high contrast header variant in landing page (${width}px)`, + path: 'app-layout/landing-page', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + }, + ]), + }, + + // ── High contrast header variant ────────────────────────────────────── + { + description: 'High contrast header variant', + tests: [ + ...[1400, 600].flatMap(width => [ + { + description: `with breadcrumbs and notifications at ${width}px`, + path: 'app-layout/high-contrast-header-variant', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + queryParams: { hasBreadcrumbs: 'true', hasNotifications: 'true', hasContainer: 'true' }, + } as TestDefinition, + { + description: `without overlap at ${width}px`, + path: 'app-layout/high-contrast-header-variant', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + queryParams: { disableOverlap: 'true' }, + } as TestDefinition, + { + description: `with content layout at ${width}px`, + path: 'app-layout/high-contrast-header-variant', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + queryParams: { + hasBreadcrumbs: 'true', + hasNotifications: 'true', + hasContainer: 'true', + hasContentLayout: 'true', + }, + } as TestDefinition, + ]), + ], + }, + ], +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-multi.ts b/test/definitions/visual/app-layout-multi.ts new file mode 100644 index 0000000000..7efbbcd473 --- /dev/null +++ b/test/definitions/visual/app-layout-multi.ts @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Multiple instances', + componentName: 'app-layout', + tests: [600, 1280].flatMap(width => [ + { + description: `simple (${width}px)`, + path: 'app-layout/multi-layout-simple', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + }, + { + description: `iframe (${width}px)`, + path: 'app-layout/multi-layout-iframe', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + }, + ]), +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-responsive.ts b/test/definitions/visual/app-layout-responsive.ts new file mode 100644 index 0000000000..8f227a5814 --- /dev/null +++ b/test/definitions/visual/app-layout-responsive.ts @@ -0,0 +1,169 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +function responsiveTests(width: number): TestSuite { + return { + description: `width ${width}px`, + componentName: 'app-layout', + tests: [ + { + description: 'default', + path: 'app-layout/default', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'navigation drawer is open', + path: 'app-layout/with-wizard', + screenshotType: 'screenshotArea', + configuration: { width }, + setup: async page => { + await page.click('[aria-label="Open navigation"]'); + }, + }, + { + description: 'wizard', + path: 'app-layout/with-wizard', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'with wizard and table', + path: 'app-layout/with-wizard-and-table', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'with wizard, table, and breadcrumbs', + path: 'app-layout/with-wizard-and-table', + screenshotType: 'screenshotArea', + configuration: { width }, + queryParams: { hasBreadcrumbs: 'true' }, + }, + { + description: 'notifications', + path: 'app-layout/with-notifications', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'breadcrumbs', + path: 'app-layout/with-breadcrumbs', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'notifications and breadcrumbs', + path: 'app-layout/with-breadcrumbs-notifications', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'dashboard content type', + path: 'app-layout/dashboard-content-type', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'fixed header and footer', + path: 'app-layout/with-fixed-header-footer', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'disableBodyScroll - empty', + path: 'app-layout/legacy-nav-empty', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'disableBodyScroll - with content', + path: 'app-layout/legacy-nav-scrollable', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'disableBodyScroll - with split panel', + path: 'app-layout/legacy-nav-scrollable-with-split-panel', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'disable paddings', + path: 'app-layout/disable-paddings', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'disable paddings with breadcrumbs', + path: 'app-layout/disable-paddings-breadcrumbs', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'sticky notifications', + path: 'app-layout/with-sticky-notifications', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'sticky notifications scrolled down', + path: 'app-layout/with-sticky-notifications', + screenshotType: 'screenshotArea', + configuration: { width }, + setup: async page => { + await page.windowScrollTo({ top: 2000 }); + }, + }, + { + description: 'layout without panels', + path: 'app-layout/no-panels', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'layout without panels but with notifications', + path: 'app-layout/no-panels-with-notifications', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'with drawers', + path: 'app-layout/with-drawers', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'with empty drawers', + path: 'app-layout/with-drawers-empty', + screenshotType: 'screenshotArea', + configuration: { width }, + }, + { + description: 'with open drawer', + path: 'app-layout/with-drawers', + screenshotType: 'screenshotArea', + configuration: { width }, + setup: async page => { + await page.click('[aria-label="Security trigger button"]'); + }, + }, + ], + }; +} + +const suite: TestSuite = { + description: 'AppLayout responsive', + componentName: 'app-layout', + tests: [ + responsiveTests(600), + responsiveTests(1280), + responsiveTests(1400), + responsiveTests(1920), + responsiveTests(2540), + ], +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-sticky-table-header-split-panel.ts b/test/definitions/visual/app-layout-sticky-table-header-split-panel.ts new file mode 100644 index 0000000000..91628802ff --- /dev/null +++ b/test/definitions/visual/app-layout-sticky-table-header-split-panel.ts @@ -0,0 +1,78 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Sticky header with split panel', + componentName: 'app-layout', + tests: [ + { + description: 'scrolling to bottom with closed split panel (1 table row)', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'screenshotArea', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-1"]'); + await page.scrollToBottom('html'); + }, + }, + { + description: 'scrolling to bottom with closed split panel (30 table rows)', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'screenshotArea', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-30"]'); + await page.scrollToBottom('html'); + }, + }, + { + description: 'header stays sticky with open split panel (1 table row)', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'screenshotArea', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-1"]'); + await page.click('aria/Open panel'); + await page.scrollToBottom('html'); + }, + }, + { + description: 'header stays sticky with open split panel (30 table rows)', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'screenshotArea', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-30"]'); + await page.click('aria/Open panel'); + await page.scrollToBottom('html'); + }, + }, + { + description: 'header stays sticky when mounting and unmounting a second table', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'screenshotArea', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-30"]'); + await page.click('aria/Open panel'); + await page.windowScrollTo({ top: 0 }); + await page.click('aria/Close panel'); + await page.scrollToBottom('html'); + }, + }, + // ── Max content width ───────────────────────────────────────────────── + { + description: 'maxContentWidth set to Number.MAX_VALUE', + path: 'app-layout/refresh-content-width', + screenshotType: 'screenshotArea', + configuration: { width: 1280, height: 700 }, + setup: async page => { + await page.click('[data-test-id="button_width-number-max_value"]'); + }, + }, + ], +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-toolbar.ts b/test/definitions/visual/app-layout-toolbar.ts new file mode 100644 index 0000000000..dc9f5861fe --- /dev/null +++ b/test/definitions/visual/app-layout-toolbar.ts @@ -0,0 +1,23 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Toolbar', + componentName: 'app-layout', + tests: [ + { + description: 'multiple nested instances (no breadcrumbs dedup)', + path: 'app-layout-toolbar/multi-layout-with-hidden-instances', + screenshotType: 'screenshotArea', + }, + { + description: 'no toolbar', + path: 'app-layout-toolbar/without-toolbar', + screenshotType: 'screenshotArea', + }, + ], +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-z-index.ts b/test/definitions/visual/app-layout-z-index.ts new file mode 100644 index 0000000000..7765a496a9 --- /dev/null +++ b/test/definitions/visual/app-layout-z-index.ts @@ -0,0 +1,60 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestDefinition, TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Z-index', + componentName: 'app-layout', + tests: [ + ...[600, 1280].flatMap(width => [ + { + description: `button dropdown (${width}px)`, + path: 'app-layout/with-absolute-components', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + setup: async page => { + await page.click('button=Button dropdown'); + await page.click('[data-testid="2"]'); + await page.windowScrollTo({ top: 300 }); + }, + } as TestDefinition, + { + description: `select (${width}px)`, + path: 'app-layout/with-absolute-components', + screenshotType: 'screenshotArea' as const, + configuration: { width, height: 800 }, + setup: async page => { + await page.click('[data-testid="select-demo"] button'); + await page.windowScrollTo({ top: 300 }); + }, + } as TestDefinition, + { + description: `split-panel and full-page table (${width}px)`, + path: 'app-layout/with-full-page-table-and-split-panel', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + }, + ]), + { + description: 'split-panel and full-page with open navigation (600px)', + path: 'app-layout/with-full-page-table-and-split-panel', + screenshotType: 'screenshotArea' as const, + configuration: { width: 600 }, + setup: async page => { + await page.click('button[aria-label="Open navigation"]'); + }, + }, + { + description: 'split-panel and full-page with open tools (600px)', + path: 'app-layout/with-full-page-table-and-split-panel', + screenshotType: 'screenshotArea' as const, + configuration: { width: 600 }, + setup: async page => { + await page.click('button[aria-label="Open tools"]'); + }, + }, + ], +}; + +export default suite; diff --git a/test/definitions/visual/app-layout.ts b/test/definitions/visual/app-layout.ts index 12c76d9138..c2f6a41f5c 100644 --- a/test/definitions/visual/app-layout.ts +++ b/test/definitions/visual/app-layout.ts @@ -1,174 +1,12 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import createWrapper from '../../../lib/components/test-utils/selectors'; -import { TestDefinition, TestSuite } from '../types'; - -const wrapper = createWrapper(); - -function responsiveTests(width: number): TestSuite { - return { - description: `width ${width}px`, - componentName: 'app-layout', - tests: [ - { - description: 'default', - path: 'app-layout/default', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'navigation drawer is open', - path: 'app-layout/with-wizard', - screenshotType: 'screenshotArea', - configuration: { width }, - setup: async page => { - await page.click('[aria-label="Open navigation"]'); - }, - }, - { - description: 'wizard', - path: 'app-layout/with-wizard', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'with wizard and table', - path: 'app-layout/with-wizard-and-table', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'with wizard, table, and breadcrumbs', - path: 'app-layout/with-wizard-and-table', - screenshotType: 'screenshotArea', - configuration: { width }, - queryParams: { hasBreadcrumbs: 'true' }, - }, - { - description: 'notifications', - path: 'app-layout/with-notifications', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'breadcrumbs', - path: 'app-layout/with-breadcrumbs', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'notifications and breadcrumbs', - path: 'app-layout/with-breadcrumbs-notifications', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'dashboard content type', - path: 'app-layout/dashboard-content-type', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'fixed header and footer', - path: 'app-layout/with-fixed-header-footer', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'disableBodyScroll - empty', - path: 'app-layout/legacy-nav-empty', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'disableBodyScroll - with content', - path: 'app-layout/legacy-nav-scrollable', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'disableBodyScroll - with split panel', - path: 'app-layout/legacy-nav-scrollable-with-split-panel', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'disable paddings', - path: 'app-layout/disable-paddings', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'disable paddings with breadcrumbs', - path: 'app-layout/disable-paddings-breadcrumbs', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'sticky notifications', - path: 'app-layout/with-sticky-notifications', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'sticky notifications scrolled down', - path: 'app-layout/with-sticky-notifications', - screenshotType: 'screenshotArea', - configuration: { width }, - setup: async page => { - await page.windowScrollTo({ top: 2000 }); - }, - }, - { - description: 'layout without panels', - path: 'app-layout/no-panels', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'layout without panels but with notifications', - path: 'app-layout/no-panels-with-notifications', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'with drawers', - path: 'app-layout/with-drawers', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'with empty drawers', - path: 'app-layout/with-drawers-empty', - screenshotType: 'screenshotArea', - configuration: { width }, - }, - { - description: 'with open drawer', - path: 'app-layout/with-drawers', - screenshotType: 'screenshotArea', - configuration: { width }, - setup: async page => { - await page.click('[aria-label="Security trigger button"]'); - }, - }, - ], - }; -} +import { TestSuite } from '../types'; const suite: TestSuite = { description: 'AppLayout', componentName: 'app-layout', tests: [ - // ── Responsive tests at multiple breakpoints ────────────────────────── - responsiveTests(600), - responsiveTests(1280), - responsiveTests(1400), - responsiveTests(1920), - responsiveTests(2540), - - // ── General tests ───────────────────────────────────────────────────── { description: 'no scrollbars at 320px', path: 'app-layout/default', @@ -227,361 +65,38 @@ const suite: TestSuite = { }, }, - // ── Content paddings ────────────────────────────────────────────────── - { - description: 'Content paddings', - tests: [ - ...(['true', 'false'] as const).flatMap(toolsEnabled => - (['true', 'false'] as const).flatMap(splitPanelEnabled => - (['bottom', 'side'] as const).map(splitPanelPosition => ({ - description: `toolsEnabled=${toolsEnabled} splitPanelEnabled=${splitPanelEnabled} splitPanelPosition=${splitPanelPosition}`, - path: 'app-layout/with-split-panel', - screenshotType: 'screenshotArea' as const, - queryParams: { toolsEnabled, splitPanelEnabled, splitPanelPosition }, - })) - ) - ), - ...[1500, 600].map(width => ({ - description: `with split panel and disabled content paddings - width=${width}`, - path: 'app-layout/disable-paddings-with-split-panel', - screenshotType: 'screenshotArea' as const, - configuration: { width }, - queryParams: { splitPanelOpen: 'true', splitPanelPosition: 'side' }, - })), - ], - }, - - // ── Drawers ─────────────────────────────────────────────────────────── - { - description: 'Drawers', - tests: [ - { - description: 'with split panel', - path: 'app-layout/with-drawers', - screenshotType: 'screenshotArea', - setup: async page => { - await page.click(wrapper.findAppLayout().findDrawerTriggerById('pro-help').toSelector()); - }, - }, - { - description: 'with tooltip on hover', - path: 'app-layout/with-drawers', - screenshotType: 'screenshotArea', - setup: async page => { - await page.hoverElement(wrapper.findAppLayout().findDrawerTriggerById('pro-help').toSelector()); - }, - }, - { - description: 'with custom scrollable drawer content', - path: 'app-layout/with-drawers-scrollable', - screenshotType: 'screenshotArea', - queryParams: { sideNavFill: 'false' }, - setup: async page => { - await page.click(wrapper.findAppLayout().findDrawerTriggerById('chat').toSelector()); - }, - }, - ], - }, - - // ── Headers ─────────────────────────────────────────────────────────── - { - description: 'Headers', - tests: [600, 1280].flatMap(width => [ - { - description: `alignment with full-page table (${width}px)`, - path: 'app-layout/with-table', - screenshotType: 'screenshotArea' as const, - configuration: { width }, - }, - { - description: `alignment with full-page table in sticky state (${width}px)`, - path: 'app-layout/with-table', - screenshotType: 'screenshotArea' as const, - configuration: { width }, - setup: async page => { - await page.windowScrollTo({ top: 200 }); - }, - }, - { - description: `alignment with full-page table in sticky state with sticky notifications (${width}px)`, - path: 'app-layout/with-table', - screenshotType: 'screenshotArea' as const, - configuration: { width }, - queryParams: { stickyNotifications: 'true' }, - setup: async page => { - await page.windowScrollTo({ top: 200 }); - }, - }, - { - description: `high contrast header variant in landing page (${width}px)`, - path: 'app-layout/landing-page', - screenshotType: 'screenshotArea' as const, - configuration: { width }, - }, - ]), - }, - - // ── High contrast header variant ────────────────────────────────────── + // regression for https://github.com/cloudscape-design/components/pull/1612 { - description: 'High contrast header variant', - tests: [ - ...[1400, 600].flatMap(width => [ - { - description: `with breadcrumbs and notifications at ${width}px`, - path: 'app-layout/high-contrast-header-variant', - screenshotType: 'screenshotArea' as const, - configuration: { width }, - queryParams: { hasBreadcrumbs: 'true', hasNotifications: 'true', hasContainer: 'true' }, - }, - { - description: `without overlap at ${width}px`, - path: 'app-layout/high-contrast-header-variant', - screenshotType: 'screenshotArea' as const, - configuration: { width }, - queryParams: { disableOverlap: 'true' }, - }, - { - description: `with content layout at ${width}px`, - path: 'app-layout/high-contrast-header-variant', - screenshotType: 'screenshotArea' as const, - configuration: { width }, - queryParams: { - hasBreadcrumbs: 'true', - hasNotifications: 'true', - hasContainer: 'true', - hasContentLayout: 'true', - }, - }, - ]), - ], - }, - - // ── Multiple instances ───────────────────────────────────────────────── - { - description: 'Multiple instances', - tests: [600, 1280].flatMap(width => [ - { - description: `simple (${width}px)`, - path: 'app-layout/multi-layout-simple', - screenshotType: 'screenshotArea' as const, - configuration: { width }, - }, - { - description: `iframe (${width}px)`, - path: 'app-layout/multi-layout-iframe', - screenshotType: 'screenshotArea' as const, - configuration: { width }, - }, - ]), - }, - - // ── Z-index (absolute components) ───────────────────────────────────── - { - description: 'Z-index', - tests: [ - ...[600, 1280].flatMap(width => [ - { - description: `button dropdown (${width}px)`, - path: 'app-layout/with-absolute-components', - screenshotType: 'screenshotArea' as const, - configuration: { width }, - setup: async page => { - await page.click('button=Button dropdown'); - await page.click('[data-testid="2"]'); - await page.windowScrollTo({ top: 300 }); - }, - } as TestDefinition, - { - description: `select (${width}px)`, - path: 'app-layout/with-absolute-components', - screenshotType: 'screenshotArea' as const, - configuration: { width, height: 800 }, - setup: async page => { - await page.click('[data-testid="select-demo"] button'); - await page.windowScrollTo({ top: 300 }); - }, - } as TestDefinition, - { - description: `split-panel and full-page table (${width}px)`, - path: 'app-layout/with-full-page-table-and-split-panel', - screenshotType: 'screenshotArea' as const, - configuration: { width }, - }, - ]), - { - description: 'split-panel and full-page with open navigation (600px)', - path: 'app-layout/with-full-page-table-and-split-panel', - screenshotType: 'screenshotArea' as const, - configuration: { width: 600 }, - setup: async page => { - await page.click('button[aria-label="Open navigation"]'); - }, - }, - { - description: 'split-panel and full-page with open tools (600px)', - path: 'app-layout/with-full-page-table-and-split-panel', - screenshotType: 'screenshotArea' as const, - configuration: { width: 600 }, - setup: async page => { - await page.click('button[aria-label="Open tools"]'); - }, - }, - ], - }, - - // ── Toolbar ─────────────────────────────────────────────────────────── - { - description: 'Toolbar', - tests: [ - { - description: 'multiple nested instances (no breadcrumbs dedup)', - path: 'app-layout-toolbar/multi-layout-with-hidden-instances', - screenshotType: 'screenshotArea', - }, - { - description: 'no toolbar', - path: 'app-layout-toolbar/without-toolbar', - screenshotType: 'screenshotArea', - }, - ], - }, - - // ── Max content width ───────────────────────────────────────────────── - { - description: 'Max content width', - tests: [ - { - description: 'maxContentWidth set to Number.MAX_VALUE', - path: 'app-layout/refresh-content-width', - screenshotType: 'screenshotArea', - configuration: { width: 1280, height: 700 }, - setup: async page => { - await page.click('[data-test-id="button_width-number-max_value"]'); - }, - }, - ], - }, - - // ── Sticky table header with split panel ────────────────────────────── - { - description: 'Sticky header with split panel', - tests: [ - { - description: 'scrolling to bottom with closed split panel (1 table row)', - path: 'app-layout/with-sticky-table-and-split-panel', - screenshotType: 'screenshotArea', - configuration: { width: 1280, height: 900 }, - setup: async page => { - await page.click('[data-testid="set-item-count-to-1"]'); - await page.scrollToBottom('html'); - }, - }, - { - description: 'scrolling to bottom with closed split panel (30 table rows)', - path: 'app-layout/with-sticky-table-and-split-panel', - screenshotType: 'screenshotArea', - configuration: { width: 1280, height: 900 }, - setup: async page => { - await page.click('[data-testid="set-item-count-to-30"]'); - await page.scrollToBottom('html'); - }, - }, - { - description: 'header stays sticky with open split panel (1 table row)', - path: 'app-layout/with-sticky-table-and-split-panel', - screenshotType: 'screenshotArea', - configuration: { width: 1280, height: 900 }, - setup: async page => { - await page.click('[data-testid="set-item-count-to-1"]'); - await page.click('aria/Open panel'); - await page.scrollToBottom('html'); - }, - }, - { - description: 'header stays sticky with open split panel (30 table rows)', - path: 'app-layout/with-sticky-table-and-split-panel', - screenshotType: 'screenshotArea', - configuration: { width: 1280, height: 900 }, - setup: async page => { - await page.click('[data-testid="set-item-count-to-30"]'); - await page.click('aria/Open panel'); - await page.scrollToBottom('html'); - }, - }, - { - description: 'header stays sticky when mounting and unmounting a second table', - path: 'app-layout/with-sticky-table-and-split-panel', - screenshotType: 'screenshotArea', - configuration: { width: 1280, height: 900 }, - setup: async page => { - await page.click('[data-testid="set-item-count-to-30"]'); - await page.click('aria/Open panel'); - await page.windowScrollTo({ top: 0 }); - await page.click('aria/Close panel'); - await page.scrollToBottom('html'); - }, - }, - ], + description: 'with open drawer and open side split panel after resize', + path: 'app-layout/with-drawers', + screenshotType: 'screenshotArea', + configuration: { width: 1500 }, + queryParams: { splitPanelPosition: 'side' }, + setup: async page => { + await page.click('[aria-label="Security trigger button"]'); + await page.click('[aria-label="Open panel"]'); + await page.setWindowSize({ width: 1400, height: 800 }); + }, }, - // ── Flashbar ────────────────────────────────────────────────────────── + // ── Transitions ─────────────────────────────────────────────────────── { - description: 'Flashbar', - tests: [true, false].flatMap(disableContentPaddings => - [true, false].flatMap(stickyNotifications => - [true, false].flatMap(stickyTableHeader => - [true, false].map(stackNotifications => ({ - description: `disableContentPaddings: ${disableContentPaddings}, stickyNotifications: ${stickyNotifications}, stickyTableHeader: ${stickyTableHeader}, stackNotifications: ${stackNotifications}`, - path: 'app-layout/with-stacked-notifications-and-table', - screenshotType: 'screenshotArea' as const, - configuration: { width: 1280, height: 900 }, - setup: async ( - page: import('@cloudscape-design/browser-test-tools/page-objects').ScreenshotPageObject - ) => { - if (!disableContentPaddings) { - await page.click('[data-id="toggle-content-paddings"]'); - } - if (stickyNotifications) { - await page.click('[data-id="toggle-sticky-notifications"]'); - } - if (!stickyTableHeader) { - await page.click('[data-id="toggle-sticky-table-header"]'); - } - if (!stackNotifications) { - await page.click('[data-id="toggle-stack-items"]'); - } - await page.click('[data-id="add-notification"]'); - await page.click('[data-id="add-notification"]'); - }, - })) - ) - ) - ), + description: 'transition from 400px to 1800px', + path: 'app-layout/default', + screenshotType: 'screenshotArea', + configuration: { width: 400, height: 400 }, + setup: async page => { + await page.setWindowSize({ width: 1800, height: 400 }); + }, }, - - // ── Transitions ─────────────────────────────────────────────────────── { - description: 'Transitions', - tests: [ - { - description: 'transition from 400px to 1800px', - path: 'app-layout/default', - screenshotType: 'screenshotArea', - configuration: { width: 400, height: 400 }, - setup: async page => { - await page.setWindowSize({ width: 1800, height: 400 }); - }, - }, - { - description: 'transition from 1800px to 400px', - path: 'app-layout/default', - screenshotType: 'screenshotArea', - configuration: { width: 1800, height: 400 }, - setup: async page => { - await page.setWindowSize({ width: 400, height: 400 }); - }, - }, - ], + description: 'transition from 1800px to 400px', + path: 'app-layout/default', + screenshotType: 'screenshotArea', + configuration: { width: 1800, height: 400 }, + setup: async page => { + await page.setWindowSize({ width: 400, height: 400 }); + }, }, ], }; diff --git a/test/visual/app-layout-content-paddings.test.ts b/test/visual/app-layout-content-paddings.test.ts new file mode 100644 index 0000000000..e566d79817 --- /dev/null +++ b/test/visual/app-layout-content-paddings.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/app-layout-content-paddings'; + +runTestSuites([suite]); diff --git a/test/visual/app-layout-drawers.test.ts b/test/visual/app-layout-drawers.test.ts new file mode 100644 index 0000000000..c66454010d --- /dev/null +++ b/test/visual/app-layout-drawers.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/app-layout-drawers'; + +runTestSuites([suite]); diff --git a/test/visual/app-layout-flashbar.test.ts b/test/visual/app-layout-flashbar.test.ts new file mode 100644 index 0000000000..333642e5f3 --- /dev/null +++ b/test/visual/app-layout-flashbar.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/app-layout-flashbar'; + +runTestSuites([suite]); diff --git a/test/visual/app-layout-header.test.ts b/test/visual/app-layout-header.test.ts new file mode 100644 index 0000000000..682f71ffe2 --- /dev/null +++ b/test/visual/app-layout-header.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/app-layout-header'; + +runTestSuites([suite]); diff --git a/test/visual/app-layout-multi.test.ts b/test/visual/app-layout-multi.test.ts new file mode 100644 index 0000000000..244019c8fb --- /dev/null +++ b/test/visual/app-layout-multi.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/app-layout-multi'; + +runTestSuites([suite]); diff --git a/test/visual/app-layout-responsive.test.ts b/test/visual/app-layout-responsive.test.ts new file mode 100644 index 0000000000..668d4b3522 --- /dev/null +++ b/test/visual/app-layout-responsive.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/app-layout-responsive'; + +runTestSuites([suite]); diff --git a/test/visual/app-layout-sticky-table-header-split-panel.test.ts b/test/visual/app-layout-sticky-table-header-split-panel.test.ts new file mode 100644 index 0000000000..c1ad3016a1 --- /dev/null +++ b/test/visual/app-layout-sticky-table-header-split-panel.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/app-layout-sticky-table-header-split-panel'; + +runTestSuites([suite]); diff --git a/test/visual/app-layout-toolbar.test.ts b/test/visual/app-layout-toolbar.test.ts new file mode 100644 index 0000000000..398d6386f8 --- /dev/null +++ b/test/visual/app-layout-toolbar.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/app-layout-toolbar'; + +runTestSuites([suite]); diff --git a/test/visual/app-layout-z-index.test.ts b/test/visual/app-layout-z-index.test.ts new file mode 100644 index 0000000000..5f69f77b71 --- /dev/null +++ b/test/visual/app-layout-z-index.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/app-layout-z-index'; + +runTestSuites([suite]); diff --git a/test/visual/app-layout.test.ts b/test/visual/app-layout.test.ts index 93500354cc..21c3a6ce25 100644 --- a/test/visual/app-layout.test.ts +++ b/test/visual/app-layout.test.ts @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 import { runTestSuites } from '../definitions/utils'; -import appLayout from '../definitions/visual/app-layout'; +import suite from '../definitions/visual/app-layout'; -runTestSuites([appLayout]); +runTestSuites([suite]); From 10b48b9ed2f1c5eff9eaacea4213630351b8adfd Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Sat, 30 May 2026 08:58:50 +0200 Subject: [PATCH 70/81] Add viewport screenshot type --- test/definitions/types.ts | 2 +- test/definitions/utils.ts | 28 +++++++----- .../visual/app-layout-content-paddings.ts | 4 +- test/definitions/visual/app-layout-drawers.ts | 6 +-- test/definitions/visual/app-layout-header.ts | 8 ++-- test/definitions/visual/app-layout-multi.ts | 4 +- .../visual/app-layout-responsive.ts | 44 +++++++++---------- ...-layout-sticky-table-header-split-panel.ts | 12 ++--- test/definitions/visual/app-layout-toolbar.ts | 4 +- test/definitions/visual/app-layout-z-index.ts | 10 ++--- test/definitions/visual/app-layout.ts | 22 +++++----- test/definitions/visual/area-chart.ts | 14 +++--- 12 files changed, 83 insertions(+), 75 deletions(-) diff --git a/test/definitions/types.ts b/test/definitions/types.ts index 8cdca9996a..9f8a187c48 100644 --- a/test/definitions/types.ts +++ b/test/definitions/types.ts @@ -9,7 +9,7 @@ export interface ScreenshotTestConfiguration { // 'screenshotArea' — captures the .screenshot-area element on the page. // 'permutations' — captures the entire page and crops permutations out of it. -export type ScreenshotType = 'screenshotArea' | 'permutations'; +export type ScreenshotType = 'screenshotArea' | 'permutations' | 'viewport'; export interface TestDefinition { description: string; diff --git a/test/definitions/utils.ts b/test/definitions/utils.ts index c5b237dc73..683e50cc80 100644 --- a/test/definitions/utils.ts +++ b/test/definitions/utils.ts @@ -57,22 +57,25 @@ function registerSuites(suites: Array, getBrowser: ( } /** - * Captures the .screenshot-area element using a viewport-only screenshot (fast). + * Captures a screenshot based on the test's screenshotType. */ -async function captureScreenshotArea( +async function capture( browser: WebdriverIO.Browser, page: ScreenshotPageObject, url: string, - windowSize: { width: number; height: number } | undefined, - setup?: (page: ScreenshotPageObject) => Promise + testDef: TestDefinition, + windowSize: { width: number; height: number } | undefined ): Promise { if (windowSize) { await browser.setWindowSize(windowSize.width, windowSize.height); } await browser.url(url); await page.waitForVisible(screenshotAreaSelector); - if (setup) { - await setup(page); + if (testDef.setup) { + await testDef.setup(page); + } + if (testDef.screenshotType === 'viewport') { + return page.captureViewport(); } return page.captureBySelector(screenshotAreaSelector, { viewportOnly: true }); } @@ -86,9 +89,8 @@ function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Bro const newUrl = buildUrl(newHost, testDef.path, testDef.queryParams); const oldUrl = buildUrl(oldHost, testDef.path, testDef.queryParams); - // Fast path: compare the screenshot area (viewport-only, no scroll-and-merge). - const newScreenshot = await captureScreenshotArea(browser, page, newUrl, windowSize, testDef.setup); - const oldScreenshot = await captureScreenshotArea(browser, page, oldUrl, windowSize, testDef.setup); + const newScreenshot = await capture(browser, page, newUrl, testDef, windowSize); + const oldScreenshot = await capture(browser, page, oldUrl, testDef, windowSize); const { diffPixels } = await cropAndCompare(newScreenshot, oldScreenshot); if (diffPixels === 0) { @@ -100,6 +102,9 @@ function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Bro // full capturePermutations strategy which resizes the window to fit all // content and returns individual permutation crops for precise comparison. if (testDef.screenshotType === 'permutations') { + if (windowSize) { + await browser.setWindowSize(windowSize.width, windowSize.height); + } await browser.url(newUrl); await page.waitForVisible(screenshotAreaSelector); if (testDef.setup) { @@ -107,6 +112,9 @@ function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Bro } const newPermutations = await page.capturePermutations(); + if (windowSize) { + await browser.setWindowSize(windowSize.width, windowSize.height); + } await browser.url(oldUrl); await page.waitForVisible(screenshotAreaSelector); if (testDef.setup) { @@ -122,7 +130,7 @@ function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Bro return; } - // For screenshotArea type, the diff is a real failure. + // For screenshotArea and viewport types, the diff is a real failure. expect(diffPixels).toBe(0); }); } diff --git a/test/definitions/visual/app-layout-content-paddings.ts b/test/definitions/visual/app-layout-content-paddings.ts index 068e5dd08e..e76d055749 100644 --- a/test/definitions/visual/app-layout-content-paddings.ts +++ b/test/definitions/visual/app-layout-content-paddings.ts @@ -12,7 +12,7 @@ const suite: TestSuite = { (['bottom', 'side'] as const).map(splitPanelPosition => ({ description: `toolsEnabled=${toolsEnabled} splitPanelEnabled=${splitPanelEnabled} splitPanelPosition=${splitPanelPosition}`, path: 'app-layout/with-split-panel', - screenshotType: 'screenshotArea' as const, + screenshotType: 'viewport' as const, queryParams: { toolsEnabled, splitPanelEnabled, splitPanelPosition }, })) ) @@ -20,7 +20,7 @@ const suite: TestSuite = { ...[1500, 600].map(width => ({ description: `with split panel and disabled content paddings - width=${width}`, path: 'app-layout/disable-paddings-with-split-panel', - screenshotType: 'screenshotArea' as const, + screenshotType: 'viewport' as const, configuration: { width }, queryParams: { splitPanelOpen: 'true', splitPanelPosition: 'side' }, })), diff --git a/test/definitions/visual/app-layout-drawers.ts b/test/definitions/visual/app-layout-drawers.ts index 2af69c2ca1..39c2089783 100644 --- a/test/definitions/visual/app-layout-drawers.ts +++ b/test/definitions/visual/app-layout-drawers.ts @@ -13,7 +13,7 @@ const suite: TestSuite = { { description: 'with split panel', path: 'app-layout/with-drawers', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', setup: async page => { await page.click(wrapper.findAppLayout().findDrawerTriggerById('pro-help').toSelector()); }, @@ -21,7 +21,7 @@ const suite: TestSuite = { { description: 'with tooltip on hover', path: 'app-layout/with-drawers', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', setup: async page => { await page.hoverElement(wrapper.findAppLayout().findDrawerTriggerById('pro-help').toSelector()); }, @@ -29,7 +29,7 @@ const suite: TestSuite = { { description: 'with custom scrollable drawer content', path: 'app-layout/with-drawers-scrollable', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', queryParams: { sideNavFill: 'false' }, setup: async page => { await page.click(wrapper.findAppLayout().findDrawerTriggerById('chat').toSelector()); diff --git a/test/definitions/visual/app-layout-header.ts b/test/definitions/visual/app-layout-header.ts index 55f674860c..3a128a266e 100644 --- a/test/definitions/visual/app-layout-header.ts +++ b/test/definitions/visual/app-layout-header.ts @@ -14,13 +14,13 @@ const suite: TestSuite = { { description: `alignment with full-page table (${width}px)`, path: 'app-layout/with-table', - screenshotType: 'screenshotArea' as const, + screenshotType: 'viewport' as const, configuration: { width }, }, { description: `alignment with full-page table in sticky state (${width}px)`, path: 'app-layout/with-table', - screenshotType: 'screenshotArea' as const, + screenshotType: 'viewport' as const, configuration: { width }, setup: async page => { await page.windowScrollTo({ top: 200 }); @@ -29,7 +29,7 @@ const suite: TestSuite = { { description: `alignment with full-page table in sticky state with sticky notifications (${width}px)`, path: 'app-layout/with-table', - screenshotType: 'screenshotArea' as const, + screenshotType: 'viewport' as const, configuration: { width }, queryParams: { stickyNotifications: 'true' }, setup: async page => { @@ -39,7 +39,7 @@ const suite: TestSuite = { { description: `high contrast header variant in landing page (${width}px)`, path: 'app-layout/landing-page', - screenshotType: 'screenshotArea' as const, + screenshotType: 'viewport' as const, configuration: { width }, }, ]), diff --git a/test/definitions/visual/app-layout-multi.ts b/test/definitions/visual/app-layout-multi.ts index 7efbbcd473..babf9733cf 100644 --- a/test/definitions/visual/app-layout-multi.ts +++ b/test/definitions/visual/app-layout-multi.ts @@ -10,13 +10,13 @@ const suite: TestSuite = { { description: `simple (${width}px)`, path: 'app-layout/multi-layout-simple', - screenshotType: 'screenshotArea' as const, + screenshotType: 'viewport' as const, configuration: { width }, }, { description: `iframe (${width}px)`, path: 'app-layout/multi-layout-iframe', - screenshotType: 'screenshotArea' as const, + screenshotType: 'viewport' as const, configuration: { width }, }, ]), diff --git a/test/definitions/visual/app-layout-responsive.ts b/test/definitions/visual/app-layout-responsive.ts index 8f227a5814..8a53da6a09 100644 --- a/test/definitions/visual/app-layout-responsive.ts +++ b/test/definitions/visual/app-layout-responsive.ts @@ -11,13 +11,13 @@ function responsiveTests(width: number): TestSuite { { description: 'default', path: 'app-layout/default', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'navigation drawer is open', path: 'app-layout/with-wizard', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, setup: async page => { await page.click('[aria-label="Open navigation"]'); @@ -26,92 +26,92 @@ function responsiveTests(width: number): TestSuite { { description: 'wizard', path: 'app-layout/with-wizard', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'with wizard and table', path: 'app-layout/with-wizard-and-table', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'with wizard, table, and breadcrumbs', path: 'app-layout/with-wizard-and-table', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, queryParams: { hasBreadcrumbs: 'true' }, }, { description: 'notifications', path: 'app-layout/with-notifications', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'breadcrumbs', path: 'app-layout/with-breadcrumbs', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'notifications and breadcrumbs', path: 'app-layout/with-breadcrumbs-notifications', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'dashboard content type', path: 'app-layout/dashboard-content-type', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'fixed header and footer', path: 'app-layout/with-fixed-header-footer', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'disableBodyScroll - empty', path: 'app-layout/legacy-nav-empty', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'disableBodyScroll - with content', path: 'app-layout/legacy-nav-scrollable', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'disableBodyScroll - with split panel', path: 'app-layout/legacy-nav-scrollable-with-split-panel', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'disable paddings', path: 'app-layout/disable-paddings', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'disable paddings with breadcrumbs', path: 'app-layout/disable-paddings-breadcrumbs', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'sticky notifications', path: 'app-layout/with-sticky-notifications', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'sticky notifications scrolled down', path: 'app-layout/with-sticky-notifications', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, setup: async page => { await page.windowScrollTo({ top: 2000 }); @@ -120,31 +120,31 @@ function responsiveTests(width: number): TestSuite { { description: 'layout without panels', path: 'app-layout/no-panels', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'layout without panels but with notifications', path: 'app-layout/no-panels-with-notifications', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'with drawers', path: 'app-layout/with-drawers', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'with empty drawers', path: 'app-layout/with-drawers-empty', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, }, { description: 'with open drawer', path: 'app-layout/with-drawers', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width }, setup: async page => { await page.click('[aria-label="Security trigger button"]'); diff --git a/test/definitions/visual/app-layout-sticky-table-header-split-panel.ts b/test/definitions/visual/app-layout-sticky-table-header-split-panel.ts index 91628802ff..6a0b899686 100644 --- a/test/definitions/visual/app-layout-sticky-table-header-split-panel.ts +++ b/test/definitions/visual/app-layout-sticky-table-header-split-panel.ts @@ -10,7 +10,7 @@ const suite: TestSuite = { { description: 'scrolling to bottom with closed split panel (1 table row)', path: 'app-layout/with-sticky-table-and-split-panel', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 1280, height: 900 }, setup: async page => { await page.click('[data-testid="set-item-count-to-1"]'); @@ -20,7 +20,7 @@ const suite: TestSuite = { { description: 'scrolling to bottom with closed split panel (30 table rows)', path: 'app-layout/with-sticky-table-and-split-panel', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 1280, height: 900 }, setup: async page => { await page.click('[data-testid="set-item-count-to-30"]'); @@ -30,7 +30,7 @@ const suite: TestSuite = { { description: 'header stays sticky with open split panel (1 table row)', path: 'app-layout/with-sticky-table-and-split-panel', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 1280, height: 900 }, setup: async page => { await page.click('[data-testid="set-item-count-to-1"]'); @@ -41,7 +41,7 @@ const suite: TestSuite = { { description: 'header stays sticky with open split panel (30 table rows)', path: 'app-layout/with-sticky-table-and-split-panel', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 1280, height: 900 }, setup: async page => { await page.click('[data-testid="set-item-count-to-30"]'); @@ -52,7 +52,7 @@ const suite: TestSuite = { { description: 'header stays sticky when mounting and unmounting a second table', path: 'app-layout/with-sticky-table-and-split-panel', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 1280, height: 900 }, setup: async page => { await page.click('[data-testid="set-item-count-to-30"]'); @@ -66,7 +66,7 @@ const suite: TestSuite = { { description: 'maxContentWidth set to Number.MAX_VALUE', path: 'app-layout/refresh-content-width', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 1280, height: 700 }, setup: async page => { await page.click('[data-test-id="button_width-number-max_value"]'); diff --git a/test/definitions/visual/app-layout-toolbar.ts b/test/definitions/visual/app-layout-toolbar.ts index dc9f5861fe..fb174b0a71 100644 --- a/test/definitions/visual/app-layout-toolbar.ts +++ b/test/definitions/visual/app-layout-toolbar.ts @@ -10,12 +10,12 @@ const suite: TestSuite = { { description: 'multiple nested instances (no breadcrumbs dedup)', path: 'app-layout-toolbar/multi-layout-with-hidden-instances', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', }, { description: 'no toolbar', path: 'app-layout-toolbar/without-toolbar', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', }, ], }; diff --git a/test/definitions/visual/app-layout-z-index.ts b/test/definitions/visual/app-layout-z-index.ts index 7765a496a9..b0dacf8bc5 100644 --- a/test/definitions/visual/app-layout-z-index.ts +++ b/test/definitions/visual/app-layout-z-index.ts @@ -11,7 +11,7 @@ const suite: TestSuite = { { description: `button dropdown (${width}px)`, path: 'app-layout/with-absolute-components', - screenshotType: 'screenshotArea' as const, + screenshotType: 'viewport' as const, configuration: { width }, setup: async page => { await page.click('button=Button dropdown'); @@ -22,7 +22,7 @@ const suite: TestSuite = { { description: `select (${width}px)`, path: 'app-layout/with-absolute-components', - screenshotType: 'screenshotArea' as const, + screenshotType: 'viewport' as const, configuration: { width, height: 800 }, setup: async page => { await page.click('[data-testid="select-demo"] button'); @@ -32,14 +32,14 @@ const suite: TestSuite = { { description: `split-panel and full-page table (${width}px)`, path: 'app-layout/with-full-page-table-and-split-panel', - screenshotType: 'screenshotArea' as const, + screenshotType: 'viewport' as const, configuration: { width }, }, ]), { description: 'split-panel and full-page with open navigation (600px)', path: 'app-layout/with-full-page-table-and-split-panel', - screenshotType: 'screenshotArea' as const, + screenshotType: 'viewport' as const, configuration: { width: 600 }, setup: async page => { await page.click('button[aria-label="Open navigation"]'); @@ -48,7 +48,7 @@ const suite: TestSuite = { { description: 'split-panel and full-page with open tools (600px)', path: 'app-layout/with-full-page-table-and-split-panel', - screenshotType: 'screenshotArea' as const, + screenshotType: 'viewport' as const, configuration: { width: 600 }, setup: async page => { await page.click('button[aria-label="Open tools"]'); diff --git a/test/definitions/visual/app-layout.ts b/test/definitions/visual/app-layout.ts index c2f6a41f5c..683efdcc25 100644 --- a/test/definitions/visual/app-layout.ts +++ b/test/definitions/visual/app-layout.ts @@ -10,13 +10,13 @@ const suite: TestSuite = { { description: 'no scrollbars at 320px', path: 'app-layout/default', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 320 }, }, { description: 'drawer buttons alignment', path: 'app-layout/default', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 800 }, setup: async page => { await page.click('[aria-label="Open tools"]'); @@ -25,7 +25,7 @@ const suite: TestSuite = { { description: 'disable paddings - navigation closed', path: 'app-layout/disable-paddings', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 1280 }, setup: async page => { await page.click('[aria-label="Close navigation"]'); @@ -34,29 +34,29 @@ const suite: TestSuite = { { description: 'panels stacking on mobile', path: 'app-layout/all-panels-open', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 600 }, }, { description: 'wrapping long words', path: 'app-layout/text-wrap', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', }, { description: 'fill content area', path: 'app-layout/fill-content-area', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', }, { description: 'with tools and drawers', path: 'app-layout/with-drawers', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', queryParams: { hasTools: 'true' }, }, { description: 'with open drawer and open side split panel', path: 'app-layout/with-drawers', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 1400 }, queryParams: { splitPanelPosition: 'side' }, setup: async page => { @@ -69,7 +69,7 @@ const suite: TestSuite = { { description: 'with open drawer and open side split panel after resize', path: 'app-layout/with-drawers', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 1500 }, queryParams: { splitPanelPosition: 'side' }, setup: async page => { @@ -83,7 +83,7 @@ const suite: TestSuite = { { description: 'transition from 400px to 1800px', path: 'app-layout/default', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 400, height: 400 }, setup: async page => { await page.setWindowSize({ width: 1800, height: 400 }); @@ -92,7 +92,7 @@ const suite: TestSuite = { { description: 'transition from 1800px to 400px', path: 'app-layout/default', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 1800, height: 400 }, setup: async page => { await page.setWindowSize({ width: 400, height: 400 }); diff --git a/test/definitions/visual/area-chart.ts b/test/definitions/visual/area-chart.ts index 10855239f6..cfa0b5e47b 100644 --- a/test/definitions/visual/area-chart.ts +++ b/test/definitions/visual/area-chart.ts @@ -35,7 +35,7 @@ const suite: TestSuite = { { description: 'chart plot has a focus outline', path: 'area-chart/test', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 800, height: 800 }, setup: async page => { await page.click(TEST_CHART_FILTER_TRIGGER); @@ -46,7 +46,7 @@ const suite: TestSuite = { { description: 'can navigate along X axis highlighting all series with keyboard', path: 'area-chart/test', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 800, height: 800 }, setup: async page => { await page.click(TEST_CHART_FILTER_TRIGGER); @@ -59,7 +59,7 @@ const suite: TestSuite = { { description: 'can navigate a specific series with keyboard', path: 'area-chart/test', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 800, height: 800 }, setup: async page => { await page.click(TEST_CHART_FILTER_TRIGGER); @@ -74,7 +74,7 @@ const suite: TestSuite = { { description: 'selects correct series when navigated back from legend', path: 'area-chart/test', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 800, height: 800 }, setup: async page => { await page.click(TEST_CHART_FILTER_TRIGGER); @@ -90,7 +90,7 @@ const suite: TestSuite = { { description: 'can pin popover for all data points at a given X coordinate with keyboard', path: 'area-chart/test', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 800, height: 800 }, setup: async page => { await page.click(TEST_CHART_FILTER_TRIGGER); @@ -106,7 +106,7 @@ const suite: TestSuite = { { description: 'can pin popover for a point in a specific series with keyboard', path: 'area-chart/test', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 800, height: 800 }, setup: async page => { await page.click(TEST_CHART_FILTER_TRIGGER); @@ -123,7 +123,7 @@ const suite: TestSuite = { { description: 'shows popover on hover', path: 'area-chart/test', - screenshotType: 'screenshotArea', + screenshotType: 'viewport', configuration: { width: 800, height: 800 }, setup: async page => { await page.hoverElement('[aria-label="Linear latency chart"]', 200, 50); From d0a3012bcd2696dd97861eb7c8fb3e362d102c6f Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 5 Jun 2026 06:35:49 +0200 Subject: [PATCH 71/81] Split app layout responsive test definitions --- test/definitions/index.ts | 12 ++++++++-- .../visual/app-layout-responsive-1280.ts | 7 ++++++ .../visual/app-layout-responsive-1400.ts | 7 ++++++ .../visual/app-layout-responsive-1920.ts | 7 ++++++ .../visual/app-layout-responsive-2540.ts | 7 ++++++ .../visual/app-layout-responsive-600.ts | 7 ++++++ ...sive.ts => app-layout-responsive-tests.ts} | 22 +++++-------------- .../visual/app-layout-responsive-1280.test.ts | 6 +++++ .../visual/app-layout-responsive-1400.test.ts | 6 +++++ .../visual/app-layout-responsive-1920.test.ts | 6 +++++ .../visual/app-layout-responsive-2540.test.ts | 6 +++++ ...t.ts => app-layout-responsive-600.test.ts} | 2 +- 12 files changed, 76 insertions(+), 19 deletions(-) create mode 100644 test/definitions/visual/app-layout-responsive-1280.ts create mode 100644 test/definitions/visual/app-layout-responsive-1400.ts create mode 100644 test/definitions/visual/app-layout-responsive-1920.ts create mode 100644 test/definitions/visual/app-layout-responsive-2540.ts create mode 100644 test/definitions/visual/app-layout-responsive-600.ts rename test/definitions/visual/{app-layout-responsive.ts => app-layout-responsive-tests.ts} (92%) create mode 100644 test/visual/app-layout-responsive-1280.test.ts create mode 100644 test/visual/app-layout-responsive-1400.test.ts create mode 100644 test/visual/app-layout-responsive-1920.test.ts create mode 100644 test/visual/app-layout-responsive-2540.test.ts rename test/visual/{app-layout-responsive.test.ts => app-layout-responsive-600.test.ts} (73%) diff --git a/test/definitions/index.ts b/test/definitions/index.ts index 17dc1902a0..3bd6fa66e2 100644 --- a/test/definitions/index.ts +++ b/test/definitions/index.ts @@ -12,7 +12,11 @@ import appLayoutDrawers from './visual/app-layout-drawers'; import appLayoutFlashbar from './visual/app-layout-flashbar'; import appLayoutHeader from './visual/app-layout-header'; import appLayoutMulti from './visual/app-layout-multi'; -import appLayoutResponsive from './visual/app-layout-responsive'; +import appLayoutResponsive600 from './visual/app-layout-responsive-600'; +import appLayoutResponsive1280 from './visual/app-layout-responsive-1280'; +import appLayoutResponsive1400 from './visual/app-layout-responsive-1400'; +import appLayoutResponsive1920 from './visual/app-layout-responsive-1920'; +import appLayoutResponsive2540 from './visual/app-layout-responsive-2540'; import appLayoutStickyTableHeaderSplitPanel from './visual/app-layout-sticky-table-header-split-panel'; import appLayoutToolbar from './visual/app-layout-toolbar'; import appLayoutZIndex from './visual/app-layout-z-index'; @@ -30,7 +34,11 @@ export const allSuites: TestSuite[] = [ appLayoutFlashbar, appLayoutHeader, appLayoutMulti, - appLayoutResponsive, + appLayoutResponsive600, + appLayoutResponsive1280, + appLayoutResponsive1400, + appLayoutResponsive1920, + appLayoutResponsive2540, appLayoutStickyTableHeaderSplitPanel, appLayoutToolbar, appLayoutZIndex, diff --git a/test/definitions/visual/app-layout-responsive-1280.ts b/test/definitions/visual/app-layout-responsive-1280.ts new file mode 100644 index 0000000000..d5fe823f54 --- /dev/null +++ b/test/definitions/visual/app-layout-responsive-1280.ts @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { responsiveTests } from './app-layout-responsive-tests'; + +const suite = responsiveTests(1280); +export default suite; diff --git a/test/definitions/visual/app-layout-responsive-1400.ts b/test/definitions/visual/app-layout-responsive-1400.ts new file mode 100644 index 0000000000..1cb519005c --- /dev/null +++ b/test/definitions/visual/app-layout-responsive-1400.ts @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { responsiveTests } from './app-layout-responsive-tests'; + +const suite = responsiveTests(1400); +export default suite; diff --git a/test/definitions/visual/app-layout-responsive-1920.ts b/test/definitions/visual/app-layout-responsive-1920.ts new file mode 100644 index 0000000000..88b8c6caf3 --- /dev/null +++ b/test/definitions/visual/app-layout-responsive-1920.ts @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { responsiveTests } from './app-layout-responsive-tests'; + +const suite = responsiveTests(1920); +export default suite; diff --git a/test/definitions/visual/app-layout-responsive-2540.ts b/test/definitions/visual/app-layout-responsive-2540.ts new file mode 100644 index 0000000000..9dd62c00db --- /dev/null +++ b/test/definitions/visual/app-layout-responsive-2540.ts @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { responsiveTests } from './app-layout-responsive-tests'; + +const suite = responsiveTests(2540); +export default suite; diff --git a/test/definitions/visual/app-layout-responsive-600.ts b/test/definitions/visual/app-layout-responsive-600.ts new file mode 100644 index 0000000000..f6fd3665cc --- /dev/null +++ b/test/definitions/visual/app-layout-responsive-600.ts @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { responsiveTests } from './app-layout-responsive-tests'; + +const suite = responsiveTests(600); +export default suite; diff --git a/test/definitions/visual/app-layout-responsive.ts b/test/definitions/visual/app-layout-responsive-tests.ts similarity index 92% rename from test/definitions/visual/app-layout-responsive.ts rename to test/definitions/visual/app-layout-responsive-tests.ts index 8a53da6a09..f0ac91d121 100644 --- a/test/definitions/visual/app-layout-responsive.ts +++ b/test/definitions/visual/app-layout-responsive-tests.ts @@ -3,9 +3,13 @@ import { TestSuite } from '../types'; -function responsiveTests(width: number): TestSuite { +/** + * Shared responsive test scenarios for app-layout. Each width gets its own + * definition file and test runner so that Jest sharding can parallelize them. + */ +export function responsiveTests(width: number): TestSuite { return { - description: `width ${width}px`, + description: `AppLayout responsive width ${width}px`, componentName: 'app-layout', tests: [ { @@ -153,17 +157,3 @@ function responsiveTests(width: number): TestSuite { ], }; } - -const suite: TestSuite = { - description: 'AppLayout responsive', - componentName: 'app-layout', - tests: [ - responsiveTests(600), - responsiveTests(1280), - responsiveTests(1400), - responsiveTests(1920), - responsiveTests(2540), - ], -}; - -export default suite; diff --git a/test/visual/app-layout-responsive-1280.test.ts b/test/visual/app-layout-responsive-1280.test.ts new file mode 100644 index 0000000000..0324b2bf39 --- /dev/null +++ b/test/visual/app-layout-responsive-1280.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/app-layout-responsive-1280'; + +runTestSuites([suite]); diff --git a/test/visual/app-layout-responsive-1400.test.ts b/test/visual/app-layout-responsive-1400.test.ts new file mode 100644 index 0000000000..745372c34e --- /dev/null +++ b/test/visual/app-layout-responsive-1400.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/app-layout-responsive-1400'; + +runTestSuites([suite]); diff --git a/test/visual/app-layout-responsive-1920.test.ts b/test/visual/app-layout-responsive-1920.test.ts new file mode 100644 index 0000000000..bee5fc8763 --- /dev/null +++ b/test/visual/app-layout-responsive-1920.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/app-layout-responsive-1920'; + +runTestSuites([suite]); diff --git a/test/visual/app-layout-responsive-2540.test.ts b/test/visual/app-layout-responsive-2540.test.ts new file mode 100644 index 0000000000..48bc8580da --- /dev/null +++ b/test/visual/app-layout-responsive-2540.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/app-layout-responsive-2540'; + +runTestSuites([suite]); diff --git a/test/visual/app-layout-responsive.test.ts b/test/visual/app-layout-responsive-600.test.ts similarity index 73% rename from test/visual/app-layout-responsive.test.ts rename to test/visual/app-layout-responsive-600.test.ts index 668d4b3522..cd9243cb32 100644 --- a/test/visual/app-layout-responsive.test.ts +++ b/test/visual/app-layout-responsive-600.test.ts @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout-responsive'; +import suite from '../definitions/visual/app-layout-responsive-600'; runTestSuites([suite]); From 94c3f2a1a0407a08b5c802abea1a6c6920c8c315 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 5 Jun 2026 06:59:26 +0200 Subject: [PATCH 72/81] Autogenerate test files --- .github/workflows/visual-regression.yml | 3 + .gitignore | 1 + build-tools/visual/generate-tests.js | 71 +++++++++++++++++++ test/visual/action-card.test.ts | 6 -- test/visual/alert.test.ts | 6 -- .../app-layout-content-paddings.test.ts | 6 -- test/visual/app-layout-drawers.test.ts | 6 -- test/visual/app-layout-flashbar.test.ts | 6 -- test/visual/app-layout-header.test.ts | 6 -- test/visual/app-layout-multi.test.ts | 6 -- .../visual/app-layout-responsive-1280.test.ts | 6 -- .../visual/app-layout-responsive-1400.test.ts | 6 -- .../visual/app-layout-responsive-1920.test.ts | 6 -- .../visual/app-layout-responsive-2540.test.ts | 6 -- test/visual/app-layout-responsive-600.test.ts | 6 -- ...ut-sticky-table-header-split-panel.test.ts | 6 -- test/visual/app-layout-toolbar.test.ts | 6 -- test/visual/app-layout-z-index.test.ts | 6 -- test/visual/app-layout.test.ts | 6 -- test/visual/area-chart.test.ts | 6 -- test/visual/attribute-editor.test.ts | 5 -- test/visual/autosuggest.test.ts | 6 -- test/visual/badge.test.ts | 6 -- 23 files changed, 75 insertions(+), 119 deletions(-) create mode 100644 build-tools/visual/generate-tests.js delete mode 100644 test/visual/action-card.test.ts delete mode 100644 test/visual/alert.test.ts delete mode 100644 test/visual/app-layout-content-paddings.test.ts delete mode 100644 test/visual/app-layout-drawers.test.ts delete mode 100644 test/visual/app-layout-flashbar.test.ts delete mode 100644 test/visual/app-layout-header.test.ts delete mode 100644 test/visual/app-layout-multi.test.ts delete mode 100644 test/visual/app-layout-responsive-1280.test.ts delete mode 100644 test/visual/app-layout-responsive-1400.test.ts delete mode 100644 test/visual/app-layout-responsive-1920.test.ts delete mode 100644 test/visual/app-layout-responsive-2540.test.ts delete mode 100644 test/visual/app-layout-responsive-600.test.ts delete mode 100644 test/visual/app-layout-sticky-table-header-split-panel.test.ts delete mode 100644 test/visual/app-layout-toolbar.test.ts delete mode 100644 test/visual/app-layout-z-index.test.ts delete mode 100644 test/visual/app-layout.test.ts delete mode 100644 test/visual/area-chart.test.ts delete mode 100644 test/visual/attribute-editor.test.ts delete mode 100644 test/visual/autosuggest.test.ts delete mode 100644 test/visual/badge.test.ts diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 8e03c9967c..007be5a144 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -124,6 +124,9 @@ jobs: # ── Run tests ───────────────────────────────────────────────────────── + - name: Generate visual test files + run: node build-tools/visual/generate-tests.js + - name: Start test server (port 8080) run: npx --yes serve --no-clipboard --listen 8080 pages/lib/static-default & diff --git a/.gitignore b/.gitignore index db3d62b944..3aa354895f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ coverage lib # generated sources src/index.ts +test/visual src/test-utils/dom/index.ts src/test-utils/selectors src/icon/generated diff --git a/build-tools/visual/generate-tests.js b/build-tools/visual/generate-tests.js new file mode 100644 index 0000000000..a56b996b3d --- /dev/null +++ b/build-tools/visual/generate-tests.js @@ -0,0 +1,71 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Auto-generates test/visual/*.test.ts files from test/definitions/visual/*.ts. + * + * Each definition file that exports a default TestSuite gets a corresponding + * test runner. Helper files (those without a default export, e.g. shared utils + * like app-layout-responsive-tests.ts) are skipped. + * + * Run this script before executing the visual test suite: + * node build-tools/visual/generate-tests.js + */ +const fs = require('fs'); +const path = require('path'); + +const definitionsDir = path.resolve(__dirname, '../../test/definitions/visual'); +const outputDir = path.resolve(__dirname, '../../test/visual'); + +// Files that are shared helpers (export named functions, not a default suite). +// These are detected by checking if the file contains "export default" or +// a conventional "const suite" pattern followed by "export default suite". +const HELPER_SUFFIXES = ['-tests']; + +function isHelperFile(basename) { + return HELPER_SUFFIXES.some(suffix => basename.endsWith(suffix)); +} + +function generate() { + // Ensure output directory exists + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + const files = fs.readdirSync(definitionsDir).filter(f => f.endsWith('.ts') && !f.endsWith('.d.ts')); + + const generated = []; + + for (const file of files) { + const basename = file.replace(/\.ts$/, ''); + + if (isHelperFile(basename)) { + continue; + } + + // Verify the file has a default export by scanning for the pattern + const content = fs.readFileSync(path.join(definitionsDir, file), 'utf-8'); + if (!content.includes('export default')) { + continue; + } + + const testContent = [ + '// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.', + '// SPDX-License-Identifier: Apache-2.0', + '// Auto-generated by build-tools/visual/generate-tests.js — do not edit manually.', + `import { runTestSuites } from '../definitions/utils';`, + `import suite from '../definitions/visual/${basename}';`, + '', + 'runTestSuites([suite]);', + '', + ].join('\n'); + + const outputPath = path.join(outputDir, `${basename}.test.ts`); + fs.writeFileSync(outputPath, testContent); + generated.push(basename); + } + + console.log(`Generated ${generated.length} visual test files in test/visual/`); +} + +generate(); diff --git a/test/visual/action-card.test.ts b/test/visual/action-card.test.ts deleted file mode 100644 index 8505dad23b..0000000000 --- a/test/visual/action-card.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import actionCard from '../definitions/visual/action-card'; - -runTestSuites([actionCard]); diff --git a/test/visual/alert.test.ts b/test/visual/alert.test.ts deleted file mode 100644 index 431e4907e4..0000000000 --- a/test/visual/alert.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import alert from '../definitions/visual/alert'; - -runTestSuites([alert]); diff --git a/test/visual/app-layout-content-paddings.test.ts b/test/visual/app-layout-content-paddings.test.ts deleted file mode 100644 index e566d79817..0000000000 --- a/test/visual/app-layout-content-paddings.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout-content-paddings'; - -runTestSuites([suite]); diff --git a/test/visual/app-layout-drawers.test.ts b/test/visual/app-layout-drawers.test.ts deleted file mode 100644 index c66454010d..0000000000 --- a/test/visual/app-layout-drawers.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout-drawers'; - -runTestSuites([suite]); diff --git a/test/visual/app-layout-flashbar.test.ts b/test/visual/app-layout-flashbar.test.ts deleted file mode 100644 index 333642e5f3..0000000000 --- a/test/visual/app-layout-flashbar.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout-flashbar'; - -runTestSuites([suite]); diff --git a/test/visual/app-layout-header.test.ts b/test/visual/app-layout-header.test.ts deleted file mode 100644 index 682f71ffe2..0000000000 --- a/test/visual/app-layout-header.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout-header'; - -runTestSuites([suite]); diff --git a/test/visual/app-layout-multi.test.ts b/test/visual/app-layout-multi.test.ts deleted file mode 100644 index 244019c8fb..0000000000 --- a/test/visual/app-layout-multi.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout-multi'; - -runTestSuites([suite]); diff --git a/test/visual/app-layout-responsive-1280.test.ts b/test/visual/app-layout-responsive-1280.test.ts deleted file mode 100644 index 0324b2bf39..0000000000 --- a/test/visual/app-layout-responsive-1280.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout-responsive-1280'; - -runTestSuites([suite]); diff --git a/test/visual/app-layout-responsive-1400.test.ts b/test/visual/app-layout-responsive-1400.test.ts deleted file mode 100644 index 745372c34e..0000000000 --- a/test/visual/app-layout-responsive-1400.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout-responsive-1400'; - -runTestSuites([suite]); diff --git a/test/visual/app-layout-responsive-1920.test.ts b/test/visual/app-layout-responsive-1920.test.ts deleted file mode 100644 index bee5fc8763..0000000000 --- a/test/visual/app-layout-responsive-1920.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout-responsive-1920'; - -runTestSuites([suite]); diff --git a/test/visual/app-layout-responsive-2540.test.ts b/test/visual/app-layout-responsive-2540.test.ts deleted file mode 100644 index 48bc8580da..0000000000 --- a/test/visual/app-layout-responsive-2540.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout-responsive-2540'; - -runTestSuites([suite]); diff --git a/test/visual/app-layout-responsive-600.test.ts b/test/visual/app-layout-responsive-600.test.ts deleted file mode 100644 index cd9243cb32..0000000000 --- a/test/visual/app-layout-responsive-600.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout-responsive-600'; - -runTestSuites([suite]); diff --git a/test/visual/app-layout-sticky-table-header-split-panel.test.ts b/test/visual/app-layout-sticky-table-header-split-panel.test.ts deleted file mode 100644 index c1ad3016a1..0000000000 --- a/test/visual/app-layout-sticky-table-header-split-panel.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout-sticky-table-header-split-panel'; - -runTestSuites([suite]); diff --git a/test/visual/app-layout-toolbar.test.ts b/test/visual/app-layout-toolbar.test.ts deleted file mode 100644 index 398d6386f8..0000000000 --- a/test/visual/app-layout-toolbar.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout-toolbar'; - -runTestSuites([suite]); diff --git a/test/visual/app-layout-z-index.test.ts b/test/visual/app-layout-z-index.test.ts deleted file mode 100644 index 5f69f77b71..0000000000 --- a/test/visual/app-layout-z-index.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout-z-index'; - -runTestSuites([suite]); diff --git a/test/visual/app-layout.test.ts b/test/visual/app-layout.test.ts deleted file mode 100644 index 21c3a6ce25..0000000000 --- a/test/visual/app-layout.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import suite from '../definitions/visual/app-layout'; - -runTestSuites([suite]); diff --git a/test/visual/area-chart.test.ts b/test/visual/area-chart.test.ts deleted file mode 100644 index 51d6631016..0000000000 --- a/test/visual/area-chart.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import areaChart from '../definitions/visual/area-chart'; - -runTestSuites([areaChart]); diff --git a/test/visual/attribute-editor.test.ts b/test/visual/attribute-editor.test.ts deleted file mode 100644 index f57e8785b2..0000000000 --- a/test/visual/attribute-editor.test.ts +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import attributeEditor from '../definitions/visual/attribute-editor'; -runTestSuites([attributeEditor]); diff --git a/test/visual/autosuggest.test.ts b/test/visual/autosuggest.test.ts deleted file mode 100644 index 454c00c648..0000000000 --- a/test/visual/autosuggest.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import autosuggest from '../definitions/visual/autosuggest'; - -runTestSuites([autosuggest]); diff --git a/test/visual/badge.test.ts b/test/visual/badge.test.ts deleted file mode 100644 index 6b24838a42..0000000000 --- a/test/visual/badge.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { runTestSuites } from '../definitions/utils'; -import badge from '../definitions/visual/badge'; - -runTestSuites([badge]); From 234bb8164562c148ab74c600c20c591e882fe64f Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 5 Jun 2026 07:12:36 +0200 Subject: [PATCH 73/81] Generate Allure reports --- .github/workflows/visual-regression.yml | 34 ++ .gitignore | 2 + jest.visual.config.js | 11 +- package-lock.json | 737 ++++++++++++++++++++++++ package.json | 1 + 5 files changed, 784 insertions(+), 1 deletion(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 007be5a144..0b4e20a816 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -148,3 +148,37 @@ jobs: name: visual-regression-diffs-shard-${{ matrix.shard }} path: visual-regression-output/ retention-days: 14 + + - name: Upload Allure results + if: always() + uses: actions/upload-artifact@v4 + with: + name: allure-results-shard-${{ matrix.shard }} + path: allure-results/ + retention-days: 3 + + report: + name: Generate Allure Report + if: always() + needs: [visual] + runs-on: ubuntu-latest + steps: + - name: Download all Allure results + uses: actions/download-artifact@v4 + with: + pattern: allure-results-shard-* + path: allure-results + merge-multiple: true + + - name: Generate Allure HTML report + uses: simple-elf/allure-report-action@v1.12 + with: + allure_results: allure-results + allure_report: allure-report + + - name: Upload Allure report + uses: actions/upload-artifact@v4 + with: + name: allure-report + path: allure-report/ + retention-days: 14 diff --git a/.gitignore b/.gitignore index 3aa354895f..38d6053611 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ lib # generated sources src/index.ts test/visual +allure-results +allure-report src/test-utils/dom/index.ts src/test-utils/selectors src/icon/generated diff --git a/jest.visual.config.js b/jest.visual.config.js index 19418a86dd..5cb7f033fe 100644 --- a/jest.visual.config.js +++ b/jest.visual.config.js @@ -14,7 +14,16 @@ module.exports = { }, ], }, - reporters: ['default', 'github-actions'], + reporters: [ + 'default', + 'github-actions', + [ + 'jest-allure2-reporter', + { + resultsDir: 'allure-results', + }, + ], + ], testTimeout: 240_000, // 4min — pages can be tall and slow to capture maxWorkers: os.cpus().length * (process.env.GITHUB_ACTION ? 3 : 1), globalSetup: '/build-tools/visual/global-setup.js', diff --git a/package-lock.json b/package-lock.json index 555d3fabdc..7a1ac68bdb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -88,6 +88,7 @@ "html-webpack-plugin": "^5.5.0", "husky": "^9.0.0", "jest": "^29.7.0", + "jest-allure2-reporter": "^2.3.0", "jest-environment-jsdom": "^29.7.0", "lint-staged": "^15.2.2", "loader-utils": "^3.2.1", @@ -2083,6 +2084,13 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@flatten-js/interval-tree": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@flatten-js/interval-tree/-/interval-tree-1.1.4.tgz", + "integrity": "sha512-o4emRDDvGdkwX18BSVSXH8q27qAL7Z2WDHSN75C8xyRSE4A8UOkig0mWSGoT5M5KaTHZxoLmalFwOTQmbRusUg==", + "dev": true, + "license": "MIT" + }, "node_modules/@formatjs/ecma402-abstract": { "version": "2.3.4", "license": "MIT", @@ -5904,6 +5912,19 @@ "dev": true, "license": "MIT" }, + "node_modules/allure-store": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/allure-store/-/allure-store-1.2.0.tgz", + "integrity": "sha512-YUKLBrYA/qmZTqBgmC/ZgoTBTcbaFLV6OcQ/DsruWX5YwqVSCJUmMWi5cdGDzw5QL+b9EindJJ6KVHYjIL66Ww==", + "dev": true, + "license": "MIT", + "dependencies": { + "properties": "^1.2.1" + }, + "engines": { + "node": ">=16.14.0" + } + }, "node_modules/ansi-escapes": { "version": "4.3.2", "dev": true, @@ -6856,6 +6877,13 @@ "node": ">=8" } }, + "node_modules/browser-process-hrtime": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", + "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==", + "dev": true, + "license": "BSD-2-Clause" + }, "node_modules/browserslist": { "version": "4.28.1", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", @@ -6971,6 +6999,94 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bunyamin": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/bunyamin/-/bunyamin-1.6.3.tgz", + "integrity": "sha512-m1hAijFhu8pFiidsVc0XEDic46uxPK+mKNLqkb5mluNx0nTolNzx/DjwMqHChQWCgfOLMjKYJJ2uPTQLE6t4Ng==", + "dev": true, + "license": "MIT", + "dependencies": { + "@flatten-js/interval-tree": "^1.1.2", + "multi-sort-stream": "^1.0.4", + "stream-json": "^1.7.5", + "trace-event-lib": "^1.3.1" + }, + "engines": { + "node": ">=14.18.2" + }, + "peerDependencies": { + "@types/bunyan": "^1.8.8", + "bunyan": "^1.8.15 || ^2.0.0" + }, + "peerDependenciesMeta": { + "@types/bunyan": { + "optional": true + }, + "bunyan": { + "optional": true + } + } + }, + "node_modules/bunyan": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-2.0.5.tgz", + "integrity": "sha512-Jvl74TdxCN6rSP9W1I6+UOUtwslTDqsSFkDqZlFb/ilaSvQ+bZAnXT/GT97IZ5L+Vph0joPZPhxUyn6FLNmFAA==", + "dev": true, + "engines": [ + "node >=0.10.0" + ], + "license": "MIT", + "dependencies": { + "exeunt": "1.1.0" + }, + "bin": { + "bunyan": "bin/bunyan" + }, + "optionalDependencies": { + "dtrace-provider": "~0.8", + "moment": "^2.19.3", + "mv": "~2", + "safe-json-stringify": "~1" + } + }, + "node_modules/bunyan-debug-stream": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/bunyan-debug-stream/-/bunyan-debug-stream-3.1.1.tgz", + "integrity": "sha512-LfMcz4yKM6s9BP5dfT63Prb5B2hAjReLAfQzLbNQF7qBHtn3P1v+/yn0SZ6UAr4PC3VZRX/QzK7HYkkY0ytokQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2" + }, + "engines": { + "node": ">=0.12.0" + }, + "peerDependencies": { + "bunyan": "*" + }, + "peerDependenciesMeta": { + "bunyan": { + "optional": true + } + } + }, + "node_modules/bunyan-debug-stream/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, "node_modules/bytes": { "version": "3.1.2", "dev": true, @@ -9042,6 +9158,21 @@ "tslib": "^2.0.3" } }, + "node_modules/dtrace-provider": { + "version": "0.8.8", + "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz", + "integrity": "sha512-b7Z7cNtHPhH9EJhNNbbeqTcXB8LGFFZhq1PGgEvpeHlzd36bhbdTWoE/Ba/YguqpBSlAPKnARWhVlhunCMwfxg==", + "dev": true, + "hasInstallScript": true, + "license": "BSD-2-Clause", + "optional": true, + "dependencies": { + "nan": "^2.14.0" + }, + "engines": { + "node": ">=0.10" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "dev": true, @@ -9083,6 +9214,16 @@ "dev": true, "license": "MIT" }, + "node_modules/easy-stack": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/easy-stack/-/easy-stack-1.0.1.tgz", + "integrity": "sha512-wK2sCs4feiiJeFXn3zvY0p41mdU5VUgbgs1rNsc/y5ngFUijdWd+iIN8eoyuZHKB8xN6BL4PdWmzqFmxNg6V2w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/edge-paths": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/edge-paths/-/edge-paths-3.0.5.tgz", @@ -9318,6 +9459,16 @@ "is-arrayish": "^0.2.1" } }, + "node_modules/error-stack-parser": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.1.4.tgz", + "integrity": "sha512-Sk5V6wVazPhq5MhpO+AUxJn5x7XSXGl1R93Vn7i+zS15KDVxQijejNCrz8340/2bgLBjR9GtEG8ZVKONDjcqGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "stackframe": "^1.3.4" + } + }, "node_modules/es-abstract": { "version": "1.24.0", "dev": true, @@ -10154,6 +10305,16 @@ "node": ">= 0.6" } }, + "node_modules/event-pubsub": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/event-pubsub/-/event-pubsub-4.3.0.tgz", + "integrity": "sha512-z7IyloorXvKbFx9Bpie2+vMJKKx1fH1EN5yiTfp8CiLOTptSYy1g8H4yDpGlEdshL1PBiFtBHepF2cNsqeEeFQ==", + "dev": true, + "license": "Unlicense", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/event-target-shim": { "version": "5.0.1", "dev": true, @@ -10211,6 +10372,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/exeunt": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/exeunt/-/exeunt-1.1.0.tgz", + "integrity": "sha512-dd++Yn/0Fp+gtJ04YHov7MeAii+LFivJc6KqnJNfplzLVUkUDrfKoQDTLlCgzcW15vY5hKlHasWeIsQJ8agHsw==", + "dev": true, + "license": "MPL-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/exit": { "version": "0.1.2", "dev": true, @@ -11188,6 +11359,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/funpermaproxy": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/funpermaproxy/-/funpermaproxy-1.1.0.tgz", + "integrity": "sha512-2Sp1hWuO8m5fqeFDusyhKqYPT+7rGLw34N3qonDcdRP8+n7M7Gl/yKp/q7oCxnnJ6pWCectOmLFJpsMU/++KrQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8.3.0" + } + }, "node_modules/geckodriver": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/geckodriver/-/geckodriver-6.1.0.tgz", @@ -12365,6 +12546,19 @@ "node": ">=4" } }, + "node_modules/import-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/import-from/-/import-from-4.0.0.tgz", + "integrity": "sha512-P9J71vT5nLlDeV8FHs5nNxaLbrpfAV5cF5srvbZfpwpcJoM/xZR3hiv+q+SAnuSmuGbXMWud063iIMx/V/EWZQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.2" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/import-local": { "version": "3.2.0", "dev": true, @@ -13181,6 +13375,59 @@ } } }, + "node_modules/jest-allure2-reporter": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/jest-allure2-reporter/-/jest-allure2-reporter-2.3.0.tgz", + "integrity": "sha512-snDc5geSUcMbIegjqGuEPyYrZZxlajUH61Z+3EHQsJEF+m50lPonX9KQdqDC+xksCuwpM6mxGuBjQDGOIa2w6w==", + "dev": true, + "license": "MIT", + "workspaces": [ + "e2e", + "package-e2e" + ], + "dependencies": { + "allure-store": "^1.1.0", + "bunyamin": "^1.6.1", + "handlebars": "^4.7.8", + "import-from": "^4.0.0", + "jest-metadata": "^1.6.0", + "lodash": "^4.17.21", + "node-fetch": "^2.6.7", + "pkg-up": "^3.1.0", + "properties": "^1.2.1", + "stacktrace-js": "^2.0.2", + "strip-ansi": "^6.0.0", + "uuid": "^8.3.2" + }, + "engines": { + "node": ">=16.20.0" + }, + "peerDependencies": { + "jest": ">=27.2.5", + "jest-docblock": ">=27.2.5" + }, + "peerDependenciesMeta": { + "jest": { + "optional": true + }, + "jest-docblock": { + "optional": true + } + } + }, + "node_modules/jest-allure2-reporter/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/jest-changed-files": { "version": "29.7.0", "dev": true, @@ -13630,6 +13877,63 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/jest-environment-emit": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/jest-environment-emit/-/jest-environment-emit-1.2.0.tgz", + "integrity": "sha512-dSFBrRuIiWbHK2LSUA6CutXpMcNGjjuhvxFLF+TVz5tYFAAH0eesrZgrQ3UtOptajDYNt/fIGRqtlHqGq/bLbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "bunyamin": "^1.5.2", + "bunyan": "^2.0.5", + "bunyan-debug-stream": "^3.1.0", + "funpermaproxy": "^1.1.0", + "lodash.merge": "^4.6.2", + "node-ipc": "9.2.1", + "strip-ansi": "^6.0.0", + "tslib": "^2.5.3" + }, + "engines": { + "node": ">=16.14.0" + }, + "peerDependencies": { + "@jest/environment": ">=27.2.5", + "@jest/types": ">=27.2.5", + "jest": ">=27.2.5", + "jest-environment-jsdom": ">=27.2.5", + "jest-environment-node": ">=27.2.5" + }, + "peerDependenciesMeta": { + "@jest/environment": { + "optional": true + }, + "@jest/types": { + "optional": true + }, + "jest": { + "optional": true + }, + "jest-environment-jsdom": { + "optional": true + }, + "jest-environment-node": { + "optional": true + } + } + }, + "node_modules/jest-environment-emit/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/jest-environment-jsdom": { "version": "29.7.0", "dev": true, @@ -13879,6 +14183,67 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/jest-metadata": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/jest-metadata/-/jest-metadata-1.6.0.tgz", + "integrity": "sha512-penaOkD6tN0Vpqd+9xnbS+iYSLqaZpsx08gz44mOBvyNGBHPglnNKOsBMr3cbIe0bFYGlnouDy4N5SfLtNgVBQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bunyamin": "^1.5.2", + "funpermaproxy": "^1.1.0", + "jest-environment-emit": "^1.0.8", + "lodash.merge": "^4.6.2", + "lodash.snakecase": "^4.1.1", + "node-ipc": "9.2.1", + "strip-ansi": "^6.0.0", + "tslib": "^2.5.3" + }, + "engines": { + "node": ">=16.14.0" + }, + "peerDependencies": { + "@jest/environment": ">=27.2.5", + "@jest/reporters": ">=27.2.5", + "@jest/types": ">=27.2.5", + "jest": ">=27.2.5", + "jest-environment-jsdom": ">=27.2.5", + "jest-environment-node": ">=27.2.5" + }, + "peerDependenciesMeta": { + "@jest/environment": { + "optional": true + }, + "@jest/reporters": { + "optional": true + }, + "@jest/types": { + "optional": true + }, + "jest": { + "optional": true + }, + "jest-environment-jsdom": { + "optional": true + }, + "jest-environment-node": { + "optional": true + } + } + }, + "node_modules/jest-metadata/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/jest-mock": { "version": "29.7.0", "dev": true, @@ -14387,6 +14752,29 @@ "@sideway/pinpoint": "^2.0.0" } }, + "node_modules/js-message": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/js-message/-/js-message-1.0.7.tgz", + "integrity": "sha512-efJLHhLjIyKRewNS9EGZ4UpI8NguuL6fKkhRxVuMmrGV2xN/0APGdQYwLFky5w9naebSZ0OwAGp0G6/2Cg90rA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/js-queue": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/js-queue/-/js-queue-2.0.2.tgz", + "integrity": "sha512-pbKLsbCfi7kriM3s1J4DDCo7jQkI58zPLHi0heXPzPlj0hjUsm+FesPUbE0DSbIVIK503A36aUBoCN7eMFedkA==", + "dev": true, + "license": "MIT", + "dependencies": { + "easy-stack": "^1.0.1" + }, + "engines": { + "node": ">=1.0.0" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "license": "MIT" @@ -15058,6 +15446,13 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.snakecase": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz", + "integrity": "sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw==", + "dev": true, + "license": "MIT" + }, "node_modules/lodash.truncate": { "version": "4.4.2", "dev": true, @@ -15539,6 +15934,20 @@ "dev": true, "license": "MIT" }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, "node_modules/mnth": { "version": "2.0.0", "license": "MIT", @@ -15577,6 +15986,13 @@ "dev": true, "license": "MIT" }, + "node_modules/multi-sort-stream": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/multi-sort-stream/-/multi-sort-stream-1.0.4.tgz", + "integrity": "sha512-hAZ8JOEQFbgdLe8HWZbb7gdZg0/yAIHF00Qfo3kd0rXFv96nXe+/bPTrKHZ2QMHugGX4FiAyET1Lt+jiB+7Qlg==", + "dev": true, + "license": "bsd" + }, "node_modules/multicast-dns": { "version": "7.2.5", "dev": true, @@ -15630,6 +16046,78 @@ "node": ">= 10.13.0" } }, + "node_modules/mv": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", + "integrity": "sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "mkdirp": "~0.5.1", + "ncp": "~2.0.0", + "rimraf": "~2.4.0" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/mv/node_modules/glob": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", + "integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "optional": true, + "dependencies": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mv/node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "dev": true, + "license": "ISC", + "optional": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mv/node_modules/rimraf": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", + "integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dev": true, + "license": "ISC", + "optional": true, + "dependencies": { + "glob": "^6.0.1" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/nan": { + "version": "2.27.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.27.0.tgz", + "integrity": "sha512-hC+0LidcL3XE4rp1C4H54KujgXKzbfyTngZTwBByQxsOxCEKZT0MPQ4hOKUH2jU1OYstqdDH4onyHPDzcV0XdQ==", + "dev": true, + "license": "MIT", + "optional": true + }, "node_modules/nanoid": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.6.tgz", @@ -15662,6 +16150,17 @@ "dev": true, "license": "MIT" }, + "node_modules/ncp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", + "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==", + "dev": true, + "license": "MIT", + "optional": true, + "bin": { + "ncp": "bin/ncp" + } + }, "node_modules/negotiator": { "version": "0.6.4", "dev": true, @@ -15705,11 +16204,72 @@ "license": "MIT", "optional": true }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-fetch/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-fetch/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/node-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/node-int64": { "version": "0.4.0", "dev": true, "license": "MIT" }, + "node_modules/node-ipc": { + "version": "9.2.1", + "resolved": "https://registry.npmjs.org/node-ipc/-/node-ipc-9.2.1.tgz", + "integrity": "sha512-mJzaM6O3xHf9VT8BULvJSbdVbmHUKRNOH7zDDkCrA1/T+CVjq2WVIDfLt0azZRXpgArJtl3rtmEozrbXPZ9GaQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "event-pubsub": "4.3.0", + "js-message": "1.0.7", + "js-queue": "2.0.2" + }, + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/node-releases": { "version": "2.0.27", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", @@ -16684,6 +17244,85 @@ "node": ">=8" } }, + "node_modules/pkg-up": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/pkg-up/-/pkg-up-3.1.0.tgz", + "integrity": "sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==", + "dev": true, + "license": "MIT", + "dependencies": { + "find-up": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-up/node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-up/node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-up/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pkg-up/node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-up/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/pkijs": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/pkijs/-/pkijs-3.4.0.tgz", @@ -17585,6 +18224,16 @@ "version": "16.13.1", "license": "MIT" }, + "node_modules/properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/properties/-/properties-1.2.1.tgz", + "integrity": "sha512-qYNxyMj1JeW54i/EWEFsM1cVwxJbtgPp8+0Wg9XjNaK6VE/c4oRi6PNu5p7w1mNXEIQIjV5Wwn8v8Gz82/QzdQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "dev": true, @@ -18762,6 +19411,14 @@ ], "license": "MIT" }, + "node_modules/safe-json-stringify": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz", + "integrity": "sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg==", + "dev": true, + "license": "MIT", + "optional": true + }, "node_modules/safe-push-apply": { "version": "1.0.0", "dev": true, @@ -19669,6 +20326,16 @@ "dev": true, "license": "MIT" }, + "node_modules/stack-generator": { + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/stack-generator/-/stack-generator-2.0.10.tgz", + "integrity": "sha512-mwnua/hkqM6pF4k8SnmZ2zfETsRUpWXREfA/goT8SLCV4iOFa4bzOX2nDipWAZFPTjLvQB82f5yaodMVhK0yJQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "stackframe": "^1.3.4" + } + }, "node_modules/stack-utils": { "version": "2.0.6", "dev": true, @@ -19688,6 +20355,46 @@ "node": ">=8" } }, + "node_modules/stackframe": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.3.4.tgz", + "integrity": "sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==", + "dev": true, + "license": "MIT" + }, + "node_modules/stacktrace-gps": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/stacktrace-gps/-/stacktrace-gps-3.1.2.tgz", + "integrity": "sha512-GcUgbO4Jsqqg6RxfyTHFiPxdPqF+3LFmQhm7MgCuYQOYuWyqxo5pwRPz5d/u6/WYJdEnWfK4r+jGbyD8TSggXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "source-map": "0.5.6", + "stackframe": "^1.3.4" + } + }, + "node_modules/stacktrace-gps/node_modules/source-map": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.6.tgz", + "integrity": "sha512-MjZkVp0NHr5+TPihLcadqnlVoGIoWo4IBHptutGh9wI3ttUYvCG26HkSuDi+K6lsZ25syXJXcctwgyVCt//xqA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stacktrace-js": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/stacktrace-js/-/stacktrace-js-2.0.2.tgz", + "integrity": "sha512-Je5vBeY4S1r/RnLydLl0TBTi3F2qdfWmYsGvtfZgEI+SCprPppaIhQf5nGcal4gI4cGpCV/duLcAzT1np6sQqg==", + "dev": true, + "license": "MIT", + "dependencies": { + "error-stack-parser": "^2.0.6", + "stack-generator": "^2.0.5", + "stacktrace-gps": "^3.0.4" + } + }, "node_modules/statuses": { "version": "2.0.1", "dev": true, @@ -19708,6 +20415,13 @@ "node": ">= 0.4" } }, + "node_modules/stream-chain": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", + "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/stream-composer": { "version": "1.0.2", "dev": true, @@ -19730,6 +20444,16 @@ "dev": true, "license": "MIT" }, + "node_modules/stream-json": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.9.1.tgz", + "integrity": "sha512-uWkjJ+2Nt/LO9Z/JyKZbMusL8Dkh97uUBTv3AJQ74y07lVahLY4eEFsPsE97pxYBwr8nnjMAIch5eqI0gPShyw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "stream-chain": "^2.2.5" + } + }, "node_modules/stream-shift": { "version": "1.0.3", "dev": true, @@ -20905,6 +21629,19 @@ "node": ">=12" } }, + "node_modules/trace-event-lib": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/trace-event-lib/-/trace-event-lib-1.4.1.tgz", + "integrity": "sha512-TOgFolKG8JFY+9d5EohGWMvwvteRafcyfPWWNIqcuD1W/FUvxWcy2MSCZ/beYHM63oYPHYHCd3tkbgCctHVP7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "browser-process-hrtime": "^1.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/tree-dump": { "version": "1.0.3", "dev": true, diff --git a/package.json b/package.json index 8110f028b0..faaf828b39 100644 --- a/package.json +++ b/package.json @@ -111,6 +111,7 @@ "html-webpack-plugin": "^5.5.0", "husky": "^9.0.0", "jest": "^29.7.0", + "jest-allure2-reporter": "^2.3.0", "jest-environment-jsdom": "^29.7.0", "lint-staged": "^15.2.2", "loader-utils": "^3.2.1", From 3537daa3c56c0ebc4e319c6cb1136bd6236e5d89 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 5 Jun 2026 19:30:35 +0200 Subject: [PATCH 74/81] Increase to 10 shards --- .github/workflows/visual-regression.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 0b4e20a816..4942a136f9 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -82,7 +82,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8] + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] steps: - uses: actions/checkout@v4 @@ -137,7 +137,7 @@ jobs: run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081 - name: Run visual regression tests - run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js --shard=${{ matrix.shard }}/8 + run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js --shard=${{ matrix.shard }}/10 env: TZ: UTC From 3ca00aa0843fee00d4ba04abeb034e86798f4b3c Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 5 Jun 2026 19:33:04 +0200 Subject: [PATCH 75/81] Download Allure directly --- .github/workflows/visual-regression.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 4942a136f9..6c37dafdb5 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -170,11 +170,13 @@ jobs: path: allure-results merge-multiple: true + - name: Install Allure CLI + run: | + curl -sL https://github.com/allure-framework/allure2/releases/download/2.32.0/allure-2.32.0.tgz | tar -xz + echo "$PWD/allure-2.32.0/bin" >> $GITHUB_PATH + - name: Generate Allure HTML report - uses: simple-elf/allure-report-action@v1.12 - with: - allure_results: allure-results - allure_report: allure-report + run: allure generate allure-results --clean -o allure-report - name: Upload Allure report uses: actions/upload-artifact@v4 From 9342f83c9406044267843315119543cf4ece86fa Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 5 Jun 2026 23:22:46 +0200 Subject: [PATCH 76/81] Deploy Allure results --- .github/workflows/visual-regression.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 6c37dafdb5..ded9ff9344 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -24,6 +24,7 @@ permissions: id-token: write contents: read actions: read + deployments: write jobs: # Build the baseline (main branch) pages once and share them across all browser jobs. @@ -178,9 +179,19 @@ jobs: - name: Generate Allure HTML report run: allure generate allure-results --clean -o allure-report - - name: Upload Allure report + - name: Upload Allure report artifact uses: actions/upload-artifact@v4 with: name: allure-report path: allure-report/ retention-days: 14 + + deploy-report: + name: Deploy Allure Report + if: always() + needs: [report] + uses: cloudscape-design/actions/.github/workflows/deploy.yml@main + secrets: inherit + with: + artifact-name: allure-report + deployment-path: allure-report From e69aede82f0d33d1a203d358efee9c3a0226251c Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 5 Jun 2026 23:22:59 +0200 Subject: [PATCH 77/81] Add image diffs to Allure reports --- test/definitions/utils.ts | 71 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 5 deletions(-) diff --git a/test/definitions/utils.ts b/test/definitions/utils.ts index 683e50cc80..1b4dd0fee2 100644 --- a/test/definitions/utils.ts +++ b/test/definitions/utils.ts @@ -1,5 +1,9 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import * as crypto from 'crypto'; +import * as fs from 'fs'; +import * as path from 'path'; + import { cropAndCompare } from '@cloudscape-design/browser-test-tools/image-utils'; import { ScreenshotPageObject, ScreenshotWithOffset } from '@cloudscape-design/browser-test-tools/page-objects'; @@ -12,6 +16,8 @@ const defaultWindowSize = { width: 1600, height: 800 }; const newHost = process.env.NEW_HOST || 'http://localhost:8080'; const oldHost = process.env.OLD_HOST || 'http://localhost:8081'; +const allureResultsDir = path.resolve(process.cwd(), 'allure-results'); + function buildUrl(host: string, path: string, queryParams?: Record): string { const params = new URLSearchParams(queryParams); const qs = params.toString(); @@ -22,6 +28,55 @@ function isTestDefinition(item: TestDefinition | TestSuite): item is TestDefinit return (item as TestDefinition).path !== undefined; } +/** + * Writes a PNG buffer to the allure-results directory and returns the filename. + * Allure picks up files in this directory and attaches them to the report. + */ +function writeAllureAttachment(buffer: Buffer): string { + if (!fs.existsSync(allureResultsDir)) { + fs.mkdirSync(allureResultsDir, { recursive: true }); + } + const uuid = crypto.randomUUID(); + const filename = `${uuid}-attachment.png`; + fs.writeFileSync(path.join(allureResultsDir, filename), buffer); + return filename; +} + +/** + * Attaches visual diff images (new, baseline, diff) to the Allure results + * for the current test by writing an attachment JSON file. + */ +function attachDiffImages( + result: { firstImage: Buffer; secondImage: Buffer; diffImage: Buffer | null }, + testName: string +): void { + const newFile = writeAllureAttachment(result.firstImage); + const baselineFile = writeAllureAttachment(result.secondImage); + + const attachments: Array<{ name: string; source: string; type: string }> = [ + { name: `${testName} — new (PR)`, source: newFile, type: 'image/png' }, + { name: `${testName} — baseline (main)`, source: baselineFile, type: 'image/png' }, + ]; + + if (result.diffImage) { + const diffFile = writeAllureAttachment(result.diffImage); + attachments.push({ name: `${testName} — diff`, source: diffFile, type: 'image/png' }); + } + + // Write a container JSON that Allure merges into the test result. + // jest-allure2-reporter reads attachment files from allure-results/. + const containerUuid = crypto.randomUUID(); + const containerFile = path.join(allureResultsDir, `${containerUuid}-container.json`); + fs.writeFileSync( + containerFile, + JSON.stringify({ + uuid: containerUuid, + name: testName, + attachments, + }) + ); +} + /** * Registers all test suites with a single shared browser session per worker. * This avoids the per-test session creation overhead. @@ -91,9 +146,9 @@ function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Bro const newScreenshot = await capture(browser, page, newUrl, testDef, windowSize); const oldScreenshot = await capture(browser, page, oldUrl, testDef, windowSize); - const { diffPixels } = await cropAndCompare(newScreenshot, oldScreenshot); + const result = await cropAndCompare(newScreenshot, oldScreenshot); - if (diffPixels === 0) { + if (result.diffPixels === 0) { return; } @@ -124,13 +179,19 @@ function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Bro expect(newPermutations.length).toBe(oldPermutations.length); for (let i = 0; i < newPermutations.length; i++) { - const { diffPixels: permDiff } = await cropAndCompare(newPermutations[i], oldPermutations[i]); - expect(permDiff).toBe(0); + const permResult = await cropAndCompare(newPermutations[i], oldPermutations[i]); + if (permResult.diffPixels !== 0) { + attachDiffImages(permResult, `${testDef.description} [permutation ${i}]`); + } + expect(permResult.diffPixels).toBe(0); } return; } + // Attach diff images to Allure report for visual inspection. + attachDiffImages(result, testDef.description); + // For screenshotArea and viewport types, the diff is a real failure. - expect(diffPixels).toBe(0); + expect(result.diffPixels).toBe(0); }); } From 42cc955b4772f8870c0c6cd31045f1fb0a08e432 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Fri, 5 Jun 2026 23:25:33 +0200 Subject: [PATCH 78/81] Use Allure 3 --- .github/workflows/visual-regression.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index ded9ff9344..977c6cd96d 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -164,6 +164,11 @@ jobs: needs: [visual] runs-on: ubuntu-latest steps: + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + - name: Download all Allure results uses: actions/download-artifact@v4 with: @@ -171,13 +176,8 @@ jobs: path: allure-results merge-multiple: true - - name: Install Allure CLI - run: | - curl -sL https://github.com/allure-framework/allure2/releases/download/2.32.0/allure-2.32.0.tgz | tar -xz - echo "$PWD/allure-2.32.0/bin" >> $GITHUB_PATH - - name: Generate Allure HTML report - run: allure generate allure-results --clean -o allure-report + run: npx --yes allure generate allure-results -o allure-report - name: Upload Allure report artifact uses: actions/upload-artifact@v4 From e8257320701c5231fe3bbf86c5f18ed40d84d8f6 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Sat, 6 Jun 2026 08:28:36 +0200 Subject: [PATCH 79/81] Fix image attachments --- jest.visual.config.js | 16 +- package-lock.json | 829 +++++--------------------------------- package.json | 2 +- test/definitions/utils.ts | 58 +-- 4 files changed, 108 insertions(+), 797 deletions(-) diff --git a/jest.visual.config.js b/jest.visual.config.js index 5cb7f033fe..98938768b3 100644 --- a/jest.visual.config.js +++ b/jest.visual.config.js @@ -5,7 +5,10 @@ const os = require('os'); module.exports = { verbose: true, - testEnvironment: 'node', + testEnvironment: 'allure-jest/node', + testEnvironmentOptions: { + resultsDir: 'allure-results', + }, transform: { '^.+\\.tsx?$': [ 'ts-jest', @@ -14,16 +17,7 @@ module.exports = { }, ], }, - reporters: [ - 'default', - 'github-actions', - [ - 'jest-allure2-reporter', - { - resultsDir: 'allure-results', - }, - ], - ], + reporters: ['default', 'github-actions'], testTimeout: 240_000, // 4min — pages can be tall and slow to capture maxWorkers: os.cpus().length * (process.env.GITHUB_ACTION ? 3 : 1), globalSetup: '/build-tools/visual/global-setup.js', diff --git a/package-lock.json b/package-lock.json index 7a1ac68bdb..22ac706458 100644 --- a/package-lock.json +++ b/package-lock.json @@ -61,6 +61,7 @@ "@types/react-test-renderer": "^16.9.12", "@types/react-transition-group": "^4.4.4", "@types/webpack-env": "^1.16.3", + "allure-jest": "^3.9.0", "axe-core": "^4.7.2", "babel-jest": "^29.7.0", "change-case": "^4.1.2", @@ -88,7 +89,6 @@ "html-webpack-plugin": "^5.5.0", "husky": "^9.0.0", "jest": "^29.7.0", - "jest-allure2-reporter": "^2.3.0", "jest-environment-jsdom": "^29.7.0", "lint-staged": "^15.2.2", "loader-utils": "^3.2.1", @@ -2084,13 +2084,6 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, - "node_modules/@flatten-js/interval-tree": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/@flatten-js/interval-tree/-/interval-tree-1.1.4.tgz", - "integrity": "sha512-o4emRDDvGdkwX18BSVSXH8q27qAL7Z2WDHSN75C8xyRSE4A8UOkig0mWSGoT5M5KaTHZxoLmalFwOTQmbRusUg==", - "dev": true, - "license": "MIT" - }, "node_modules/@formatjs/ecma402-abstract": { "version": "2.3.4", "license": "MIT", @@ -5912,17 +5905,56 @@ "dev": true, "license": "MIT" }, - "node_modules/allure-store": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/allure-store/-/allure-store-1.2.0.tgz", - "integrity": "sha512-YUKLBrYA/qmZTqBgmC/ZgoTBTcbaFLV6OcQ/DsruWX5YwqVSCJUmMWi5cdGDzw5QL+b9EindJJ6KVHYjIL66Ww==", + "node_modules/allure-jest": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/allure-jest/-/allure-jest-3.9.0.tgz", + "integrity": "sha512-hEW4DKjvb3engGoHUPQaDEdyrFkUxQnqULiSQAehL1eDEggqdPbQro86Nch8Cj1yuIqUTn9UP1FMuuuwl/5jnQ==", "dev": true, - "license": "MIT", + "license": "Apache-2.0", "dependencies": { - "properties": "^1.2.1" + "allure-js-commons": "3.9.0" }, - "engines": { - "node": ">=16.14.0" + "peerDependencies": { + "jest": ">=24.8.0", + "jest-circus": ">=24.8.0", + "jest-cli": ">=24.8.0", + "jest-environment-jsdom": ">=24.8.0", + "jest-environment-node": ">=24.8.0" + }, + "peerDependenciesMeta": { + "jest": { + "optional": true + }, + "jest-circus": { + "optional": true + }, + "jest-cli": { + "optional": true + }, + "jest-environment-jsdom": { + "optional": true + }, + "jest-environment-node": { + "optional": true + } + } + }, + "node_modules/allure-js-commons": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/allure-js-commons/-/allure-js-commons-3.9.0.tgz", + "integrity": "sha512-uVQcGE6MWIvGR/zW1XEUwHXUQa1EJKY0Cah+0TZK1qKuw6ptyhftDr34XE3wExTyCZirRrI98dbRtPeYYuyI+g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "md5": "^2.3.0" + }, + "peerDependencies": { + "allure-playwright": "3.9.0" + }, + "peerDependenciesMeta": { + "allure-playwright": { + "optional": true + } } }, "node_modules/ansi-escapes": { @@ -6877,13 +6909,6 @@ "node": ">=8" } }, - "node_modules/browser-process-hrtime": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", - "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==", - "dev": true, - "license": "BSD-2-Clause" - }, "node_modules/browserslist": { "version": "4.28.1", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", @@ -6999,94 +7024,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/bunyamin": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/bunyamin/-/bunyamin-1.6.3.tgz", - "integrity": "sha512-m1hAijFhu8pFiidsVc0XEDic46uxPK+mKNLqkb5mluNx0nTolNzx/DjwMqHChQWCgfOLMjKYJJ2uPTQLE6t4Ng==", - "dev": true, - "license": "MIT", - "dependencies": { - "@flatten-js/interval-tree": "^1.1.2", - "multi-sort-stream": "^1.0.4", - "stream-json": "^1.7.5", - "trace-event-lib": "^1.3.1" - }, - "engines": { - "node": ">=14.18.2" - }, - "peerDependencies": { - "@types/bunyan": "^1.8.8", - "bunyan": "^1.8.15 || ^2.0.0" - }, - "peerDependenciesMeta": { - "@types/bunyan": { - "optional": true - }, - "bunyan": { - "optional": true - } - } - }, - "node_modules/bunyan": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-2.0.5.tgz", - "integrity": "sha512-Jvl74TdxCN6rSP9W1I6+UOUtwslTDqsSFkDqZlFb/ilaSvQ+bZAnXT/GT97IZ5L+Vph0joPZPhxUyn6FLNmFAA==", - "dev": true, - "engines": [ - "node >=0.10.0" - ], - "license": "MIT", - "dependencies": { - "exeunt": "1.1.0" - }, - "bin": { - "bunyan": "bin/bunyan" - }, - "optionalDependencies": { - "dtrace-provider": "~0.8", - "moment": "^2.19.3", - "mv": "~2", - "safe-json-stringify": "~1" - } - }, - "node_modules/bunyan-debug-stream": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/bunyan-debug-stream/-/bunyan-debug-stream-3.1.1.tgz", - "integrity": "sha512-LfMcz4yKM6s9BP5dfT63Prb5B2hAjReLAfQzLbNQF7qBHtn3P1v+/yn0SZ6UAr4PC3VZRX/QzK7HYkkY0ytokQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.1.2" - }, - "engines": { - "node": ">=0.12.0" - }, - "peerDependencies": { - "bunyan": "*" - }, - "peerDependenciesMeta": { - "bunyan": { - "optional": true - } - } - }, - "node_modules/bunyan-debug-stream/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, "node_modules/bytes": { "version": "3.1.2", "dev": true, @@ -7282,6 +7219,16 @@ "node": ">=10" } }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, "node_modules/cheerio": { "version": "1.1.0", "dev": true, @@ -7990,6 +7937,16 @@ "node": ">= 8" } }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, "node_modules/css-declaration-sorter": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/css-declaration-sorter/-/css-declaration-sorter-7.3.1.tgz", @@ -9158,21 +9115,6 @@ "tslib": "^2.0.3" } }, - "node_modules/dtrace-provider": { - "version": "0.8.8", - "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz", - "integrity": "sha512-b7Z7cNtHPhH9EJhNNbbeqTcXB8LGFFZhq1PGgEvpeHlzd36bhbdTWoE/Ba/YguqpBSlAPKnARWhVlhunCMwfxg==", - "dev": true, - "hasInstallScript": true, - "license": "BSD-2-Clause", - "optional": true, - "dependencies": { - "nan": "^2.14.0" - }, - "engines": { - "node": ">=0.10" - } - }, "node_modules/dunder-proto": { "version": "1.0.1", "dev": true, @@ -9214,16 +9156,6 @@ "dev": true, "license": "MIT" }, - "node_modules/easy-stack": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/easy-stack/-/easy-stack-1.0.1.tgz", - "integrity": "sha512-wK2sCs4feiiJeFXn3zvY0p41mdU5VUgbgs1rNsc/y5ngFUijdWd+iIN8eoyuZHKB8xN6BL4PdWmzqFmxNg6V2w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/edge-paths": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/edge-paths/-/edge-paths-3.0.5.tgz", @@ -9459,16 +9391,6 @@ "is-arrayish": "^0.2.1" } }, - "node_modules/error-stack-parser": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.1.4.tgz", - "integrity": "sha512-Sk5V6wVazPhq5MhpO+AUxJn5x7XSXGl1R93Vn7i+zS15KDVxQijejNCrz8340/2bgLBjR9GtEG8ZVKONDjcqGQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "stackframe": "^1.3.4" - } - }, "node_modules/es-abstract": { "version": "1.24.0", "dev": true, @@ -10305,16 +10227,6 @@ "node": ">= 0.6" } }, - "node_modules/event-pubsub": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/event-pubsub/-/event-pubsub-4.3.0.tgz", - "integrity": "sha512-z7IyloorXvKbFx9Bpie2+vMJKKx1fH1EN5yiTfp8CiLOTptSYy1g8H4yDpGlEdshL1PBiFtBHepF2cNsqeEeFQ==", - "dev": true, - "license": "Unlicense", - "engines": { - "node": ">=4.0.0" - } - }, "node_modules/event-target-shim": { "version": "5.0.1", "dev": true, @@ -10372,16 +10284,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/exeunt": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/exeunt/-/exeunt-1.1.0.tgz", - "integrity": "sha512-dd++Yn/0Fp+gtJ04YHov7MeAii+LFivJc6KqnJNfplzLVUkUDrfKoQDTLlCgzcW15vY5hKlHasWeIsQJ8agHsw==", - "dev": true, - "license": "MPL-2.0", - "engines": { - "node": ">=0.10" - } - }, "node_modules/exit": { "version": "0.1.2", "dev": true, @@ -11359,16 +11261,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/funpermaproxy": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/funpermaproxy/-/funpermaproxy-1.1.0.tgz", - "integrity": "sha512-2Sp1hWuO8m5fqeFDusyhKqYPT+7rGLw34N3qonDcdRP8+n7M7Gl/yKp/q7oCxnnJ6pWCectOmLFJpsMU/++KrQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=8.3.0" - } - }, "node_modules/geckodriver": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/geckodriver/-/geckodriver-6.1.0.tgz", @@ -12546,19 +12438,6 @@ "node": ">=4" } }, - "node_modules/import-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/import-from/-/import-from-4.0.0.tgz", - "integrity": "sha512-P9J71vT5nLlDeV8FHs5nNxaLbrpfAV5cF5srvbZfpwpcJoM/xZR3hiv+q+SAnuSmuGbXMWud063iIMx/V/EWZQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.2" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/import-local": { "version": "3.2.0", "dev": true, @@ -12786,6 +12665,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dev": true, + "license": "MIT" + }, "node_modules/is-builtin-module": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-5.0.0.tgz", @@ -13375,59 +13261,6 @@ } } }, - "node_modules/jest-allure2-reporter": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/jest-allure2-reporter/-/jest-allure2-reporter-2.3.0.tgz", - "integrity": "sha512-snDc5geSUcMbIegjqGuEPyYrZZxlajUH61Z+3EHQsJEF+m50lPonX9KQdqDC+xksCuwpM6mxGuBjQDGOIa2w6w==", - "dev": true, - "license": "MIT", - "workspaces": [ - "e2e", - "package-e2e" - ], - "dependencies": { - "allure-store": "^1.1.0", - "bunyamin": "^1.6.1", - "handlebars": "^4.7.8", - "import-from": "^4.0.0", - "jest-metadata": "^1.6.0", - "lodash": "^4.17.21", - "node-fetch": "^2.6.7", - "pkg-up": "^3.1.0", - "properties": "^1.2.1", - "stacktrace-js": "^2.0.2", - "strip-ansi": "^6.0.0", - "uuid": "^8.3.2" - }, - "engines": { - "node": ">=16.20.0" - }, - "peerDependencies": { - "jest": ">=27.2.5", - "jest-docblock": ">=27.2.5" - }, - "peerDependenciesMeta": { - "jest": { - "optional": true - }, - "jest-docblock": { - "optional": true - } - } - }, - "node_modules/jest-allure2-reporter/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/jest-changed-files": { "version": "29.7.0", "dev": true, @@ -13877,63 +13710,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/jest-environment-emit": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/jest-environment-emit/-/jest-environment-emit-1.2.0.tgz", - "integrity": "sha512-dSFBrRuIiWbHK2LSUA6CutXpMcNGjjuhvxFLF+TVz5tYFAAH0eesrZgrQ3UtOptajDYNt/fIGRqtlHqGq/bLbA==", - "dev": true, - "license": "MIT", - "dependencies": { - "bunyamin": "^1.5.2", - "bunyan": "^2.0.5", - "bunyan-debug-stream": "^3.1.0", - "funpermaproxy": "^1.1.0", - "lodash.merge": "^4.6.2", - "node-ipc": "9.2.1", - "strip-ansi": "^6.0.0", - "tslib": "^2.5.3" - }, - "engines": { - "node": ">=16.14.0" - }, - "peerDependencies": { - "@jest/environment": ">=27.2.5", - "@jest/types": ">=27.2.5", - "jest": ">=27.2.5", - "jest-environment-jsdom": ">=27.2.5", - "jest-environment-node": ">=27.2.5" - }, - "peerDependenciesMeta": { - "@jest/environment": { - "optional": true - }, - "@jest/types": { - "optional": true - }, - "jest": { - "optional": true - }, - "jest-environment-jsdom": { - "optional": true - }, - "jest-environment-node": { - "optional": true - } - } - }, - "node_modules/jest-environment-emit/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/jest-environment-jsdom": { "version": "29.7.0", "dev": true, @@ -14183,75 +13959,14 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/jest-metadata": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/jest-metadata/-/jest-metadata-1.6.0.tgz", - "integrity": "sha512-penaOkD6tN0Vpqd+9xnbS+iYSLqaZpsx08gz44mOBvyNGBHPglnNKOsBMr3cbIe0bFYGlnouDy4N5SfLtNgVBQ==", + "node_modules/jest-mock": { + "version": "29.7.0", "dev": true, "license": "MIT", "dependencies": { - "bunyamin": "^1.5.2", - "funpermaproxy": "^1.1.0", - "jest-environment-emit": "^1.0.8", - "lodash.merge": "^4.6.2", - "lodash.snakecase": "^4.1.1", - "node-ipc": "9.2.1", - "strip-ansi": "^6.0.0", - "tslib": "^2.5.3" - }, - "engines": { - "node": ">=16.14.0" - }, - "peerDependencies": { - "@jest/environment": ">=27.2.5", - "@jest/reporters": ">=27.2.5", - "@jest/types": ">=27.2.5", - "jest": ">=27.2.5", - "jest-environment-jsdom": ">=27.2.5", - "jest-environment-node": ">=27.2.5" - }, - "peerDependenciesMeta": { - "@jest/environment": { - "optional": true - }, - "@jest/reporters": { - "optional": true - }, - "@jest/types": { - "optional": true - }, - "jest": { - "optional": true - }, - "jest-environment-jsdom": { - "optional": true - }, - "jest-environment-node": { - "optional": true - } - } - }, - "node_modules/jest-metadata/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/jest-mock": { - "version": "29.7.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@types/node": "*", - "jest-util": "^29.7.0" + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-util": "^29.7.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" @@ -14752,29 +14467,6 @@ "@sideway/pinpoint": "^2.0.0" } }, - "node_modules/js-message": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/js-message/-/js-message-1.0.7.tgz", - "integrity": "sha512-efJLHhLjIyKRewNS9EGZ4UpI8NguuL6fKkhRxVuMmrGV2xN/0APGdQYwLFky5w9naebSZ0OwAGp0G6/2Cg90rA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.6.0" - } - }, - "node_modules/js-queue": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/js-queue/-/js-queue-2.0.2.tgz", - "integrity": "sha512-pbKLsbCfi7kriM3s1J4DDCo7jQkI58zPLHi0heXPzPlj0hjUsm+FesPUbE0DSbIVIK503A36aUBoCN7eMFedkA==", - "dev": true, - "license": "MIT", - "dependencies": { - "easy-stack": "^1.0.1" - }, - "engines": { - "node": ">=1.0.0" - } - }, "node_modules/js-tokens": { "version": "4.0.0", "license": "MIT" @@ -15446,13 +15138,6 @@ "dev": true, "license": "MIT" }, - "node_modules/lodash.snakecase": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz", - "integrity": "sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw==", - "dev": true, - "license": "MIT" - }, "node_modules/lodash.truncate": { "version": "4.4.2", "dev": true, @@ -15666,6 +15351,18 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, "node_modules/mdn-data": { "version": "2.12.2", "dev": true, @@ -15934,20 +15631,6 @@ "dev": true, "license": "MIT" }, - "node_modules/mkdirp": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", - "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "minimist": "^1.2.6" - }, - "bin": { - "mkdirp": "bin/cmd.js" - } - }, "node_modules/mnth": { "version": "2.0.0", "license": "MIT", @@ -15986,13 +15669,6 @@ "dev": true, "license": "MIT" }, - "node_modules/multi-sort-stream": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/multi-sort-stream/-/multi-sort-stream-1.0.4.tgz", - "integrity": "sha512-hAZ8JOEQFbgdLe8HWZbb7gdZg0/yAIHF00Qfo3kd0rXFv96nXe+/bPTrKHZ2QMHugGX4FiAyET1Lt+jiB+7Qlg==", - "dev": true, - "license": "bsd" - }, "node_modules/multicast-dns": { "version": "7.2.5", "dev": true, @@ -16046,78 +15722,6 @@ "node": ">= 10.13.0" } }, - "node_modules/mv": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", - "integrity": "sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "mkdirp": "~0.5.1", - "ncp": "~2.0.0", - "rimraf": "~2.4.0" - }, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/mv/node_modules/glob": { - "version": "6.0.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", - "integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==", - "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", - "dev": true, - "license": "ISC", - "optional": true, - "dependencies": { - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "2 || 3", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - } - }, - "node_modules/mv/node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "dev": true, - "license": "ISC", - "optional": true, - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/mv/node_modules/rimraf": { - "version": "2.4.5", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", - "integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==", - "deprecated": "Rimraf versions prior to v4 are no longer supported", - "dev": true, - "license": "ISC", - "optional": true, - "dependencies": { - "glob": "^6.0.1" - }, - "bin": { - "rimraf": "bin.js" - } - }, - "node_modules/nan": { - "version": "2.27.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.27.0.tgz", - "integrity": "sha512-hC+0LidcL3XE4rp1C4H54KujgXKzbfyTngZTwBByQxsOxCEKZT0MPQ4hOKUH2jU1OYstqdDH4onyHPDzcV0XdQ==", - "dev": true, - "license": "MIT", - "optional": true - }, "node_modules/nanoid": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.6.tgz", @@ -16150,17 +15754,6 @@ "dev": true, "license": "MIT" }, - "node_modules/ncp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", - "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==", - "dev": true, - "license": "MIT", - "optional": true, - "bin": { - "ncp": "bin/ncp" - } - }, "node_modules/negotiator": { "version": "0.6.4", "dev": true, @@ -16204,72 +15797,11 @@ "license": "MIT", "optional": true }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dev": true, - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/node-fetch/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "dev": true, - "license": "MIT" - }, - "node_modules/node-fetch/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "dev": true, - "license": "BSD-2-Clause" - }, - "node_modules/node-fetch/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dev": true, - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/node-int64": { "version": "0.4.0", "dev": true, "license": "MIT" }, - "node_modules/node-ipc": { - "version": "9.2.1", - "resolved": "https://registry.npmjs.org/node-ipc/-/node-ipc-9.2.1.tgz", - "integrity": "sha512-mJzaM6O3xHf9VT8BULvJSbdVbmHUKRNOH7zDDkCrA1/T+CVjq2WVIDfLt0azZRXpgArJtl3rtmEozrbXPZ9GaQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "event-pubsub": "4.3.0", - "js-message": "1.0.7", - "js-queue": "2.0.2" - }, - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/node-releases": { "version": "2.0.27", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", @@ -17244,85 +16776,6 @@ "node": ">=8" } }, - "node_modules/pkg-up": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/pkg-up/-/pkg-up-3.1.0.tgz", - "integrity": "sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==", - "dev": true, - "license": "MIT", - "dependencies": { - "find-up": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pkg-up/node_modules/find-up": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", - "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/pkg-up/node_modules/locate-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", - "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^3.0.0", - "path-exists": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/pkg-up/node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/pkg-up/node_modules/p-locate": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", - "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/pkg-up/node_modules/path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/pkijs": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/pkijs/-/pkijs-3.4.0.tgz", @@ -18224,16 +17677,6 @@ "version": "16.13.1", "license": "MIT" }, - "node_modules/properties": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/properties/-/properties-1.2.1.tgz", - "integrity": "sha512-qYNxyMj1JeW54i/EWEFsM1cVwxJbtgPp8+0Wg9XjNaK6VE/c4oRi6PNu5p7w1mNXEIQIjV5Wwn8v8Gz82/QzdQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10" - } - }, "node_modules/proxy-addr": { "version": "2.0.7", "dev": true, @@ -19411,14 +18854,6 @@ ], "license": "MIT" }, - "node_modules/safe-json-stringify": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz", - "integrity": "sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg==", - "dev": true, - "license": "MIT", - "optional": true - }, "node_modules/safe-push-apply": { "version": "1.0.0", "dev": true, @@ -20326,16 +19761,6 @@ "dev": true, "license": "MIT" }, - "node_modules/stack-generator": { - "version": "2.0.10", - "resolved": "https://registry.npmjs.org/stack-generator/-/stack-generator-2.0.10.tgz", - "integrity": "sha512-mwnua/hkqM6pF4k8SnmZ2zfETsRUpWXREfA/goT8SLCV4iOFa4bzOX2nDipWAZFPTjLvQB82f5yaodMVhK0yJQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "stackframe": "^1.3.4" - } - }, "node_modules/stack-utils": { "version": "2.0.6", "dev": true, @@ -20355,46 +19780,6 @@ "node": ">=8" } }, - "node_modules/stackframe": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.3.4.tgz", - "integrity": "sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==", - "dev": true, - "license": "MIT" - }, - "node_modules/stacktrace-gps": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/stacktrace-gps/-/stacktrace-gps-3.1.2.tgz", - "integrity": "sha512-GcUgbO4Jsqqg6RxfyTHFiPxdPqF+3LFmQhm7MgCuYQOYuWyqxo5pwRPz5d/u6/WYJdEnWfK4r+jGbyD8TSggXQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "source-map": "0.5.6", - "stackframe": "^1.3.4" - } - }, - "node_modules/stacktrace-gps/node_modules/source-map": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.6.tgz", - "integrity": "sha512-MjZkVp0NHr5+TPihLcadqnlVoGIoWo4IBHptutGh9wI3ttUYvCG26HkSuDi+K6lsZ25syXJXcctwgyVCt//xqA==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/stacktrace-js": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/stacktrace-js/-/stacktrace-js-2.0.2.tgz", - "integrity": "sha512-Je5vBeY4S1r/RnLydLl0TBTi3F2qdfWmYsGvtfZgEI+SCprPppaIhQf5nGcal4gI4cGpCV/duLcAzT1np6sQqg==", - "dev": true, - "license": "MIT", - "dependencies": { - "error-stack-parser": "^2.0.6", - "stack-generator": "^2.0.5", - "stacktrace-gps": "^3.0.4" - } - }, "node_modules/statuses": { "version": "2.0.1", "dev": true, @@ -20415,13 +19800,6 @@ "node": ">= 0.4" } }, - "node_modules/stream-chain": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", - "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==", - "dev": true, - "license": "BSD-3-Clause" - }, "node_modules/stream-composer": { "version": "1.0.2", "dev": true, @@ -20444,16 +19822,6 @@ "dev": true, "license": "MIT" }, - "node_modules/stream-json": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.9.1.tgz", - "integrity": "sha512-uWkjJ+2Nt/LO9Z/JyKZbMusL8Dkh97uUBTv3AJQ74y07lVahLY4eEFsPsE97pxYBwr8nnjMAIch5eqI0gPShyw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "stream-chain": "^2.2.5" - } - }, "node_modules/stream-shift": { "version": "1.0.3", "dev": true, @@ -21629,19 +20997,6 @@ "node": ">=12" } }, - "node_modules/trace-event-lib": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/trace-event-lib/-/trace-event-lib-1.4.1.tgz", - "integrity": "sha512-TOgFolKG8JFY+9d5EohGWMvwvteRafcyfPWWNIqcuD1W/FUvxWcy2MSCZ/beYHM63oYPHYHCd3tkbgCctHVP7w==", - "dev": true, - "license": "MIT", - "dependencies": { - "browser-process-hrtime": "^1.0.0" - }, - "engines": { - "node": ">=12.0.0" - } - }, "node_modules/tree-dump": { "version": "1.0.3", "dev": true, diff --git a/package.json b/package.json index faaf828b39..b672732db3 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,7 @@ "@types/react-test-renderer": "^16.9.12", "@types/react-transition-group": "^4.4.4", "@types/webpack-env": "^1.16.3", + "allure-jest": "^3.9.0", "axe-core": "^4.7.2", "babel-jest": "^29.7.0", "change-case": "^4.1.2", @@ -111,7 +112,6 @@ "html-webpack-plugin": "^5.5.0", "husky": "^9.0.0", "jest": "^29.7.0", - "jest-allure2-reporter": "^2.3.0", "jest-environment-jsdom": "^29.7.0", "lint-staged": "^15.2.2", "loader-utils": "^3.2.1", diff --git a/test/definitions/utils.ts b/test/definitions/utils.ts index 1b4dd0fee2..67e38799c7 100644 --- a/test/definitions/utils.ts +++ b/test/definitions/utils.ts @@ -1,8 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import * as crypto from 'crypto'; -import * as fs from 'fs'; -import * as path from 'path'; +import { attachment, ContentType } from 'allure-js-commons'; import { cropAndCompare } from '@cloudscape-design/browser-test-tools/image-utils'; import { ScreenshotPageObject, ScreenshotWithOffset } from '@cloudscape-design/browser-test-tools/page-objects'; @@ -16,8 +14,6 @@ const defaultWindowSize = { width: 1600, height: 800 }; const newHost = process.env.NEW_HOST || 'http://localhost:8080'; const oldHost = process.env.OLD_HOST || 'http://localhost:8081'; -const allureResultsDir = path.resolve(process.cwd(), 'allure-results'); - function buildUrl(host: string, path: string, queryParams?: Record): string { const params = new URLSearchParams(queryParams); const qs = params.toString(); @@ -29,52 +25,18 @@ function isTestDefinition(item: TestDefinition | TestSuite): item is TestDefinit } /** - * Writes a PNG buffer to the allure-results directory and returns the filename. - * Allure picks up files in this directory and attaches them to the report. + * Attaches visual diff images (new, baseline, diff) to the Allure report + * via the allure-js-commons runtime API. */ -function writeAllureAttachment(buffer: Buffer): string { - if (!fs.existsSync(allureResultsDir)) { - fs.mkdirSync(allureResultsDir, { recursive: true }); - } - const uuid = crypto.randomUUID(); - const filename = `${uuid}-attachment.png`; - fs.writeFileSync(path.join(allureResultsDir, filename), buffer); - return filename; -} - -/** - * Attaches visual diff images (new, baseline, diff) to the Allure results - * for the current test by writing an attachment JSON file. - */ -function attachDiffImages( +async function attachDiffImages( result: { firstImage: Buffer; secondImage: Buffer; diffImage: Buffer | null }, testName: string -): void { - const newFile = writeAllureAttachment(result.firstImage); - const baselineFile = writeAllureAttachment(result.secondImage); - - const attachments: Array<{ name: string; source: string; type: string }> = [ - { name: `${testName} — new (PR)`, source: newFile, type: 'image/png' }, - { name: `${testName} — baseline (main)`, source: baselineFile, type: 'image/png' }, - ]; - +): Promise { + await attachment(`${testName} — new (PR)`, result.firstImage, ContentType.PNG); + await attachment(`${testName} — baseline (main)`, result.secondImage, ContentType.PNG); if (result.diffImage) { - const diffFile = writeAllureAttachment(result.diffImage); - attachments.push({ name: `${testName} — diff`, source: diffFile, type: 'image/png' }); + await attachment(`${testName} — diff`, result.diffImage, ContentType.PNG); } - - // Write a container JSON that Allure merges into the test result. - // jest-allure2-reporter reads attachment files from allure-results/. - const containerUuid = crypto.randomUUID(); - const containerFile = path.join(allureResultsDir, `${containerUuid}-container.json`); - fs.writeFileSync( - containerFile, - JSON.stringify({ - uuid: containerUuid, - name: testName, - attachments, - }) - ); } /** @@ -181,7 +143,7 @@ function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Bro for (let i = 0; i < newPermutations.length; i++) { const permResult = await cropAndCompare(newPermutations[i], oldPermutations[i]); if (permResult.diffPixels !== 0) { - attachDiffImages(permResult, `${testDef.description} [permutation ${i}]`); + await attachDiffImages(permResult, `${testDef.description} [permutation ${i}]`); } expect(permResult.diffPixels).toBe(0); } @@ -189,7 +151,7 @@ function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Bro } // Attach diff images to Allure report for visual inspection. - attachDiffImages(result, testDef.description); + await attachDiffImages(result, testDef.description); // For screenshotArea and viewport types, the diff is a real failure. expect(result.diffPixels).toBe(0); From 37f3e1735ba6ba10424aec4262c4d3a78fca467d Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Sat, 6 Jun 2026 20:07:54 +0200 Subject: [PATCH 80/81] Optimize comparison --- test/definitions/utils.ts | 55 ++++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/test/definitions/utils.ts b/test/definitions/utils.ts index 67e38799c7..1bffb732df 100644 --- a/test/definitions/utils.ts +++ b/test/definitions/utils.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { attachment, ContentType } from 'allure-js-commons'; -import { cropAndCompare } from '@cloudscape-design/browser-test-tools/image-utils'; +import { cropAndCompare, parsePng } from '@cloudscape-design/browser-test-tools/image-utils'; import { ScreenshotPageObject, ScreenshotWithOffset } from '@cloudscape-design/browser-test-tools/page-objects'; import { TestDefinition, TestSuite } from './types'; @@ -14,6 +14,13 @@ const defaultWindowSize = { width: 1600, height: 800 }; const newHost = process.env.NEW_HOST || 'http://localhost:8080'; const oldHost = process.env.OLD_HOST || 'http://localhost:8081'; +interface RawCapture { + /** The raw base64-encoded PNG string from WebDriver (before decoding). */ + rawBase64: string; + /** The fully parsed screenshot with offset metadata (lazily resolved). */ + screenshot: () => Promise; +} + function buildUrl(host: string, path: string, queryParams?: Record): string { const params = new URLSearchParams(queryParams); const qs = params.toString(); @@ -74,15 +81,16 @@ function registerSuites(suites: Array, getBrowser: ( } /** - * Captures a screenshot based on the test's screenshotType. + * Captures a screenshot and returns both the raw PNG base64 and the parsed result. + * Having the raw base64 allows a fast byte-equality check before expensive pixel decoding. */ -async function capture( +async function captureRaw( browser: WebdriverIO.Browser, page: ScreenshotPageObject, url: string, testDef: TestDefinition, windowSize: { width: number; height: number } | undefined -): Promise { +): Promise { if (windowSize) { await browser.setWindowSize(windowSize.width, windowSize.height); } @@ -91,10 +99,29 @@ async function capture( if (testDef.setup) { await testDef.setup(page); } + if (testDef.screenshotType === 'viewport') { - return page.captureViewport(); + const { height, width } = await page.getViewportSize(); + const rawBase64 = await browser.takeScreenshot(); + return { + rawBase64, + screenshot: async () => { + const image = await parsePng(rawBase64); + return { image, offset: { top: 0, left: 0 }, height, width }; + }, + }; } - return page.captureBySelector(screenshotAreaSelector, { viewportOnly: true }); + + // screenshotArea / permutations — capture by selector with viewportOnly + const box = await page.getBoundingBox(screenshotAreaSelector); + const rawBase64 = await browser.takeScreenshot(); + return { + rawBase64, + screenshot: async () => { + const image = await parsePng(rawBase64); + return { image, offset: { top: box.top, left: box.left }, height: box.height, width: box.width }; + }, + }; } function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Browser) { @@ -106,9 +133,19 @@ function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Bro const newUrl = buildUrl(newHost, testDef.path, testDef.queryParams); const oldUrl = buildUrl(oldHost, testDef.path, testDef.queryParams); - const newScreenshot = await capture(browser, page, newUrl, testDef, windowSize); - const oldScreenshot = await capture(browser, page, oldUrl, testDef, windowSize); - const result = await cropAndCompare(newScreenshot, oldScreenshot); + const newCapture = await captureRaw(browser, page, newUrl, testDef, windowSize); + const oldCapture = await captureRaw(browser, page, oldUrl, testDef, windowSize); + + // Fast path: if the raw PNG bytes are identical, the images are guaranteed + // to be the same. This skips the expensive crop + pixelmatch decode path + // for the common case (no visual difference). + if (newCapture.rawBase64 === oldCapture.rawBase64) { + return; + } + + // Raw bytes differ — could be a real diff or just offset/crop differences. + // Fall through to full pixel comparison. + const result = await cropAndCompare(await newCapture.screenshot(), await oldCapture.screenshot()); if (result.diffPixels === 0) { return; From 23bea365d29ff1890a59d12a167c3b5cf91911c5 Mon Sep 17 00:00:00 2001 From: Joan Perals Tresserra Date: Sat, 6 Jun 2026 20:55:25 +0200 Subject: [PATCH 81/81] Export types --- .gitignore | 1 + build-tools/tasks/package-json.js | 4 ++ build-tools/visual/generate-tests.js | 63 +++++++++++++++++++++------- docs/RUNNING_TESTS.md | 10 ++--- test/definitions/index.ts | 49 ---------------------- 5 files changed, 59 insertions(+), 68 deletions(-) delete mode 100644 test/definitions/index.ts diff --git a/.gitignore b/.gitignore index 38d6053611..1f46343d58 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ lib # generated sources src/index.ts test/visual +test/definitions/index.ts allure-results allure-report src/test-utils/dom/index.ts diff --git a/build-tools/tasks/package-json.js b/build-tools/tasks/package-json.js index a7c69a53f5..2c447682a1 100644 --- a/build-tools/tasks/package-json.js +++ b/build-tools/tasks/package-json.js @@ -103,6 +103,10 @@ const devPagesPackageJson = generatePackageJson(path.join(workspace.targetPath, const testDefinitionsPackageJson = generatePackageJson(path.join(workspace.targetPath, 'test-definitions'), { name: '@cloudscape-design/test-definitions', + exports: { + '.': './index.js', + './types': './types.js', + }, }); module.exports = parallel([ diff --git a/build-tools/visual/generate-tests.js b/build-tools/visual/generate-tests.js index a56b996b3d..f82b54d037 100644 --- a/build-tools/visual/generate-tests.js +++ b/build-tools/visual/generate-tests.js @@ -2,11 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 /** - * Auto-generates test/visual/*.test.ts files from test/definitions/visual/*.ts. + * Auto-generates: + * 1. test/visual/*.test.ts — one test runner per definition file (for Jest sharding) + * 2. test/definitions/index.ts — barrel that exports allSuites from all definitions * - * Each definition file that exports a default TestSuite gets a corresponding - * test runner. Helper files (those without a default export, e.g. shared utils - * like app-layout-responsive-tests.ts) are skipped. + * Each definition file that exports a default TestSuite gets included. + * Helper files (those without a default export, e.g. shared utils like + * app-layout-responsive-tests.ts) are skipped. * * Run this script before executing the visual test suite: * node build-tools/visual/generate-tests.js @@ -15,26 +17,29 @@ const fs = require('fs'); const path = require('path'); const definitionsDir = path.resolve(__dirname, '../../test/definitions/visual'); -const outputDir = path.resolve(__dirname, '../../test/visual'); +const testOutputDir = path.resolve(__dirname, '../../test/visual'); +const indexOutputPath = path.resolve(__dirname, '../../test/definitions/index.ts'); // Files that are shared helpers (export named functions, not a default suite). -// These are detected by checking if the file contains "export default" or -// a conventional "const suite" pattern followed by "export default suite". const HELPER_SUFFIXES = ['-tests']; function isHelperFile(basename) { return HELPER_SUFFIXES.some(suffix => basename.endsWith(suffix)); } +function toCamelCase(basename) { + return basename.replace(/-([a-z0-9])/g, (_, char) => char.toUpperCase()); +} + function generate() { // Ensure output directory exists - if (!fs.existsSync(outputDir)) { - fs.mkdirSync(outputDir, { recursive: true }); + if (!fs.existsSync(testOutputDir)) { + fs.mkdirSync(testOutputDir, { recursive: true }); } const files = fs.readdirSync(definitionsDir).filter(f => f.endsWith('.ts') && !f.endsWith('.d.ts')); - const generated = []; + const suiteFiles = []; for (const file of files) { const basename = file.replace(/\.ts$/, ''); @@ -49,6 +54,9 @@ function generate() { continue; } + suiteFiles.push(basename); + + // Generate test/visual/.test.ts const testContent = [ '// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.', '// SPDX-License-Identifier: Apache-2.0', @@ -60,12 +68,39 @@ function generate() { '', ].join('\n'); - const outputPath = path.join(outputDir, `${basename}.test.ts`); - fs.writeFileSync(outputPath, testContent); - generated.push(basename); + fs.writeFileSync(path.join(testOutputDir, `${basename}.test.ts`), testContent); } - console.log(`Generated ${generated.length} visual test files in test/visual/`); + // Generate test/definitions/index.ts + // Sort by module path for consistent import ordering + suiteFiles.sort(); + + const imports = suiteFiles.map(basename => { + const varName = toCamelCase(basename); + return `import ${varName} from './visual/${basename}';`; + }); + + const arrayEntries = suiteFiles.map(basename => ` ${toCamelCase(basename)},`); + + const indexContent = [ + '// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.', + '// SPDX-License-Identifier: Apache-2.0', + '// Auto-generated by build-tools/visual/generate-tests.js — do not edit manually.', + '', + `import { TestSuite } from './types';`, + `export { TestSuite, TestDefinition, ScreenshotType, ScreenshotTestConfiguration } from './types';`, + ...imports, + '', + 'export const allSuites: TestSuite[] = [', + ...arrayEntries, + '];', + '', + ].join('\n'); + + fs.writeFileSync(indexOutputPath, indexContent); + + console.log(`Generated ${suiteFiles.length} visual test files in test/visual/`); + console.log(`Generated test/definitions/index.ts with ${suiteFiles.length} suites`); } generate(); diff --git a/docs/RUNNING_TESTS.md b/docs/RUNNING_TESTS.md index c7832dca72..4db82e361b 100644 --- a/docs/RUNNING_TESTS.md +++ b/docs/RUNNING_TESTS.md @@ -104,14 +104,14 @@ const suite: TestSuite = { export default suite; ``` -Then import and add it to `test/definitions/visual/index.ts`: +Then run the generation script to pick it up automatically: -```ts -import myComponent from './my-component'; - -export const allSuites: TestSuite[] = [..., myComponent]; +```bash +node build-tools/visual/generate-tests.js ``` +This generates both the test runner (`test/visual/my-component.test.ts`) and updates `test/definitions/index.ts`. No manual imports needed. + ### Reviewing failures If the CI job fails, download the `visual-regression-diffs` artifact from the Actions summary. diff --git a/test/definitions/index.ts b/test/definitions/index.ts deleted file mode 100644 index 3bd6fa66e2..0000000000 --- a/test/definitions/index.ts +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Each component has its own test definition file. -// Import them here manually to form the full test suite. -import { TestSuite } from './types'; -import actionCard from './visual/action-card'; -import alert from './visual/alert'; -import appLayout from './visual/app-layout'; -import appLayoutContentPaddings from './visual/app-layout-content-paddings'; -import appLayoutDrawers from './visual/app-layout-drawers'; -import appLayoutFlashbar from './visual/app-layout-flashbar'; -import appLayoutHeader from './visual/app-layout-header'; -import appLayoutMulti from './visual/app-layout-multi'; -import appLayoutResponsive600 from './visual/app-layout-responsive-600'; -import appLayoutResponsive1280 from './visual/app-layout-responsive-1280'; -import appLayoutResponsive1400 from './visual/app-layout-responsive-1400'; -import appLayoutResponsive1920 from './visual/app-layout-responsive-1920'; -import appLayoutResponsive2540 from './visual/app-layout-responsive-2540'; -import appLayoutStickyTableHeaderSplitPanel from './visual/app-layout-sticky-table-header-split-panel'; -import appLayoutToolbar from './visual/app-layout-toolbar'; -import appLayoutZIndex from './visual/app-layout-z-index'; -import areaChart from './visual/area-chart'; -import attributeEditor from './visual/attribute-editor'; -import autosuggest from './visual/autosuggest'; -import badge from './visual/badge'; - -export const allSuites: TestSuite[] = [ - actionCard, - alert, - appLayout, - appLayoutContentPaddings, - appLayoutDrawers, - appLayoutFlashbar, - appLayoutHeader, - appLayoutMulti, - appLayoutResponsive600, - appLayoutResponsive1280, - appLayoutResponsive1400, - appLayoutResponsive1920, - appLayoutResponsive2540, - appLayoutStickyTableHeaderSplitPanel, - appLayoutToolbar, - appLayoutZIndex, - areaChart, - attributeEditor, - autosuggest, - badge, -];