diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5593984949..36964ddc1f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,6 +53,13 @@ jobs: name: dev-pages-react${{ matrix.react }} path: pages/lib/static-default + - name: Upload test utils selectors artifact + if: matrix.react == 18 + uses: actions/upload-artifact@v4 + with: + name: test-utils-selectors + path: lib/components + deploy: needs: quick-build name: deploy${{ matrix.react != 16 && format(' (React {0})', matrix.react) || '' }} @@ -65,3 +72,14 @@ jobs: with: artifact-name: dev-pages-react${{ matrix.react }} deployment-path: pages/lib/static-default + + visual: + name: Visual regression + needs: quick-build + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + uses: ./.github/workflows/visual-regression.yml + secrets: inherit + with: + pr-artifact-name: dev-pages-react18 + test-utils-artifact-name: test-utils-selectors + caller-run-id: ${{ github.run_id }} diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml new file mode 100644 index 0000000000..977c6cd96d --- /dev/null +++ b/.github/workflows/visual-regression.yml @@ -0,0 +1,197 @@ +name: Visual Regression Tests + +on: + workflow_call: + inputs: + pr-artifact-name: + description: 'Name of the artifact containing PR pages (built by the caller workflow).' + required: true + type: string + test-utils-artifact-name: + description: 'Name of the artifact containing test-utils selectors.' + required: true + type: string + caller-run-id: + description: 'The run ID of the calling workflow, used to download artifacts it uploaded.' + required: true + type: string + +defaults: + run: + shell: bash + +permissions: + id-token: write + contents: read + actions: read + deployments: write + +jobs: + # Build the baseline (main branch) pages once and share them across all browser jobs. + build-baseline: + name: Build baseline pages + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Install dependencies + run: npm i + + # Use a git worktree so the baseline has its own directory and its own + # node_modules. This means a PR that changes package-lock.json will still + # produce a correct baseline: the baseline installs from main's lockfile + # and the PR build installs from the PR's lockfile, so both sides use the + # dependency versions that are correct for their respective source trees. + - name: Create baseline worktree from origin/main + run: git worktree add /tmp/baseline origin/main + + - name: Install baseline dependencies + run: npm i + working-directory: /tmp/baseline + + - name: Build baseline pages + run: npx gulp quick-build + working-directory: /tmp/baseline + env: + NODE_ENV: production + + - name: Bundle baseline pages + run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path ${{ github.workspace }}/pages/lib/static-visual-baseline + working-directory: /tmp/baseline + env: + NODE_ENV: production + + - name: Upload baseline artifact + uses: actions/upload-artifact@v4 + with: + name: visual-baseline-pages + path: pages/lib/static-visual-baseline + retention-days: 1 + + visual: + name: Visual regression (shard ${{ matrix.shard }}) + needs: [build-baseline] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Setup Chrome and ChromeDriver + uses: browser-actions/setup-chrome@v1 + with: + chrome-version: stable + + - name: Install dependencies + run: npm i + + - name: Download PR pages artifact + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.pr-artifact-name }} + path: pages/lib/static-default + github-token: ${{ github.token }} + run-id: ${{ inputs.caller-run-id }} + + - name: Download baseline artifact + uses: actions/download-artifact@v4 + with: + name: visual-baseline-pages + path: pages/lib/static-visual-baseline + + - name: Download test utils artifact + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.test-utils-artifact-name }} + path: lib/components + github-token: ${{ github.token }} + run-id: ${{ inputs.caller-run-id }} + + # ── Run tests ───────────────────────────────────────────────────────── + + - name: Generate visual test files + run: node build-tools/visual/generate-tests.js + + - name: Start test server (port 8080) + run: npx --yes serve --no-clipboard --listen 8080 pages/lib/static-default & + + - name: Start baseline server (port 8081) + run: npx --yes serve --no-clipboard --listen 8081 pages/lib/static-visual-baseline & + + - name: Wait for servers to be ready + run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081 + + - name: Run visual regression tests + run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js --shard=${{ matrix.shard }}/10 + env: + TZ: UTC + + - name: Upload diff artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: visual-regression-diffs-shard-${{ matrix.shard }} + path: visual-regression-output/ + retention-days: 14 + + - name: Upload Allure results + if: always() + uses: actions/upload-artifact@v4 + with: + name: allure-results-shard-${{ matrix.shard }} + path: allure-results/ + retention-days: 3 + + report: + name: Generate Allure Report + if: always() + needs: [visual] + runs-on: ubuntu-latest + steps: + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Download all Allure results + uses: actions/download-artifact@v4 + with: + pattern: allure-results-shard-* + path: allure-results + merge-multiple: true + + - name: Generate Allure HTML report + run: npx --yes allure generate allure-results -o allure-report + + - name: Upload Allure report artifact + uses: actions/upload-artifact@v4 + with: + name: allure-report + path: allure-report/ + retention-days: 14 + + deploy-report: + name: Deploy Allure Report + if: always() + needs: [report] + uses: cloudscape-design/actions/.github/workflows/deploy.yml@main + secrets: inherit + with: + artifact-name: allure-report + deployment-path: allure-report diff --git a/.gitignore b/.gitignore index db3d62b944..1f46343d58 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,10 @@ coverage lib # generated sources src/index.ts +test/visual +test/definitions/index.ts +allure-results +allure-report src/test-utils/dom/index.ts src/test-utils/selectors src/icon/generated diff --git a/build-tools/tasks/package-json.js b/build-tools/tasks/package-json.js index a7c69a53f5..2c447682a1 100644 --- a/build-tools/tasks/package-json.js +++ b/build-tools/tasks/package-json.js @@ -103,6 +103,10 @@ const devPagesPackageJson = generatePackageJson(path.join(workspace.targetPath, const testDefinitionsPackageJson = generatePackageJson(path.join(workspace.targetPath, 'test-definitions'), { name: '@cloudscape-design/test-definitions', + exports: { + '.': './index.js', + './types': './types.js', + }, }); module.exports = parallel([ diff --git a/build-tools/visual/generate-tests.js b/build-tools/visual/generate-tests.js new file mode 100644 index 0000000000..f82b54d037 --- /dev/null +++ b/build-tools/visual/generate-tests.js @@ -0,0 +1,106 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Auto-generates: + * 1. test/visual/*.test.ts — one test runner per definition file (for Jest sharding) + * 2. test/definitions/index.ts — barrel that exports allSuites from all definitions + * + * Each definition file that exports a default TestSuite gets included. + * Helper files (those without a default export, e.g. shared utils like + * app-layout-responsive-tests.ts) are skipped. + * + * Run this script before executing the visual test suite: + * node build-tools/visual/generate-tests.js + */ +const fs = require('fs'); +const path = require('path'); + +const definitionsDir = path.resolve(__dirname, '../../test/definitions/visual'); +const testOutputDir = path.resolve(__dirname, '../../test/visual'); +const indexOutputPath = path.resolve(__dirname, '../../test/definitions/index.ts'); + +// Files that are shared helpers (export named functions, not a default suite). +const HELPER_SUFFIXES = ['-tests']; + +function isHelperFile(basename) { + return HELPER_SUFFIXES.some(suffix => basename.endsWith(suffix)); +} + +function toCamelCase(basename) { + return basename.replace(/-([a-z0-9])/g, (_, char) => char.toUpperCase()); +} + +function generate() { + // Ensure output directory exists + if (!fs.existsSync(testOutputDir)) { + fs.mkdirSync(testOutputDir, { recursive: true }); + } + + const files = fs.readdirSync(definitionsDir).filter(f => f.endsWith('.ts') && !f.endsWith('.d.ts')); + + const suiteFiles = []; + + for (const file of files) { + const basename = file.replace(/\.ts$/, ''); + + if (isHelperFile(basename)) { + continue; + } + + // Verify the file has a default export by scanning for the pattern + const content = fs.readFileSync(path.join(definitionsDir, file), 'utf-8'); + if (!content.includes('export default')) { + continue; + } + + suiteFiles.push(basename); + + // Generate test/visual/.test.ts + const testContent = [ + '// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.', + '// SPDX-License-Identifier: Apache-2.0', + '// Auto-generated by build-tools/visual/generate-tests.js — do not edit manually.', + `import { runTestSuites } from '../definitions/utils';`, + `import suite from '../definitions/visual/${basename}';`, + '', + 'runTestSuites([suite]);', + '', + ].join('\n'); + + fs.writeFileSync(path.join(testOutputDir, `${basename}.test.ts`), testContent); + } + + // Generate test/definitions/index.ts + // Sort by module path for consistent import ordering + suiteFiles.sort(); + + const imports = suiteFiles.map(basename => { + const varName = toCamelCase(basename); + return `import ${varName} from './visual/${basename}';`; + }); + + const arrayEntries = suiteFiles.map(basename => ` ${toCamelCase(basename)},`); + + const indexContent = [ + '// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.', + '// SPDX-License-Identifier: Apache-2.0', + '// Auto-generated by build-tools/visual/generate-tests.js — do not edit manually.', + '', + `import { TestSuite } from './types';`, + `export { TestSuite, TestDefinition, ScreenshotType, ScreenshotTestConfiguration } from './types';`, + ...imports, + '', + 'export const allSuites: TestSuite[] = [', + ...arrayEntries, + '];', + '', + ].join('\n'); + + fs.writeFileSync(indexOutputPath, indexContent); + + console.log(`Generated ${suiteFiles.length} visual test files in test/visual/`); + console.log(`Generated test/definitions/index.ts with ${suiteFiles.length} suites`); +} + +generate(); diff --git a/build-tools/visual/global-setup.js b/build-tools/visual/global-setup.js new file mode 100644 index 0000000000..52ce2f271c --- /dev/null +++ b/build-tools/visual/global-setup.js @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +module.exports = async () => { + const { startWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); + await startWebdriver(); +}; diff --git a/build-tools/visual/global-teardown.js b/build-tools/visual/global-teardown.js new file mode 100644 index 0000000000..0fa05eebfe --- /dev/null +++ b/build-tools/visual/global-teardown.js @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +module.exports = () => { + const { shutdownWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); + shutdownWebdriver(); +}; diff --git a/build-tools/visual/setup.js b/build-tools/visual/setup.js new file mode 100644 index 0000000000..d52cd606fb --- /dev/null +++ b/build-tools/visual/setup.js @@ -0,0 +1,16 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +/* global jest */ +const { configure } = require('@cloudscape-design/browser-test-tools/use-browser'); + +configure({ + browserName: 'ChromeHeadlessIntegration', + browserCreatorOptions: { + seleniumUrl: 'http://localhost:9515', + }, + webdriverOptions: { + baseUrl: 'http://localhost:8080', + }, +}); + +jest.retryTimes(2, { logErrorsBeforeRetry: true }); diff --git a/docs/RUNNING_TESTS.md b/docs/RUNNING_TESTS.md index 525cdf181d..4db82e361b 100644 --- a/docs/RUNNING_TESTS.md +++ b/docs/RUNNING_TESTS.md @@ -60,11 +60,60 @@ TZ=UTC npx jest -u -c jest.unit.config.js src/ ``` ## Visual Regression Tests -> **Note:** The components repository does not have visual regression tests on GitHub. This section applies to other repositories such as chat-components, code-view, chart-components, and board-components. +Visual regression tests run automatically when opening a pull request in GitHub (see `.github/workflows/visual-regression.yml`). -Visual regression tests for permutation pages run automatically when opening a pull request in GitHub. +They compare permutation pages between the PR build and a baseline build of `main`, both served locally in the same CI job. Each side installs from its own `package-lock.json` via a git worktree, so dependency changes in the PR are handled correctly and unpinned updates in sister repositories affect both sides equally. -To check results: look at the "Visual Regression Tests" action in the PR. The "Test for regressions" step logs which pages failed. For a full report, download the `visual-regression-snapshots-results` artifact from the action summary. +### How it works -If there are unexpected regressions, fix your pull request. -If the changes are expected, call this out in your pull request comments. +1. The PR pages are built and served on port 8080. +2. A git worktree of `origin/main` is created, its dependencies installed, and its pages built and served on port 8081. +3. The single test runner (`test/visual.test.ts`) iterates over all test definitions, captures the `.screenshot-area` element from both servers for each test, and fails if any pixels differ. + +### Running locally + +``` +npm run test:visual +``` + +This handles the full build and comparison in one command. If both outputs are already built, skip the build step: + +``` +NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js +``` + +(Requires both servers to be running — start the PR build with `npm run start:integ` on port 8080 and the baseline build on port 8081, or set `NEW_HOST` / `OLD_HOST` env vars to point at different hosts.) + +### Adding tests for a new component + +Create `test/definitions/visual/.ts`: + +```ts +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'my-component', + tests: [ + { + description: 'permutations', + path: 'my-component/permutations', + }, + ], +}; + +export default suite; +``` + +Then run the generation script to pick it up automatically: + +```bash +node build-tools/visual/generate-tests.js +``` + +This generates both the test runner (`test/visual/my-component.test.ts`) and updates `test/definitions/index.ts`. No manual imports needed. + +### Reviewing failures + +If the CI job fails, download the `visual-regression-diffs` artifact from the Actions summary. + +If the diff is expected (intentional visual change), note it in your PR description. diff --git a/eslint.config.mjs b/eslint.config.mjs index 0d9423aa5b..b92a169696 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -225,7 +225,7 @@ export default tsEslint.config( }, }, { - files: ['**/__integ__/**', '**/__motion__/**', '**/__a11y__/**'], + files: ['**/__integ__/**', '**/__motion__/**', '**/__a11y__/**', 'test/definitions/**'], rules: { // useBrowser is not a hook 'react-hooks/rules-of-hooks': 'off', diff --git a/jest.visual.config.js b/jest.visual.config.js new file mode 100644 index 0000000000..98938768b3 --- /dev/null +++ b/jest.visual.config.js @@ -0,0 +1,28 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +const path = require('path'); +const os = require('os'); + +module.exports = { + verbose: true, + testEnvironment: 'allure-jest/node', + testEnvironmentOptions: { + resultsDir: 'allure-results', + }, + transform: { + '^.+\\.tsx?$': [ + 'ts-jest', + { + tsconfig: 'tsconfig.visual.json', + }, + ], + }, + reporters: ['default', 'github-actions'], + testTimeout: 240_000, // 4min — pages can be tall and slow to capture + maxWorkers: os.cpus().length * (process.env.GITHUB_ACTION ? 3 : 1), + globalSetup: '/build-tools/visual/global-setup.js', + globalTeardown: '/build-tools/visual/global-teardown.js', + setupFilesAfterEnv: [path.join(__dirname, 'build-tools', 'visual', 'setup.js')], + moduleFileExtensions: ['js', 'ts'], + testMatch: ['/test/visual/**/*.test.ts'], +}; diff --git a/package-lock.json b/package-lock.json index 1c7a32ecfd..22ac706458 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,6 +52,7 @@ "@types/jest": "^29.5.13", "@types/lodash": "^4.14.176", "@types/node": "^20.17.14", + "@types/pixelmatch": "^5.2.6", "@types/react": "^16.14.20", "@types/react-dom": "^16.9.14", "@types/react-is": "^18.2.0", @@ -60,6 +61,7 @@ "@types/react-test-renderer": "^16.9.12", "@types/react-transition-group": "^4.4.4", "@types/webpack-env": "^1.16.3", + "allure-jest": "^3.9.0", "axe-core": "^4.7.2", "babel-jest": "^29.7.0", "change-case": "^4.1.2", @@ -96,6 +98,7 @@ "mockdate": "^3.0.5", "npm-run-all": "^4.1.5", "prettier": "^3.6.1", + "puppeteer-core": "^24.43.1", "react": "^16.14.0", "react-dom": "^16.14.0", "react-dom18": "npm:react-dom@^18.3.1", @@ -3521,9 +3524,9 @@ } }, "node_modules/@puppeteer/browsers": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.13.0.tgz", - "integrity": "sha512-46BZJYJjc/WwmKjsvDFykHtXrtomsCIrwYQPOP7VfMJoZY2bsDF9oROBABR3paDjDcmkUye1Pb1BqdcdiipaWA==", + "version": "2.13.2", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.13.2.tgz", + "integrity": "sha512-5EUZSUIc37H6aIXyWO0Z4y8NlF8NnjgmqeQgOGiswAU7pY0HOo16ho4+alIWmSfdZnjqBRawMsP3I5YqLSn6kw==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -4846,6 +4849,16 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/pixelmatch": { + "version": "5.2.6", + "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.6.tgz", + "integrity": "sha512-wC83uexE5KGuUODn6zkm9gMzTwdY5L0chiK+VrKcDfEjzxh1uadlWTvOmAbCpnM9zx/Ww3f8uKlYQVnO/TrqVg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/pngjs": { "version": "6.0.5", "dev": true, @@ -5892,6 +5905,58 @@ "dev": true, "license": "MIT" }, + "node_modules/allure-jest": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/allure-jest/-/allure-jest-3.9.0.tgz", + "integrity": "sha512-hEW4DKjvb3engGoHUPQaDEdyrFkUxQnqULiSQAehL1eDEggqdPbQro86Nch8Cj1yuIqUTn9UP1FMuuuwl/5jnQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "allure-js-commons": "3.9.0" + }, + "peerDependencies": { + "jest": ">=24.8.0", + "jest-circus": ">=24.8.0", + "jest-cli": ">=24.8.0", + "jest-environment-jsdom": ">=24.8.0", + "jest-environment-node": ">=24.8.0" + }, + "peerDependenciesMeta": { + "jest": { + "optional": true + }, + "jest-circus": { + "optional": true + }, + "jest-cli": { + "optional": true + }, + "jest-environment-jsdom": { + "optional": true + }, + "jest-environment-node": { + "optional": true + } + } + }, + "node_modules/allure-js-commons": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/allure-js-commons/-/allure-js-commons-3.9.0.tgz", + "integrity": "sha512-uVQcGE6MWIvGR/zW1XEUwHXUQa1EJKY0Cah+0TZK1qKuw6ptyhftDr34XE3wExTyCZirRrI98dbRtPeYYuyI+g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "md5": "^2.3.0" + }, + "peerDependencies": { + "allure-playwright": "3.9.0" + }, + "peerDependenciesMeta": { + "allure-playwright": { + "optional": true + } + } + }, "node_modules/ansi-escapes": { "version": "4.3.2", "dev": true, @@ -7154,6 +7219,16 @@ "node": ">=10" } }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, "node_modules/cheerio": { "version": "1.1.0", "dev": true, @@ -7262,6 +7337,20 @@ "node": ">=6.0" } }, + "node_modules/chromium-bidi": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-14.0.0.tgz", + "integrity": "sha512-9gYlLtS6tStdRWzrtXaTMnqcM4dudNegMXJxkR0I/CXObHalYeYcAMPrL19eroNZHtJ8DQmu1E+ZNOYu/IXMXw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/ci-info": { "version": "3.9.0", "dev": true, @@ -7848,6 +7937,16 @@ "node": ">= 8" } }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, "node_modules/css-declaration-sorter": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/css-declaration-sorter/-/css-declaration-sorter-7.3.1.tgz", @@ -8778,6 +8877,13 @@ "dev": true, "license": "MIT" }, + "node_modules/devtools-protocol": { + "version": "0.0.1608973", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1608973.tgz", + "integrity": "sha512-Tpm17fxYzt+J7VrGdc1k8YdRqS3YV7se/M6KeemEqvUbq/n7At1rWVuXMxQgpWkdwSdIEKYbU//Bve+Shm4YNQ==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/diff-sequences": { "version": "29.6.3", "dev": true, @@ -12559,6 +12665,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dev": true, + "license": "MIT" + }, "node_modules/is-builtin-module": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-5.0.0.tgz", @@ -15238,6 +15351,18 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, "node_modules/mdn-data": { "version": "2.12.2", "dev": true, @@ -17735,6 +17860,25 @@ "node": ">=6" } }, + "node_modules/puppeteer-core": { + "version": "24.43.1", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.43.1.tgz", + "integrity": "sha512-T5ScUMAsmhdNbgDR41AGESYeS6V9MSgetkSnVhhW+gXvzC42VesKCn5ld87gAZDJ6vLHL9GkRvY9WtQWSnwFbw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.13.2", + "chromium-bidi": "14.0.0", + "debug": "^4.4.3", + "devtools-protocol": "0.0.1608973", + "typed-query-selector": "^2.12.2", + "webdriver-bidi-protocol": "0.4.1", + "ws": "^8.20.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/pure-rand": { "version": "6.1.0", "dev": true, @@ -21147,6 +21291,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typed-query-selector": { + "version": "2.12.2", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.2.tgz", + "integrity": "sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ==", + "dev": true, + "license": "MIT" + }, "node_modules/typedarray": { "version": "0.0.6", "dev": true, @@ -21664,6 +21815,13 @@ "node": ">=18.20.0" } }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz", + "integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/webdriver/node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -22441,6 +22599,16 @@ "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index fda995324a..b672732db3 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,7 @@ "@types/jest": "^29.5.13", "@types/lodash": "^4.14.176", "@types/node": "^20.17.14", + "@types/pixelmatch": "^5.2.6", "@types/react": "^16.14.20", "@types/react-dom": "^16.9.14", "@types/react-is": "^18.2.0", @@ -83,6 +84,7 @@ "@types/react-test-renderer": "^16.9.12", "@types/react-transition-group": "^4.4.4", "@types/webpack-env": "^1.16.3", + "allure-jest": "^3.9.0", "axe-core": "^4.7.2", "babel-jest": "^29.7.0", "change-case": "^4.1.2", @@ -119,6 +121,7 @@ "mockdate": "^3.0.5", "npm-run-all": "^4.1.5", "prettier": "^3.6.1", + "puppeteer-core": "^24.43.1", "react": "^16.14.0", "react-dom": "^16.14.0", "react-dom18": "npm:react-dom@^18.3.1", diff --git a/test/definitions/index.ts b/test/definitions/index.ts deleted file mode 100644 index 91e90a89f7..0000000000 --- a/test/definitions/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Each component has its own test definition file. -// Import them here manually to form the full test suite. -import { TestSuite } from './types'; -import actionCard from './visual/action-card'; -import alert from './visual/alert'; - -export const allSuites: TestSuite[] = [actionCard, alert]; diff --git a/test/definitions/types.ts b/test/definitions/types.ts index 8cdca9996a..9f8a187c48 100644 --- a/test/definitions/types.ts +++ b/test/definitions/types.ts @@ -9,7 +9,7 @@ export interface ScreenshotTestConfiguration { // 'screenshotArea' — captures the .screenshot-area element on the page. // 'permutations' — captures the entire page and crops permutations out of it. -export type ScreenshotType = 'screenshotArea' | 'permutations'; +export type ScreenshotType = 'screenshotArea' | 'permutations' | 'viewport'; export interface TestDefinition { description: string; diff --git a/test/definitions/utils.ts b/test/definitions/utils.ts new file mode 100644 index 0000000000..1bffb732df --- /dev/null +++ b/test/definitions/utils.ts @@ -0,0 +1,196 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { attachment, ContentType } from 'allure-js-commons'; + +import { cropAndCompare, parsePng } from '@cloudscape-design/browser-test-tools/image-utils'; +import { ScreenshotPageObject, ScreenshotWithOffset } from '@cloudscape-design/browser-test-tools/page-objects'; + +import { TestDefinition, TestSuite } from './types'; + +const screenshotAreaSelector = '.screenshot-area'; +const defaultWindowSize = { width: 1600, height: 800 }; + +// NEW_HOST serves the PR's pages, OLD_HOST serves the baseline (main) pages. +const newHost = process.env.NEW_HOST || 'http://localhost:8080'; +const oldHost = process.env.OLD_HOST || 'http://localhost:8081'; + +interface RawCapture { + /** The raw base64-encoded PNG string from WebDriver (before decoding). */ + rawBase64: string; + /** The fully parsed screenshot with offset metadata (lazily resolved). */ + screenshot: () => Promise; +} + +function buildUrl(host: string, path: string, queryParams?: Record): string { + const params = new URLSearchParams(queryParams); + const qs = params.toString(); + return `${host}/#/${path}${qs ? `?${qs}` : ''}`; +} + +function isTestDefinition(item: TestDefinition | TestSuite): item is TestDefinition { + return (item as TestDefinition).path !== undefined; +} + +/** + * Attaches visual diff images (new, baseline, diff) to the Allure report + * via the allure-js-commons runtime API. + */ +async function attachDiffImages( + result: { firstImage: Buffer; secondImage: Buffer; diffImage: Buffer | null }, + testName: string +): Promise { + await attachment(`${testName} — new (PR)`, result.firstImage, ContentType.PNG); + await attachment(`${testName} — baseline (main)`, result.secondImage, ContentType.PNG); + if (result.diffImage) { + await attachment(`${testName} — diff`, result.diffImage, ContentType.PNG); + } +} + +/** + * Registers all test suites with a single shared browser session per worker. + * This avoids the per-test session creation overhead. + */ +export function runTestSuites(suites: Array) { + let browser: WebdriverIO.Browser; + + beforeAll(async () => { + const { default: getBrowserCreator } = await import('@cloudscape-design/browser-test-tools/browser'); + const creator = getBrowserCreator('ChromeHeadlessIntegration', 'local', { + seleniumUrl: 'http://localhost:9515', + }); + browser = await creator.getBrowser({ width: defaultWindowSize.width, height: defaultWindowSize.height }); + }); + + afterAll(async () => { + await browser?.deleteSession(); + }); + + registerSuites(suites, () => browser); +} + +function registerSuites(suites: Array, getBrowser: () => WebdriverIO.Browser) { + for (const item of suites) { + if (isTestDefinition(item)) { + registerTest(item, getBrowser); + } else { + describe(item.description, () => { + registerSuites(item.tests, getBrowser); + }); + } + } +} + +/** + * Captures a screenshot and returns both the raw PNG base64 and the parsed result. + * Having the raw base64 allows a fast byte-equality check before expensive pixel decoding. + */ +async function captureRaw( + browser: WebdriverIO.Browser, + page: ScreenshotPageObject, + url: string, + testDef: TestDefinition, + windowSize: { width: number; height: number } | undefined +): Promise { + if (windowSize) { + await browser.setWindowSize(windowSize.width, windowSize.height); + } + await browser.url(url); + await page.waitForVisible(screenshotAreaSelector); + if (testDef.setup) { + await testDef.setup(page); + } + + if (testDef.screenshotType === 'viewport') { + const { height, width } = await page.getViewportSize(); + const rawBase64 = await browser.takeScreenshot(); + return { + rawBase64, + screenshot: async () => { + const image = await parsePng(rawBase64); + return { image, offset: { top: 0, left: 0 }, height, width }; + }, + }; + } + + // screenshotArea / permutations — capture by selector with viewportOnly + const box = await page.getBoundingBox(screenshotAreaSelector); + const rawBase64 = await browser.takeScreenshot(); + return { + rawBase64, + screenshot: async () => { + const image = await parsePng(rawBase64); + return { image, offset: { top: box.top, left: box.left }, height: box.height, width: box.width }; + }, + }; +} + +function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Browser) { + test(testDef.description, async () => { + const browser = getBrowser(); + const windowSize = testDef.configuration ? { ...defaultWindowSize, ...testDef.configuration } : undefined; + const page = new ScreenshotPageObject(browser); + + const newUrl = buildUrl(newHost, testDef.path, testDef.queryParams); + const oldUrl = buildUrl(oldHost, testDef.path, testDef.queryParams); + + const newCapture = await captureRaw(browser, page, newUrl, testDef, windowSize); + const oldCapture = await captureRaw(browser, page, oldUrl, testDef, windowSize); + + // Fast path: if the raw PNG bytes are identical, the images are guaranteed + // to be the same. This skips the expensive crop + pixelmatch decode path + // for the common case (no visual difference). + if (newCapture.rawBase64 === oldCapture.rawBase64) { + return; + } + + // Raw bytes differ — could be a real diff or just offset/crop differences. + // Fall through to full pixel comparison. + const result = await cropAndCompare(await newCapture.screenshot(), await oldCapture.screenshot()); + + if (result.diffPixels === 0) { + return; + } + + // For permutations pages, a screenshot-area diff might be a false positive + // caused by content extending beyond the viewport. Re-capture using the + // full capturePermutations strategy which resizes the window to fit all + // content and returns individual permutation crops for precise comparison. + if (testDef.screenshotType === 'permutations') { + if (windowSize) { + await browser.setWindowSize(windowSize.width, windowSize.height); + } + await browser.url(newUrl); + await page.waitForVisible(screenshotAreaSelector); + if (testDef.setup) { + await testDef.setup(page); + } + const newPermutations = await page.capturePermutations(); + + if (windowSize) { + await browser.setWindowSize(windowSize.width, windowSize.height); + } + await browser.url(oldUrl); + await page.waitForVisible(screenshotAreaSelector); + if (testDef.setup) { + await testDef.setup(page); + } + const oldPermutations = await page.capturePermutations(); + + expect(newPermutations.length).toBe(oldPermutations.length); + for (let i = 0; i < newPermutations.length; i++) { + const permResult = await cropAndCompare(newPermutations[i], oldPermutations[i]); + if (permResult.diffPixels !== 0) { + await attachDiffImages(permResult, `${testDef.description} [permutation ${i}]`); + } + expect(permResult.diffPixels).toBe(0); + } + return; + } + + // Attach diff images to Allure report for visual inspection. + await attachDiffImages(result, testDef.description); + + // For screenshotArea and viewport types, the diff is a real failure. + expect(result.diffPixels).toBe(0); + }); +} diff --git a/test/definitions/visual/app-layout-content-paddings.ts b/test/definitions/visual/app-layout-content-paddings.ts new file mode 100644 index 0000000000..e76d055749 --- /dev/null +++ b/test/definitions/visual/app-layout-content-paddings.ts @@ -0,0 +1,30 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Content paddings', + componentName: 'app-layout', + tests: [ + ...(['true', 'false'] as const).flatMap(toolsEnabled => + (['true', 'false'] as const).flatMap(splitPanelEnabled => + (['bottom', 'side'] as const).map(splitPanelPosition => ({ + description: `toolsEnabled=${toolsEnabled} splitPanelEnabled=${splitPanelEnabled} splitPanelPosition=${splitPanelPosition}`, + path: 'app-layout/with-split-panel', + screenshotType: 'viewport' as const, + queryParams: { toolsEnabled, splitPanelEnabled, splitPanelPosition }, + })) + ) + ), + ...[1500, 600].map(width => ({ + description: `with split panel and disabled content paddings - width=${width}`, + path: 'app-layout/disable-paddings-with-split-panel', + screenshotType: 'viewport' as const, + configuration: { width }, + queryParams: { splitPanelOpen: 'true', splitPanelPosition: 'side' }, + })), + ], +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-drawers.ts b/test/definitions/visual/app-layout-drawers.ts new file mode 100644 index 0000000000..39c2089783 --- /dev/null +++ b/test/definitions/visual/app-layout-drawers.ts @@ -0,0 +1,41 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import createWrapper from '../../../lib/components/test-utils/selectors'; +import { TestSuite } from '../types'; + +const wrapper = createWrapper(); + +const suite: TestSuite = { + description: 'Drawers', + componentName: 'app-layout', + tests: [ + { + description: 'with split panel', + path: 'app-layout/with-drawers', + screenshotType: 'viewport', + setup: async page => { + await page.click(wrapper.findAppLayout().findDrawerTriggerById('pro-help').toSelector()); + }, + }, + { + description: 'with tooltip on hover', + path: 'app-layout/with-drawers', + screenshotType: 'viewport', + setup: async page => { + await page.hoverElement(wrapper.findAppLayout().findDrawerTriggerById('pro-help').toSelector()); + }, + }, + { + description: 'with custom scrollable drawer content', + path: 'app-layout/with-drawers-scrollable', + screenshotType: 'viewport', + queryParams: { sideNavFill: 'false' }, + setup: async page => { + await page.click(wrapper.findAppLayout().findDrawerTriggerById('chat').toSelector()); + }, + }, + ], +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-flashbar.ts b/test/definitions/visual/app-layout-flashbar.ts new file mode 100644 index 0000000000..5849a5cbf5 --- /dev/null +++ b/test/definitions/visual/app-layout-flashbar.ts @@ -0,0 +1,39 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Flashbar', + componentName: 'app-layout', + tests: [true, false].flatMap(disableContentPaddings => + [true, false].flatMap(stickyNotifications => + [true, false].flatMap(stickyTableHeader => + [true, false].map(stackNotifications => ({ + description: `disableContentPaddings: ${disableContentPaddings}, stickyNotifications: ${stickyNotifications}, stickyTableHeader: ${stickyTableHeader}, stackNotifications: ${stackNotifications}`, + path: 'app-layout/with-stacked-notifications-and-table', + screenshotType: 'screenshotArea' as const, + configuration: { width: 1280, height: 900 }, + setup: async (page: import('@cloudscape-design/browser-test-tools/page-objects').ScreenshotPageObject) => { + if (!disableContentPaddings) { + await page.click('[data-id="toggle-content-paddings"]'); + } + if (stickyNotifications) { + await page.click('[data-id="toggle-sticky-notifications"]'); + } + if (!stickyTableHeader) { + await page.click('[data-id="toggle-sticky-table-header"]'); + } + if (!stackNotifications) { + await page.click('[data-id="toggle-stack-items"]'); + } + await page.click('[data-id="add-notification"]'); + await page.click('[data-id="add-notification"]'); + }, + })) + ) + ) + ), +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-header.ts b/test/definitions/visual/app-layout-header.ts new file mode 100644 index 0000000000..3a128a266e --- /dev/null +++ b/test/definitions/visual/app-layout-header.ts @@ -0,0 +1,85 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestDefinition, TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Headers', + componentName: 'app-layout', + tests: [ + // ── Headers ─────────────────────────────────────────────────────────── + { + description: 'Headers', + tests: [600, 1280].flatMap(width => [ + { + description: `alignment with full-page table (${width}px)`, + path: 'app-layout/with-table', + screenshotType: 'viewport' as const, + configuration: { width }, + }, + { + description: `alignment with full-page table in sticky state (${width}px)`, + path: 'app-layout/with-table', + screenshotType: 'viewport' as const, + configuration: { width }, + setup: async page => { + await page.windowScrollTo({ top: 200 }); + }, + }, + { + description: `alignment with full-page table in sticky state with sticky notifications (${width}px)`, + path: 'app-layout/with-table', + screenshotType: 'viewport' as const, + configuration: { width }, + queryParams: { stickyNotifications: 'true' }, + setup: async page => { + await page.windowScrollTo({ top: 200 }); + }, + }, + { + description: `high contrast header variant in landing page (${width}px)`, + path: 'app-layout/landing-page', + screenshotType: 'viewport' as const, + configuration: { width }, + }, + ]), + }, + + // ── High contrast header variant ────────────────────────────────────── + { + description: 'High contrast header variant', + tests: [ + ...[1400, 600].flatMap(width => [ + { + description: `with breadcrumbs and notifications at ${width}px`, + path: 'app-layout/high-contrast-header-variant', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + queryParams: { hasBreadcrumbs: 'true', hasNotifications: 'true', hasContainer: 'true' }, + } as TestDefinition, + { + description: `without overlap at ${width}px`, + path: 'app-layout/high-contrast-header-variant', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + queryParams: { disableOverlap: 'true' }, + } as TestDefinition, + { + description: `with content layout at ${width}px`, + path: 'app-layout/high-contrast-header-variant', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + queryParams: { + hasBreadcrumbs: 'true', + hasNotifications: 'true', + hasContainer: 'true', + hasContentLayout: 'true', + }, + } as TestDefinition, + ]), + ], + }, + ], +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-multi.ts b/test/definitions/visual/app-layout-multi.ts new file mode 100644 index 0000000000..babf9733cf --- /dev/null +++ b/test/definitions/visual/app-layout-multi.ts @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Multiple instances', + componentName: 'app-layout', + tests: [600, 1280].flatMap(width => [ + { + description: `simple (${width}px)`, + path: 'app-layout/multi-layout-simple', + screenshotType: 'viewport' as const, + configuration: { width }, + }, + { + description: `iframe (${width}px)`, + path: 'app-layout/multi-layout-iframe', + screenshotType: 'viewport' as const, + configuration: { width }, + }, + ]), +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-responsive-1280.ts b/test/definitions/visual/app-layout-responsive-1280.ts new file mode 100644 index 0000000000..d5fe823f54 --- /dev/null +++ b/test/definitions/visual/app-layout-responsive-1280.ts @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { responsiveTests } from './app-layout-responsive-tests'; + +const suite = responsiveTests(1280); +export default suite; diff --git a/test/definitions/visual/app-layout-responsive-1400.ts b/test/definitions/visual/app-layout-responsive-1400.ts new file mode 100644 index 0000000000..1cb519005c --- /dev/null +++ b/test/definitions/visual/app-layout-responsive-1400.ts @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { responsiveTests } from './app-layout-responsive-tests'; + +const suite = responsiveTests(1400); +export default suite; diff --git a/test/definitions/visual/app-layout-responsive-1920.ts b/test/definitions/visual/app-layout-responsive-1920.ts new file mode 100644 index 0000000000..88b8c6caf3 --- /dev/null +++ b/test/definitions/visual/app-layout-responsive-1920.ts @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { responsiveTests } from './app-layout-responsive-tests'; + +const suite = responsiveTests(1920); +export default suite; diff --git a/test/definitions/visual/app-layout-responsive-2540.ts b/test/definitions/visual/app-layout-responsive-2540.ts new file mode 100644 index 0000000000..9dd62c00db --- /dev/null +++ b/test/definitions/visual/app-layout-responsive-2540.ts @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { responsiveTests } from './app-layout-responsive-tests'; + +const suite = responsiveTests(2540); +export default suite; diff --git a/test/definitions/visual/app-layout-responsive-600.ts b/test/definitions/visual/app-layout-responsive-600.ts new file mode 100644 index 0000000000..f6fd3665cc --- /dev/null +++ b/test/definitions/visual/app-layout-responsive-600.ts @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { responsiveTests } from './app-layout-responsive-tests'; + +const suite = responsiveTests(600); +export default suite; diff --git a/test/definitions/visual/app-layout-responsive-tests.ts b/test/definitions/visual/app-layout-responsive-tests.ts new file mode 100644 index 0000000000..f0ac91d121 --- /dev/null +++ b/test/definitions/visual/app-layout-responsive-tests.ts @@ -0,0 +1,159 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +/** + * Shared responsive test scenarios for app-layout. Each width gets its own + * definition file and test runner so that Jest sharding can parallelize them. + */ +export function responsiveTests(width: number): TestSuite { + return { + description: `AppLayout responsive width ${width}px`, + componentName: 'app-layout', + tests: [ + { + description: 'default', + path: 'app-layout/default', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'navigation drawer is open', + path: 'app-layout/with-wizard', + screenshotType: 'viewport', + configuration: { width }, + setup: async page => { + await page.click('[aria-label="Open navigation"]'); + }, + }, + { + description: 'wizard', + path: 'app-layout/with-wizard', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'with wizard and table', + path: 'app-layout/with-wizard-and-table', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'with wizard, table, and breadcrumbs', + path: 'app-layout/with-wizard-and-table', + screenshotType: 'viewport', + configuration: { width }, + queryParams: { hasBreadcrumbs: 'true' }, + }, + { + description: 'notifications', + path: 'app-layout/with-notifications', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'breadcrumbs', + path: 'app-layout/with-breadcrumbs', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'notifications and breadcrumbs', + path: 'app-layout/with-breadcrumbs-notifications', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'dashboard content type', + path: 'app-layout/dashboard-content-type', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'fixed header and footer', + path: 'app-layout/with-fixed-header-footer', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'disableBodyScroll - empty', + path: 'app-layout/legacy-nav-empty', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'disableBodyScroll - with content', + path: 'app-layout/legacy-nav-scrollable', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'disableBodyScroll - with split panel', + path: 'app-layout/legacy-nav-scrollable-with-split-panel', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'disable paddings', + path: 'app-layout/disable-paddings', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'disable paddings with breadcrumbs', + path: 'app-layout/disable-paddings-breadcrumbs', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'sticky notifications', + path: 'app-layout/with-sticky-notifications', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'sticky notifications scrolled down', + path: 'app-layout/with-sticky-notifications', + screenshotType: 'viewport', + configuration: { width }, + setup: async page => { + await page.windowScrollTo({ top: 2000 }); + }, + }, + { + description: 'layout without panels', + path: 'app-layout/no-panels', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'layout without panels but with notifications', + path: 'app-layout/no-panels-with-notifications', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'with drawers', + path: 'app-layout/with-drawers', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'with empty drawers', + path: 'app-layout/with-drawers-empty', + screenshotType: 'viewport', + configuration: { width }, + }, + { + description: 'with open drawer', + path: 'app-layout/with-drawers', + screenshotType: 'viewport', + configuration: { width }, + setup: async page => { + await page.click('[aria-label="Security trigger button"]'); + }, + }, + ], + }; +} diff --git a/test/definitions/visual/app-layout-sticky-table-header-split-panel.ts b/test/definitions/visual/app-layout-sticky-table-header-split-panel.ts new file mode 100644 index 0000000000..6a0b899686 --- /dev/null +++ b/test/definitions/visual/app-layout-sticky-table-header-split-panel.ts @@ -0,0 +1,78 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Sticky header with split panel', + componentName: 'app-layout', + tests: [ + { + description: 'scrolling to bottom with closed split panel (1 table row)', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'viewport', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-1"]'); + await page.scrollToBottom('html'); + }, + }, + { + description: 'scrolling to bottom with closed split panel (30 table rows)', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'viewport', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-30"]'); + await page.scrollToBottom('html'); + }, + }, + { + description: 'header stays sticky with open split panel (1 table row)', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'viewport', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-1"]'); + await page.click('aria/Open panel'); + await page.scrollToBottom('html'); + }, + }, + { + description: 'header stays sticky with open split panel (30 table rows)', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'viewport', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-30"]'); + await page.click('aria/Open panel'); + await page.scrollToBottom('html'); + }, + }, + { + description: 'header stays sticky when mounting and unmounting a second table', + path: 'app-layout/with-sticky-table-and-split-panel', + screenshotType: 'viewport', + configuration: { width: 1280, height: 900 }, + setup: async page => { + await page.click('[data-testid="set-item-count-to-30"]'); + await page.click('aria/Open panel'); + await page.windowScrollTo({ top: 0 }); + await page.click('aria/Close panel'); + await page.scrollToBottom('html'); + }, + }, + // ── Max content width ───────────────────────────────────────────────── + { + description: 'maxContentWidth set to Number.MAX_VALUE', + path: 'app-layout/refresh-content-width', + screenshotType: 'viewport', + configuration: { width: 1280, height: 700 }, + setup: async page => { + await page.click('[data-test-id="button_width-number-max_value"]'); + }, + }, + ], +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-toolbar.ts b/test/definitions/visual/app-layout-toolbar.ts new file mode 100644 index 0000000000..fb174b0a71 --- /dev/null +++ b/test/definitions/visual/app-layout-toolbar.ts @@ -0,0 +1,23 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Toolbar', + componentName: 'app-layout', + tests: [ + { + description: 'multiple nested instances (no breadcrumbs dedup)', + path: 'app-layout-toolbar/multi-layout-with-hidden-instances', + screenshotType: 'viewport', + }, + { + description: 'no toolbar', + path: 'app-layout-toolbar/without-toolbar', + screenshotType: 'viewport', + }, + ], +}; + +export default suite; diff --git a/test/definitions/visual/app-layout-z-index.ts b/test/definitions/visual/app-layout-z-index.ts new file mode 100644 index 0000000000..b0dacf8bc5 --- /dev/null +++ b/test/definitions/visual/app-layout-z-index.ts @@ -0,0 +1,60 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestDefinition, TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Z-index', + componentName: 'app-layout', + tests: [ + ...[600, 1280].flatMap(width => [ + { + description: `button dropdown (${width}px)`, + path: 'app-layout/with-absolute-components', + screenshotType: 'viewport' as const, + configuration: { width }, + setup: async page => { + await page.click('button=Button dropdown'); + await page.click('[data-testid="2"]'); + await page.windowScrollTo({ top: 300 }); + }, + } as TestDefinition, + { + description: `select (${width}px)`, + path: 'app-layout/with-absolute-components', + screenshotType: 'viewport' as const, + configuration: { width, height: 800 }, + setup: async page => { + await page.click('[data-testid="select-demo"] button'); + await page.windowScrollTo({ top: 300 }); + }, + } as TestDefinition, + { + description: `split-panel and full-page table (${width}px)`, + path: 'app-layout/with-full-page-table-and-split-panel', + screenshotType: 'viewport' as const, + configuration: { width }, + }, + ]), + { + description: 'split-panel and full-page with open navigation (600px)', + path: 'app-layout/with-full-page-table-and-split-panel', + screenshotType: 'viewport' as const, + configuration: { width: 600 }, + setup: async page => { + await page.click('button[aria-label="Open navigation"]'); + }, + }, + { + description: 'split-panel and full-page with open tools (600px)', + path: 'app-layout/with-full-page-table-and-split-panel', + screenshotType: 'viewport' as const, + configuration: { width: 600 }, + setup: async page => { + await page.click('button[aria-label="Open tools"]'); + }, + }, + ], +}; + +export default suite; diff --git a/test/definitions/visual/app-layout.ts b/test/definitions/visual/app-layout.ts new file mode 100644 index 0000000000..683efdcc25 --- /dev/null +++ b/test/definitions/visual/app-layout.ts @@ -0,0 +1,104 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'AppLayout', + componentName: 'app-layout', + tests: [ + { + description: 'no scrollbars at 320px', + path: 'app-layout/default', + screenshotType: 'viewport', + configuration: { width: 320 }, + }, + { + description: 'drawer buttons alignment', + path: 'app-layout/default', + screenshotType: 'viewport', + configuration: { width: 800 }, + setup: async page => { + await page.click('[aria-label="Open tools"]'); + }, + }, + { + description: 'disable paddings - navigation closed', + path: 'app-layout/disable-paddings', + screenshotType: 'viewport', + configuration: { width: 1280 }, + setup: async page => { + await page.click('[aria-label="Close navigation"]'); + }, + }, + { + description: 'panels stacking on mobile', + path: 'app-layout/all-panels-open', + screenshotType: 'viewport', + configuration: { width: 600 }, + }, + { + description: 'wrapping long words', + path: 'app-layout/text-wrap', + screenshotType: 'viewport', + }, + { + description: 'fill content area', + path: 'app-layout/fill-content-area', + screenshotType: 'viewport', + }, + { + description: 'with tools and drawers', + path: 'app-layout/with-drawers', + screenshotType: 'viewport', + queryParams: { hasTools: 'true' }, + }, + { + description: 'with open drawer and open side split panel', + path: 'app-layout/with-drawers', + screenshotType: 'viewport', + configuration: { width: 1400 }, + queryParams: { splitPanelPosition: 'side' }, + setup: async page => { + await page.click('[aria-label="Security trigger button"]'); + await page.click('[aria-label="Open panel"]'); + }, + }, + + // regression for https://github.com/cloudscape-design/components/pull/1612 + { + description: 'with open drawer and open side split panel after resize', + path: 'app-layout/with-drawers', + screenshotType: 'viewport', + configuration: { width: 1500 }, + queryParams: { splitPanelPosition: 'side' }, + setup: async page => { + await page.click('[aria-label="Security trigger button"]'); + await page.click('[aria-label="Open panel"]'); + await page.setWindowSize({ width: 1400, height: 800 }); + }, + }, + + // ── Transitions ─────────────────────────────────────────────────────── + { + description: 'transition from 400px to 1800px', + path: 'app-layout/default', + screenshotType: 'viewport', + configuration: { width: 400, height: 400 }, + setup: async page => { + await page.setWindowSize({ width: 1800, height: 400 }); + }, + }, + { + description: 'transition from 1800px to 400px', + path: 'app-layout/default', + screenshotType: 'viewport', + configuration: { width: 1800, height: 400 }, + setup: async page => { + await page.setWindowSize({ width: 400, height: 400 }); + }, + }, + ], +}; + +export default suite; diff --git a/test/definitions/visual/area-chart.ts b/test/definitions/visual/area-chart.ts new file mode 100644 index 0000000000..cfa0b5e47b --- /dev/null +++ b/test/definitions/visual/area-chart.ts @@ -0,0 +1,136 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const TEST_CHART_FILTER_TRIGGER = '#linear-latency-chart button'; +const TEST_CHART_TOOLTIP_HEADER = '#linear-latency-chart h2'; + +const suite: TestSuite = { + description: 'Area chart', + componentName: 'area-chart', + tests: [ + { + description: 'permutations', + path: 'area-chart/permutations', + screenshotType: 'permutations', + }, + { + description: 'fit-height', + path: 'area-chart/fit-height', + screenshotType: 'screenshotArea', + }, + { + description: 'fit-height no filter, no legend', + path: 'area-chart/fit-height', + screenshotType: 'screenshotArea', + queryParams: { hideFilter: 'true', hideLegend: 'true' }, + }, + { + description: 'fit-height, no legend', + path: 'area-chart/fit-height', + screenshotType: 'screenshotArea', + queryParams: { hideLegend: 'true' }, + }, + { + description: 'chart plot has a focus outline', + path: 'area-chart/test', + screenshotType: 'viewport', + configuration: { width: 800, height: 800 }, + setup: async page => { + await page.click(TEST_CHART_FILTER_TRIGGER); + await page.keys(['Escape']); + await page.focusNextElement(); + }, + }, + { + description: 'can navigate along X axis highlighting all series with keyboard', + path: 'area-chart/test', + screenshotType: 'viewport', + configuration: { width: 800, height: 800 }, + setup: async page => { + await page.click(TEST_CHART_FILTER_TRIGGER); + await page.keys(['Escape']); + await page.focusNextElement(); + await page.keys(['ArrowRight', 'ArrowRight']); + await page.waitForVisible(TEST_CHART_TOOLTIP_HEADER); + }, + }, + { + description: 'can navigate a specific series with keyboard', + path: 'area-chart/test', + screenshotType: 'viewport', + configuration: { width: 800, height: 800 }, + setup: async page => { + await page.click(TEST_CHART_FILTER_TRIGGER); + await page.keys(['Escape']); + await page.focusNextElement(); + await page.keys(['ArrowRight']); + await page.keys(['ArrowDown']); + await page.keys(['ArrowRight']); + await page.waitForVisible(TEST_CHART_TOOLTIP_HEADER); + }, + }, + { + description: 'selects correct series when navigated back from legend', + path: 'area-chart/test', + screenshotType: 'viewport', + configuration: { width: 800, height: 800 }, + setup: async page => { + await page.click(TEST_CHART_FILTER_TRIGGER); + await page.keys(['Escape']); + await page.keys(['Tab']); + await page.keys(['Tab']); + await page.keys(['ArrowRight']); + await page.keys(['Shift', 'Tab']); + await page.keys(['ArrowRight']); + await page.waitForVisible(TEST_CHART_TOOLTIP_HEADER); + }, + }, + { + description: 'can pin popover for all data points at a given X coordinate with keyboard', + path: 'area-chart/test', + screenshotType: 'viewport', + configuration: { width: 800, height: 800 }, + setup: async page => { + await page.click(TEST_CHART_FILTER_TRIGGER); + await page.keys(['Escape']); + await page.focusNextElement(); + await page.keys(['ArrowRight']); + await page.keys(['ArrowRight']); + await page.waitForVisible(TEST_CHART_TOOLTIP_HEADER); + await page.keys(['Enter']); + await page.waitForVisible('[aria-label="Dismiss"]'); + }, + }, + { + description: 'can pin popover for a point in a specific series with keyboard', + path: 'area-chart/test', + screenshotType: 'viewport', + configuration: { width: 800, height: 800 }, + setup: async page => { + await page.click(TEST_CHART_FILTER_TRIGGER); + await page.keys(['Escape']); + await page.focusNextElement(); + await page.keys(['ArrowRight']); + await page.keys(['ArrowDown']); + await page.keys(['ArrowRight']); + await page.waitForVisible(TEST_CHART_TOOLTIP_HEADER); + await page.keys(['Enter']); + await page.waitForVisible('[aria-label="Dismiss"]'); + }, + }, + { + description: 'shows popover on hover', + path: 'area-chart/test', + screenshotType: 'viewport', + configuration: { width: 800, height: 800 }, + setup: async page => { + await page.hoverElement('[aria-label="Linear latency chart"]', 200, 50); + await page.waitForVisible(TEST_CHART_TOOLTIP_HEADER); + }, + }, + ], +}; + +export default suite; diff --git a/test/definitions/visual/attribute-editor.ts b/test/definitions/visual/attribute-editor.ts new file mode 100644 index 0000000000..33da3fad86 --- /dev/null +++ b/test/definitions/visual/attribute-editor.ts @@ -0,0 +1,31 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Attribute Editor', + componentName: 'attribute-editor', + tests: [360, 768, 992].flatMap(width => [ + { + description: `permutations at ${width}px`, + path: 'attribute-editor/permutations', + screenshotType: 'permutations' as const, + configuration: { width }, + }, + { + description: `customizable-footer at ${width}px`, + path: 'attribute-editor/customizable-footer', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + }, + { + description: `with long select at ${width}px`, + path: 'attribute-editor/select-with-long-value', + screenshotType: 'screenshotArea' as const, + configuration: { width }, + }, + ]), +}; + +export default suite; diff --git a/test/definitions/visual/autosuggest.ts b/test/definitions/visual/autosuggest.ts new file mode 100644 index 0000000000..92f4d46897 --- /dev/null +++ b/test/definitions/visual/autosuggest.ts @@ -0,0 +1,70 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import createWrapper from '../../../lib/components/test-utils/selectors'; +import { TestDefinition, TestSuite } from '../types'; + +const wrapper = createWrapper(); + +const suite: TestSuite = { + description: 'Autosuggest', + componentName: 'autosuggest', + tests: [ + { + description: 'permutations', + path: 'autosuggest/permutations', + screenshotType: 'permutations', + setup: async page => { + await page.click('input'); + }, + }, + { + description: 'permutations for async properties', + path: 'autosuggest/permutations-async', + screenshotType: 'permutations', + setup: async page => { + await page.click('input'); + }, + }, + { + description: 'Displays options with groups correctly', + path: 'autosuggest/scenarios', + screenshotType: 'screenshotArea', + setup: async page => { + await page.click('input'); + }, + }, + { + description: 'Correctly displays dropdown regions', + path: 'autosuggest/regions-scenarios', + screenshotType: 'screenshotArea', + setup: async page => { + await page.click('input'); + }, + }, + { + description: 'Long virtual list - navigate to last item', + path: 'autosuggest/virtual-scroll', + screenshotType: 'screenshotArea', + setup: async page => { + await page.click(wrapper.findAutosuggest().findNativeInput().toSelector()); + await page.keys(['ArrowUp']); + }, + }, + ...[true, false].map( + virtualScroll => + ({ + description: `with custom renderOption (virtualScroll=${virtualScroll})`, + path: 'autosuggest/custom-render-option', + screenshotType: 'screenshotArea' as const, + queryParams: { virtualScroll: String(virtualScroll) }, + setup: async page => { + await page.click(wrapper.findAutosuggest().findNativeInput().toSelector()); + await page.keys(['ArrowDown']); + }, + }) as TestDefinition + ), + ], +}; + +export default suite; diff --git a/test/definitions/visual/badge.ts b/test/definitions/visual/badge.ts new file mode 100644 index 0000000000..7d405ed843 --- /dev/null +++ b/test/definitions/visual/badge.ts @@ -0,0 +1,23 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'Badge', + componentName: 'badge', + tests: [ + { + description: 'permutation page', + path: 'badge/permutations', + screenshotType: 'permutations', + }, + { + description: 'style custom page', + path: 'badge/style-custom-types', + screenshotType: 'screenshotArea', + }, + ], +}; + +export default suite; diff --git a/tsconfig.test-definitions.json b/tsconfig.test-definitions.json index 30e82b4047..3f5961a78b 100644 --- a/tsconfig.test-definitions.json +++ b/tsconfig.test-definitions.json @@ -16,5 +16,5 @@ "skipLibCheck": true }, "include": ["test/definitions", "test/types.ts"], - "exclude": [] + "exclude": ["test/definitions/utils.ts"] } diff --git a/tsconfig.visual.json b/tsconfig.visual.json new file mode 100644 index 0000000000..be61d962ef --- /dev/null +++ b/tsconfig.visual.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.integ.json", + "include": ["test/definitions/utils.ts", "test/visual/**/*.test.ts", "types"] +}