chore: Add all visual regression tests

.github/workflows/deploy.yml

@@ -53,6 +53,13 @@ jobs:
           name: dev-pages-react${{ matrix.react }}
           path: pages/lib/static-default
 
+      - name: Upload test utils selectors artifact
+        if: matrix.react == 18
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-utils-selectors
+          path: lib/components
+
   deploy:
     needs: quick-build
     name: deploy${{ matrix.react != 16 && format(' (React {0})', matrix.react) || '' }}
@@ -65,3 +72,14 @@ jobs:
     with:
       artifact-name: dev-pages-react${{ matrix.react }}
       deployment-path: pages/lib/static-default
+
+  visual:
+    name: Visual regression
+    needs: quick-build
+    if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+    uses: ./.github/workflows/visual-regression.yml
+    secrets: inherit
+    with:
+      pr-artifact-name: dev-pages-react18
+      test-utils-artifact-name: test-utils-selectors
+      caller-run-id: ${{ github.run_id }}

.github/workflows/visual-regression.yml

@@ -0,0 +1,197 @@
+name: Visual Regression Tests
+
+on:
+  workflow_call:
+    inputs:
+      pr-artifact-name:
+        description: 'Name of the artifact containing PR pages (built by the caller workflow).'
+        required: true
+        type: string
+      test-utils-artifact-name:
+        description: 'Name of the artifact containing test-utils selectors.'
+        required: true
+        type: string
+      caller-run-id:
+        description: 'The run ID of the calling workflow, used to download artifacts it uploaded.'
+        required: true
+        type: string
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+  deployments: write
+
+jobs:
+  # Build the baseline (main branch) pages once and share them across all browser jobs.
+  build-baseline:
+    name: Build baseline pages
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+
+      - name: Install dependencies
+        run: npm i
+
+      # Use a git worktree so the baseline has its own directory and its own
+      # node_modules. This means a PR that changes package-lock.json will still
+      # produce a correct baseline: the baseline installs from main's lockfile
+      # and the PR build installs from the PR's lockfile, so both sides use the
+      # dependency versions that are correct for their respective source trees.
+      - name: Create baseline worktree from origin/main
+        run: git worktree add /tmp/baseline origin/main
+
+      - name: Install baseline dependencies
+        run: npm i
+        working-directory: /tmp/baseline
+
+      - name: Build baseline pages
+        run: npx gulp quick-build
+        working-directory: /tmp/baseline
+        env:
+          NODE_ENV: production
+
+      - name: Bundle baseline pages
+        run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path ${{ github.workspace }}/pages/lib/static-visual-baseline
+        working-directory: /tmp/baseline
+        env:
+          NODE_ENV: production
+
+      - name: Upload baseline artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: visual-baseline-pages
+          path: pages/lib/static-visual-baseline
+          retention-days: 1
+
+  visual:
+    name: Visual regression (shard ${{ matrix.shard }})
+    needs: [build-baseline]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+
+      - name: Setup Chrome and ChromeDriver
+        uses: browser-actions/setup-chrome@v1
+        with:
+          chrome-version: stable
+
+      - name: Install dependencies
+        run: npm i
+
+      - name: Download PR pages artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.pr-artifact-name }}
+          path: pages/lib/static-default
+          github-token: ${{ github.token }}
+          run-id: ${{ inputs.caller-run-id }}
+
+      - name: Download baseline artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: visual-baseline-pages
+          path: pages/lib/static-visual-baseline
+
+      - name: Download test utils artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.test-utils-artifact-name }}
+          path: lib/components
+          github-token: ${{ github.token }}
+          run-id: ${{ inputs.caller-run-id }}
+
+      # ── Run tests ─────────────────────────────────────────────────────────
+
+      - name: Generate visual test files
+        run: node build-tools/visual/generate-tests.js
+
+      - name: Start test server (port 8080)
+        run: npx --yes serve --no-clipboard --listen 8080 pages/lib/static-default &
+
+      - name: Start baseline server (port 8081)
+        run: npx --yes serve --no-clipboard --listen 8081 pages/lib/static-visual-baseline &
+
+      - name: Wait for servers to be ready
+        run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081
+
+      - name: Run visual regression tests
+        run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js --shard=${{ matrix.shard }}/30
+        env:
+          TZ: UTC
+
+      - name: Upload diff artifacts
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: visual-regression-diffs-shard-${{ matrix.shard }}
+          path: visual-regression-output/
+          retention-days: 14
+
+      - name: Upload Allure results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: allure-results-shard-${{ matrix.shard }}
+          path: allure-results/
+          retention-days: 3
+
+  report:
+    name: Generate Allure Report
+    if: always()
+    needs: [visual]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Download all Allure results
+        uses: actions/download-artifact@v4
+        with:
+          pattern: allure-results-shard-*
+          path: allure-results
+          merge-multiple: true
+
+      - name: Generate Allure HTML report
+        run: npx --yes allure generate allure-results -o allure-report
+
+      - name: Upload Allure report artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: allure-report
+          path: allure-report/
+          retention-days: 14
+
+  deploy-report:
+    name: Deploy Allure Report
+    if: always()
+    needs: [report]
+    uses: cloudscape-design/actions/.github/workflows/deploy.yml@main
+    secrets: inherit
+    with:
+      artifact-name: allure-report
+      deployment-path: allure-report

.gitignore

@@ -6,6 +6,10 @@ coverage
 lib
 # generated sources
 src/index.ts
+test/visual
+test/definitions/index.ts
+allure-results
+allure-report
 src/test-utils/dom/index.ts
 src/test-utils/selectors
 src/icon/generated

build-tools/tasks/package-json.js

@@ -103,6 +103,10 @@ const devPagesPackageJson = generatePackageJson(path.join(workspace.targetPath,
 
 const testDefinitionsPackageJson = generatePackageJson(path.join(workspace.targetPath, 'test-definitions'), {
   name: '@cloudscape-design/test-definitions',
+  exports: {
+    '.': './index.js',
+    './types': './types.js',
+  },
 });
 
 module.exports = parallel([

build-tools/tasks/test-definitions.js

@@ -1,8 +1,17 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
+const { series } = require('gulp');
 const execa = require('execa');
 const { task } = require('../utils/gulp-utils');
 
-module.exports = task('test-definitions', () =>
+// Generate test/definitions/index.ts from the visual definition files before
+// compiling, so that downstream consumers always get a complete barrel export.
+const generate = task('test-definitions:generate', () =>
+  execa('node', ['build-tools/visual/generate-tests.js'], { stdio: 'inherit' })
+);
+
+const compile = task('test-definitions:compile', () =>
   execa('tsc', ['-p', 'tsconfig.test-definitions.json'], { stdio: 'inherit' })
 );
+
+module.exports = series(generate, compile);

build-tools/visual/generate-tests.js

@@ -0,0 +1,106 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Auto-generates:
+ *   1. test/visual/*.test.ts — one test runner per definition file (for Jest sharding)
+ *   2. test/definitions/index.ts — barrel that exports allSuites from all definitions
+ *
+ * Each definition file that exports a default TestSuite gets included.
+ * Helper files (those without a default export, e.g. shared utils like
+ * app-layout-responsive-tests.ts) are skipped.
+ *
+ * Run this script before executing the visual test suite:
+ *   node build-tools/visual/generate-tests.js
+ */
+const fs = require('fs');
+const path = require('path');
+
+const definitionsDir = path.resolve(__dirname, '../../test/definitions/visual');
+const testOutputDir = path.resolve(__dirname, '../../test/visual');
+const indexOutputPath = path.resolve(__dirname, '../../test/definitions/index.ts');
+
+// Files that are shared helpers (export named functions, not a default suite).
+const HELPER_SUFFIXES = ['-tests'];
+
+function isHelperFile(basename) {
+  return HELPER_SUFFIXES.some(suffix => basename.endsWith(suffix));
+}
+
+function toCamelCase(basename) {
+  return basename.replace(/-([a-z0-9])/g, (_, char) => char.toUpperCase());
+}
+
+function generate() {
+  // Ensure output directory exists
+  if (!fs.existsSync(testOutputDir)) {
+    fs.mkdirSync(testOutputDir, { recursive: true });
+  }
+
+  const files = fs.readdirSync(definitionsDir).filter(f => f.endsWith('.ts') && !f.endsWith('.d.ts'));
+
+  const suiteFiles = [];
+
+  for (const file of files) {
+    const basename = file.replace(/\.ts$/, '');
+
+    if (isHelperFile(basename)) {
+      continue;
+    }
+
+    // Verify the file has a default export by scanning for the pattern
+    const content = fs.readFileSync(path.join(definitionsDir, file), 'utf-8');
+    if (!content.includes('export default')) {
+      continue;
+    }
+
+    suiteFiles.push(basename);
+
+    // Generate test/visual/<basename>.test.ts
+    const testContent = [
+      '// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.',
+      '// SPDX-License-Identifier: Apache-2.0',
+      '// Auto-generated by build-tools/visual/generate-tests.js — do not edit manually.',
+      `import { runTestSuites } from '../definitions/utils';`,
+      `import suite from '../definitions/visual/${basename}';`,
+      '',
+      'runTestSuites([suite]);',
+      '',
+    ].join('\n');
+
+    fs.writeFileSync(path.join(testOutputDir, `${basename}.test.ts`), testContent);
+  }
+
+  // Generate test/definitions/index.ts
+  // Sort by module path for consistent import ordering
+  suiteFiles.sort();
+
+  const imports = suiteFiles.map(basename => {
+    const varName = toCamelCase(basename);
+    return `import ${varName} from './visual/${basename}';`;
+  });
+
+  const arrayEntries = suiteFiles.map(basename => `  ${toCamelCase(basename)},`);
+
+  const indexContent = [
+    '// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.',
+    '// SPDX-License-Identifier: Apache-2.0',
+    '// Auto-generated by build-tools/visual/generate-tests.js — do not edit manually.',
+    '',
+    `import { TestSuite } from './types';`,
+    `export { TestSuite, TestDefinition, ScreenshotType, ScreenshotTestConfiguration } from './types';`,
+    ...imports,
+    '',
+    'export const allSuites: TestSuite[] = [',
+    ...arrayEntries,
+    '];',
+    '',
+  ].join('\n');
+
+  fs.writeFileSync(indexOutputPath, indexContent);
+
+  console.log(`Generated ${suiteFiles.length} visual test files in test/visual/`);
+  console.log(`Generated test/definitions/index.ts with ${suiteFiles.length} suites`);
+}
+
+generate();

build-tools/visual/global-setup.js

@@ -0,0 +1,7 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+module.exports = async () => {
+  const { startWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher');
+  await startWebdriver();
+};

build-tools/visual/global-teardown.js

@@ -0,0 +1,7 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+module.exports = () => {
+  const { shutdownWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher');
+  shutdownWebdriver();
+};