From 1218cdf2c558682ee5fcf696060275e4472424e9 Mon Sep 17 00:00:00 2001 From: Rohan Date: Mon, 2 Nov 2026 00:44:42 +0530 Subject: [PATCH] ci: tier browser coverage for development and release checks --- .github/workflows/ci.yml | 22 ++++++++++++++-- README.md | 2 ++ docs/testing.md | 24 +++++++++++++++++ scripts/ci-test-plan.mjs | 45 ++++++++++++++++++++++++++++++++ scripts/ci-test-plan.test.mjs | 49 +++++++++++++++++++++++++++++++++++ tests/e2e/authoring.spec.ts | 2 +- tests/e2e/recovery.spec.ts | 2 +- tests/e2e/shell-entry.spec.ts | 4 +-- tests/e2e/workspace.spec.ts | 16 ++++++------ 9 files changed, 152 insertions(+), 14 deletions(-) create mode 100644 scripts/ci-test-plan.mjs create mode 100644 scripts/ci-test-plan.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bf5cbd..80d2caa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,9 @@ name: Validate on: pull_request: push: - branches: [main, develop, 'release/**', 'hotfix/**'] + branches: [main, 'release/**', 'hotfix/**'] + schedule: + - cron: '0 4 * * 1' workflow_dispatch: permissions: contents: read @@ -15,15 +17,31 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 with: node-version-file: .nvmrc cache: npm + - name: Select browser coverage + id: browser-plan + env: + PR_BASE_REF: ${{ github.base_ref }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: node scripts/ci-test-plan.mjs - run: npm ci - run: npm run check - run: npm audit --audit-level=low - run: npx playwright install --with-deps chromium - - run: npm run test:e2e + - name: Full browser suite + if: steps.browser-plan.outputs.full == 'true' + run: npm run test:e2e + - name: Critical browser workflows + if: steps.browser-plan.outputs.full != 'true' + run: npm run test:e2e -- --project=functional --grep @critical --reporter=list --output=sandbox/playwright/critical + - name: Visual comparisons for code changes + if: steps.browser-plan.outputs.visual == 'true' + run: npm run test:e2e -- --project=visual - name: Retain browser test evidence if: ${{ !cancelled() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 diff --git a/README.md b/README.md index 508855c..0197848 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,8 @@ npm run test:e2e # fresh build, real engines and visual compa npm run preview -- --host 127.0.0.1 ``` +PR CI runs all fast checks plus twelve critical browser workflows, adding visual comparisons for code changes. Full browser coverage runs weekly, manually and for release/main/hotfix PRs and pushes. Run `npm run test:e2e -- --project=functional --grep @critical` to reproduce the smaller browser gate; `npm run test:e2e` still runs everything. See [CI coverage](docs/testing.md#ci-coverage) for selection and deferred coverage. + Functional checks can use installed Chrome with `PLAYWRIGHT_BROWSER_CHANNEL=chrome npm run test:e2e -- --project=functional`. Visual comparisons always use bundled Chromium and reviewed platform-specific baselines. The browser suite starts its own fresh production server; port **4175** must be free. See [browser testing and baseline review](docs/testing.md). Production hosting must serve `index.html` for direct `/app` navigation while retaining normal asset paths; use the SPA fallback for your hosting provider. `/app/?example=1` is canonicalized to `/app` before the explicitly requested example opens, and reload does not repeat that opening action. diff --git a/docs/testing.md b/docs/testing.md index 4fe119a..cb49742 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -24,6 +24,30 @@ PLAYWRIGHT_BROWSER_CHANNEL=chrome npm run test:e2e -- --project=functional The browser-channel override applies only to functional checks. Visual checks always use the Chromium version installed for the locked Playwright version. The checked-in baselines were generated on Linux; use the same OS, browser and fonts for comparisons. Other platforms need separately reviewed baselines, not wider mismatch thresholds. Visual tests fix the viewport, locale, timezone, system theme and reduced-motion preference, and wait for the example diagram and local fonts before capture. +## CI coverage + +The required `checks` job always runs formatting, architecture, TypeScript, unit/script tests and the dependency audit. Browser coverage is selected separately: + +| Trigger | Browser coverage | +| ----------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Every PR | Twelve `@critical` real-engine scenarios: entry, source-defined views, invalid drafts, rename and cancellation, native view creation, typed part/connection authoring, source undo/redo, exact save/reopen, engine failure recovery and IndexedDB restore | +| PR with code, assets, examples, tooling or unknown paths | All visual comparisons as well as the critical scenarios | +| Documentation-only PR | Critical scenarios; no visual comparisons | +| PR targeting `main`, `release/**`, `hotfix/**`; weekly Monday at 04:00 UTC; manual dispatch; or push to `main`, `release/**`, `hotfix/**` | Complete functional and visual suites | + +There is no duplicate run on a merge push to `develop`; its PR must pass `checks`. Scheduled runs use the repository's default branch. The full run must pass before release, and remains available manually on any branch. Scheduled failures need triage; they do not retroactively block an already merged PR. The reduced PR suite intentionally defers the extended preservation/failure matrix, language assistance, engine identity investigations and example-library sweeps to full runs. + +The selector in `scripts/ci-test-plan.mjs` only exempts Markdown documentation, root license/notice files and CODEOWNERS from visual checks. Missing/empty change lists retain visual coverage; a missing PR base SHA or failed Git comparison fails the job. No whole-workflow path filters leave required checks pending. Critical traces/screenshots use a separate output directory so the visual run does not erase them. Baselines and assertions are unchanged; full local runs still select every test. Browser workers remain at one until isolation and resource use are measured. + +Reproduce the PR browser gate locally: + +```sh +npm run test:e2e -- --project=functional --grep @critical +npm run test:e2e -- --project=visual # also required for code changes +``` + +When adding a semantic command, keep a real-engine acceptance path in the critical set. Use `npm run test:e2e -- --project=functional --grep @critical --list` to review membership. Moving a regression out of the PR gate needs an explicit coverage decision; never remove a tag merely to avoid a failure. + ## Example library The [example catalog](../examples/README.md) supplies independent source projects, per-view engine expectations and visual targets. `npm run test:examples` runs its metadata/inventory/index checks and all library browser scenarios. These scenarios observe the actual worker replies, verify named views and node/relationship multisets, check rendered/partial/unavailable app behavior, and download/reopen exact sources. Unavailable renderers are tested as explicit app behavior rather than skipped. Current diagram PNGs and engine JSON are retained per view under `sandbox/playwright/results/examples-example-library-*/` and attached to the HTML report. These captures record current behavior; they are not visual-comparison baselines or evidence that every listed visual target is satisfied. diff --git a/scripts/ci-test-plan.mjs b/scripts/ci-test-plan.mjs new file mode 100644 index 0000000..a75080c --- /dev/null +++ b/scripts/ci-test-plan.mjs @@ -0,0 +1,45 @@ +import { execFileSync } from 'node:child_process'; +import { appendFileSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; + +// Unknown events and paths deliberately retain visual coverage. +export function testPlan(event, paths = [], baseRef = 'develop') { + if ( + event !== 'pull_request' || + /^(main|release\/.*|hotfix\/.*)$/.test(baseRef) + ) + return { full: true, visual: false }; + const documentationOnly = + paths.length > 0 && + paths.every((path) => + /^(docs\/.*\.md|[^/]+\.md|LICENSE|NOTICE|\.github\/CODEOWNERS)$/.test( + path, + ), + ); + return { full: false, visual: !documentationOnly }; +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + const event = process.env.GITHUB_EVENT_NAME; + let paths = []; + if (event === 'pull_request') { + const base = process.env.PR_BASE_SHA; + if (!/^[a-f0-9]{40}$/.test(base ?? '')) { + throw new Error( + 'A full PR base SHA is required to select browser coverage', + ); + } + paths = execFileSync('git', ['diff', '--name-only', '-z', base, 'HEAD'], { + encoding: 'utf8', + }) + .split('\0') + .filter(Boolean); + } + const plan = testPlan(event, paths, process.env.PR_BASE_REF); + const output = Object.entries(plan) + .map(([key, value]) => `${key}=${value}\n`) + .join(''); + if (process.env.GITHUB_OUTPUT) + appendFileSync(process.env.GITHUB_OUTPUT, output); + process.stdout.write(output); +} diff --git a/scripts/ci-test-plan.test.mjs b/scripts/ci-test-plan.test.mjs new file mode 100644 index 0000000..23d7f64 --- /dev/null +++ b/scripts/ci-test-plan.test.mjs @@ -0,0 +1,49 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { testPlan } from './ci-test-plan.mjs'; + +test('scheduled, manual and release pushes keep the complete browser suite', () => { + for (const event of ['schedule', 'workflow_dispatch', 'push', undefined]) { + assert.deepEqual(testPlan(event), { full: true, visual: false }); + } +}); + +test('documentation-only PRs retain critical workflows without screenshots', () => { + assert.deepEqual( + testPlan('pull_request', [ + 'README.md', + 'docs/testing.md', + '.github/CODEOWNERS', + ]), + { full: false, visual: false }, + ); +}); + +test('code, assets, examples, tooling and unknown paths retain visual coverage', () => { + for (const path of [ + 'src/lib/ui/workbench.css', + 'public/logo.svg', + 'examples/example.sysml', + 'tests/e2e/visual.spec.ts', + 'package-lock.json', + 'playwright.config.ts', + '.github/workflows/ci.yml', + 'docs/diagram.html', + 'new-surface/file', + ]) { + assert.deepEqual(testPlan('pull_request', ['README.md', path]), { + full: false, + visual: true, + }); + } + assert.deepEqual(testPlan('pull_request', []), { full: false, visual: true }); +}); + +test('release-targeted PRs require the full suite before merging', () => { + for (const branch of ['main', 'release/0.2', 'hotfix/parser']) { + assert.deepEqual(testPlan('pull_request', ['README.md'], branch), { + full: true, + visual: false, + }); + } +}); diff --git a/tests/e2e/authoring.spec.ts b/tests/e2e/authoring.spec.ts index 26c5ed5..ec55b8f 100644 --- a/tests/e2e/authoring.spec.ts +++ b/tests/e2e/authoring.spec.ts @@ -13,7 +13,7 @@ import { expectRenderedPositions, } from './support'; -test('creates a typed part and connects its actual inherited usage ports through reviewed source transactions', async ({ +test('@critical creates a typed part and connects its actual inherited usage ports through reviewed source transactions', async ({ page, }, testInfo) => { test.setTimeout(180_000); diff --git a/tests/e2e/recovery.spec.ts b/tests/e2e/recovery.spec.ts index ef4fa53..b01de1c 100644 --- a/tests/e2e/recovery.spec.ts +++ b/tests/e2e/recovery.spec.ts @@ -76,7 +76,7 @@ async function savedDraft(page: Page) { return { expected, draft, raw: (await readRecovery(page))! }; } -test('restores exact applied source, invalid drafts and both view layouts from IndexedDB after reload', async ({ +test('@critical restores exact applied source, invalid drafts and both view layouts from IndexedDB after reload', async ({ page, }) => { const { expected, draft } = await savedDraft(page); diff --git a/tests/e2e/shell-entry.spec.ts b/tests/e2e/shell-entry.spec.ts index 5ea1805..c9efa1e 100644 --- a/tests/e2e/shell-entry.spec.ts +++ b/tests/e2e/shell-entry.spec.ts @@ -40,7 +40,7 @@ async function workshopFiles() { ); } -test('opens the OpenMBEE landing and empty launcher without loading an example or model engine', async ({ +test('@critical opens the OpenMBEE landing and empty launcher without loading an example or model engine', async ({ page, }) => { const engineRequests: string[] = []; @@ -93,7 +93,7 @@ test('opens the OpenMBEE landing and empty launcher without loading an example o expect(engineRequests).toEqual([]); }); -test('landing and direct example links open the real model once and reload returns to the empty launcher', async ({ +test('@critical landing and direct example links open the real model once and reload returns to the empty launcher', async ({ page, }) => { await page.goto('/'); diff --git a/tests/e2e/workspace.spec.ts b/tests/e2e/workspace.spec.ts index 649475d..2dd9a62 100644 --- a/tests/e2e/workspace.spec.ts +++ b/tests/e2e/workspace.spec.ts @@ -15,7 +15,7 @@ import { moveNode, downloadProject, } from './support'; -test('loads real source-defined views and keeps the view stable when changing files', async ({ +test('@critical loads real source-defined views and keeps the view stable when changing files', async ({ page, }) => { await openWorkshop(page); @@ -38,7 +38,7 @@ test('loads real source-defined views and keeps the view stable when changing fi ).toHaveSource(/package WorkshopViews/); await expect(diagramNode(page, 'primary : Motor')).toBeVisible(); }); -test('retains an invalid draft and the last valid diagram', async ({ +test('@critical retains an invalid draft and the last valid diagram', async ({ page, }) => { await openWorkshop(page); @@ -67,7 +67,7 @@ test('retains an invalid draft and the last valid diagram', async ({ page.getByRole('textbox', { name: 'SysML source' }), ).toHaveSource(/package Workshop/); }); -test('uses Toolkit rename, updates both references, and supports exact undo/redo', async ({ +test('@critical uses Toolkit rename, updates both references, and supports exact undo/redo', async ({ page, }) => { await openWorkshop(page); @@ -98,7 +98,7 @@ test('uses Toolkit rename, updates both references, and supports exact undo/redo await expect(diagramNode(page, 'primary : Drive')).toBeVisible(); await expect(diagramNode(page, 'standby : Drive')).toBeVisible(); }); -test('creates a native view declaration through a reviewed transaction', async ({ +test('@critical creates a native view declaration through a reviewed transaction', async ({ page, }) => { await openWorkshop(page); @@ -132,7 +132,7 @@ test('creates a native view declaration through a reviewed transaction', async ( ).toBe(true); expect(bundle.drafts).toEqual(bundle.documents); }); -test('downloads and reopens a project with the draft and diagram positions intact', async ({ +test('@critical downloads and reopens a project with the draft and diagram positions intact', async ({ page, }) => { await openWorkshop(page); @@ -208,7 +208,7 @@ test('downloads and reopens a project with the draft and diagram positions intac expect(reopened.bundle.presentations).toEqual(bundle.presentations); }); -test('cancels a reviewed rename without changing source, references or history', async ({ +test('@critical cancels a reviewed rename without changing source, references or history', async ({ page, }) => { await openWorkshop(page); @@ -233,7 +233,7 @@ test('cancels a reviewed rename without changing source, references or history', expect((await downloadProject(page)).bundle).toEqual(before); }); -test('distinguishes repeated usages when selecting a diagram symbol', async ({ +test('@critical distinguishes repeated usages when selecting a diagram symbol', async ({ page, }) => { await openWorkshop(page); @@ -260,7 +260,7 @@ test('distinguishes repeated usages when selecting a diagram symbol', async ({ } }); -test('retains source, draft and diagram after engine asset failure, then recovers', async ({ +test('@critical retains source, draft and diagram after engine asset failure, then recovers', async ({ page, }) => { await openWorkshop(page);