diff --git a/.github/workflows/lighthouse.yml b/.github/workflows/lighthouse.yml index 5e01999..3b51d83 100644 --- a/.github/workflows/lighthouse.yml +++ b/.github/workflows/lighthouse.yml @@ -1,13 +1,11 @@ name: Lighthouse on: + # Every pull request starts this workflow, and the `changes` job skips the + # audit when only bench changed. A skipped job reports success, so "Lighthouse + # audit" can be a required check; a workflow filtered out by `paths` would + # never report and would block the merge. pull_request: - # Lighthouse audits the marketing site only. It isn't a required check, so - # a workflow-level filter is enough here; ci.yml gates its required job with - # the changed-paths action instead. - paths-ignore: - - 'bench/**' - - '.github/workflows/bench-*.yml' workflow_dispatch: permissions: @@ -21,10 +19,29 @@ concurrency: cancel-in-progress: true jobs: + changes: + name: Detect changed paths + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + site: ${{ steps.paths.outputs.site }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + persist-credentials: false + - id: paths + uses: ./.github/actions/changed-paths + lighthouse: name: Lighthouse audit + needs: changes + # Lighthouse audits the marketing site only. Skipped, which counts as + # passing, when nothing outside bench/ changed. If the path check itself + # fails, run anyway rather than skip. + if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.site == 'true') }} runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 20 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -81,9 +98,16 @@ jobs: temporaryPublicStorage: true # Build a markdown table from the .lighthouseci/lhr-*.json reports. - # `unique_by(.url)` collapses duplicate rows if `numberOfRuns > 1` - # (the action writes the median LHR per URL, but be defensive); - # `sort_by(.url)` keeps row order deterministic across runs. + # lighthouserc.cjs collects three runs per URL and asserts on the best + # one, so the table shows each URL's best run by performance, the run + # the check is judged by. `group_by(.url)` also keeps row order + # deterministic. The linked report is the one the action uploads for + # that URL and may be a different run. + # + # Fork PRs, and Dependabot's PRs from this repository, get a read-only + # GITHUB_TOKEN that can't post comments. A failed comment must not fail + # this required check, so only same-repository, non-Dependabot PRs get + # one, and both comment steps are non-fatal for any other token limit. # `if: always()` so the PR comment still posts when assertions fail — # reviewers can see *which* category regressed without digging through # the workflow log. @@ -93,8 +117,9 @@ jobs: # the 🟢 cutoff, so a cell can pass assertions and still show 🟡 — the # visual signals "we locked this in but it's improvable." - name: Format PR comment - if: always() && github.event_name == 'pull_request' && steps.lighthouse.outputs.links != '' + if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' && steps.lighthouse.outputs.links != '' id: lh-comment + continue-on-error: true env: LINKS_JSON: ${{ steps.lighthouse.outputs.links }} run: | @@ -114,8 +139,8 @@ jobs: bp: (.categories["best-practices"].score * 100 | floor), seo: (.categories.seo.score * 100 | floor) }) - | unique_by(.url) - | sort_by(.url) + | group_by(.url) + | map(max_by(.perf)) | map( "| `" + (.url | sub("^https?://[^/]+"; "")) + "`" + " | " + cell(.perf) @@ -133,6 +158,7 @@ jobs: - name: Post or update PR comment if: github.event_name == 'pull_request' && steps.lh-comment.outputs.body != '' + continue-on-error: true uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: header: lighthouse diff --git a/CLAUDE.md b/CLAUDE.md index b47bb02..5eb4812 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,7 +21,7 @@ The repository holds two independent apps: They share one Bun workspace and the CI policy, and deploy separately: - **Toolchain.** `bench/web` is a member of the root Bun workspace: one `bun.lock`, one `bunfig.toml`, one `bun install` at the root. Bun's isolated linker gives each package its own dependencies; both apps run Next.js 16.4.1, React 19.3 and TypeScript 7. The root `biome.json` lints and formats both apps: a `bench/**` override keeps bench's old Prettier style (single quotes, trailing commas, 100 columns) and sets bench's lint rules (see `bench/AGENTS.md`), and `bun run check:ci` at the root checks all of it. Both `tsconfig.json` files extend the root `tsconfig.base.json`, which holds the compiler options the apps share, including `noUncheckedIndexedAccess` and `exactOptionalPropertyTypes`. Each app keeps its own target, paths and extra flags (bench's `noFallthroughCasesInSwitch` and `noUncheckedSideEffectImports`). The root `tsconfig.json` excludes `bench/`, and `bench/web/postcss.config.mjs` keeps Next.js from picking up the root's Tailwind config. -- **CI.** `.github/actions/changed-paths` reports which app a change touches. `ci.yml` skips its site jobs on bench-only changes, and `bench-ci.yml` and `bench-deploy.yml` skip on site-only changes. The workspace's root files (`package.json`, `bun.lock`, `bunfig.toml`, `biome.json`, `tsconfig*.json`) count as both. They gate pull requests with `if:`, not `on.paths`, so a skipped required check still passes. `bench-deploy.yml` keeps `on.paths` for pushes to main, because a push run that only skips would still queue in its production concurrency group. Dependency review, the secret scan and REUSE run on every change. +- **CI.** `.github/actions/changed-paths` reports which app a change touches. `ci.yml` and `lighthouse.yml` skip their site jobs on bench-only changes, and `bench-ci.yml` and `bench-deploy.yml` skip on site-only changes. The workspace's root files (`package.json`, `bun.lock`, `bunfig.toml`, `biome.json`, `tsconfig*.json`) count as both. They gate pull requests with `if:`, not `on.paths`, so a skipped required check still passes. `bench-deploy.yml` keeps `on.paths` for pushes to main, because a push run that only skips would still queue in its production concurrency group. Dependency review, the secret scan and REUSE run on every change. - **Deploys.** The `vortex` Vercel project builds the site through the git integration, and `scripts/vercel-ignore-build.sh` skips bench-only commits. bench deploys to the `benchmarks-website` Vercel project (Root Directory `bench/web`) from `bench-deploy.yml`, with the Vercel CLI, on changes to `bench/` and to the root `package.json`, `bun.lock`, `bunfig.toml` and `tsconfig.base.json`. Schema migrations go through `bench-schema-deploy.yml`, which only runs manually from `main` because the AWS role trusts nothing else. ## Architecture Overview @@ -100,7 +100,7 @@ Three places must agree with each other and with what Vercel runs. Vercel offers - `@types/node` → `^26`, matching the runtime so `tsc` cannot green-light an API that production does not have. Keep them in lockstep when bumping, and move to a new major only after Vercel offers it; check https://vercel.com/docs/functions/runtimes/node-js/node-js-versions first. -10. **Renovate auto-merge** is patch + minor only via `:automergeStableNonMajor`. Major bumps stay open for human review. The 7-day cooldown is the first gate, and the required checks on `main` are the second. Renovate merges through GitHub's auto-merge, which waits only for required checks: `Lint, build, verify` (lint, typecheck, build, `bun run verify` and Playwright), `dependency-review`, `secret-scan`, `DCO`, `REUSE compliance`, `Submit bun.lock`, `Bench: format, lint, build, test`, `Bench: deploy` and `Vercel`, plus a CodeQL code scanning rule. Lighthouse and `bun audit` are not required, so they don't hold an auto-merge. Lock file maintenance automerges too. Renovate cannot apply `minimumReleaseAge` to a Bun lock refresh, so `renovate.json` turns it off for that update type; `bunfig.toml` enforces the same wait while Bun regenerates `bun.lock`. `biome.json` takes its `$schema` from the installed `@biomejs/biome` package rather than a versioned URL, so a Biome update can't fail on a schema version mismatch, and it leaves Markdown and YAML alone, as Biome 2.5 did. +10. **Renovate auto-merge** is patch + minor only via `:automergeStableNonMajor`. Major bumps stay open for human review. The 7-day cooldown is the first gate, and the required checks on `main` are the second. Renovate merges through GitHub's auto-merge, which waits only for required checks: `Lint, build, verify` (lint, typecheck, build, `bun run verify` and Playwright), `dependency-review`, `secret-scan`, `DCO`, `REUSE compliance`, `Submit bun.lock`, `Bench: format, lint, build, test`, `Bench: deploy`, `Lighthouse audit` and `Vercel`, plus a CodeQL code scanning rule. `bun audit` is not required, so it doesn't hold an auto-merge. Lighthouse takes the best of three runs per URL, so one slow shared runner doesn't fail it. Lock file maintenance automerges too. Renovate cannot apply `minimumReleaseAge` to a Bun lock refresh, so `renovate.json` turns it off for that update type; `bunfig.toml` enforces the same wait while Bun regenerates `bun.lock`. `biome.json` takes its `$schema` from the installed `@biomejs/biome` package rather than a versioned URL, so a Biome update can't fail on a schema version mismatch, and it leaves Markdown and YAML alone, as Biome 2.5 did. ### bench/ (Bun workspace member) diff --git a/lighthouserc.cjs b/lighthouserc.cjs index 9b68d77..4372e64 100644 --- a/lighthouserc.cjs +++ b/lighthouserc.cjs @@ -85,7 +85,13 @@ module.exports = { startServerCommand: "bun run start", startServerReadyPattern: "Ready", url: baseUrls, - numberOfRuns: 1, + // "Lighthouse audit" is a required check, so one slow shared runner + // must not fail it. With several runs, LHCI asserts on the best one + // (its default `aggregationMethod: "optimistic"`): a real regression + // shows in every run and still fails, an outlier run does not. One + // blog-post run fell to 0.74 performance on 2026-10-30 against a usual + // 1.00. + numberOfRuns: 3, settings: { preset: "desktop", throttlingMethod: "simulate",