feat(observability-map): static observability scorer for webapp route entry points #18
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 🗺️ Observability Map | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened] | |
| paths: | |
| - "apps/webapp/app/routes/**" | |
| - "internal-packages/observability-map/**" | |
| # The corpus job below is gated to this package's own paths, so a scheduled run is what still | |
| # scans the tree as it drifts. Nightly rather than per route pull request: a new route can make a | |
| # known laundering shape start paying, but that is a property of the tree accumulating, not of any | |
| # one pull request, and it does not need catching within five minutes of the merge. | |
| schedule: | |
| - cron: "0 3 * * *" | |
| workflow_dispatch: | |
| concurrency: | |
| group: observability-map-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| # The workflow's paths filter is the union of what the two jobs below want, because GitHub | |
| # evaluates it once per workflow. This narrows it again for the corpus job alone. | |
| changes: | |
| name: 🔍 Which paths moved | |
| # Only the pull request path reads this job's output. On a schedule the action has no base to | |
| # diff, warns that `before` is missing and reports the files in the last commit on main, which | |
| # nothing then consults. Skipping it there keeps the nightly off a job it does not need. | |
| if: github.event_name == 'pull_request' | |
| runs-on: warp-ubuntu-latest-x64-2x | |
| outputs: | |
| package: ${{ steps.filter.outputs.package }} | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| fetch-depth: 1 | |
| persist-credentials: false | |
| - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 | |
| id: filter | |
| with: | |
| filters: | | |
| package: | |
| - 'internal-packages/observability-map/**' | |
| - '.github/workflows/observability-map.yml' | |
| # The tree-scale mutation corpus: every known laundering shape applied to the whole route tree, | |
| # asserting the score does not rise. Roughly four and a half minutes for 45 entries, which is why | |
| # it is gated out of the package's default `pnpm test` and run here instead. Unlike the report | |
| # job below it has no token to lose, so it runs for fork PRs too, and unlike the report job it is | |
| # allowed to fail the build. | |
| # | |
| # Gated to this package's own paths rather than running on every route pull request. What the | |
| # corpus measures is the TOOL's resistance to laundering, and only an edit to the tool can weaken | |
| # that, so a routes-only change was paying four and a half minutes of a 4x runner for a result | |
| # that could not differ from the last one. It was also the worst kind of job to spend that on: a | |
| # red x that fires on a large share of webapp pull requests, is allowed to fail, and gates | |
| # nothing, which is the shape people learn to scroll past. | |
| # | |
| # What this gives up is real and small. A route landing a shape no corpus entry has seen can make | |
| # a known laundering mutation start paying, and that is now caught by the nightly rather than by | |
| # the pull request that caused it. Tree drift accrues over months, so a day is the right | |
| # granularity for it; the tool's own regressions, which are the ones a single commit can cause, | |
| # still gate per pull request. | |
| mutation-corpus: | |
| name: 🧬 Mutation corpus | |
| needs: changes | |
| # `!cancelled()` is here for the nightly, not for tidiness. `needs` carries an implicit | |
| # success() on the job it names, and that implicit test outranks the `||` below: with a plain | |
| # condition, a `changes` job that failed or was skipped skips this one, so the nightly would | |
| # stop scanning for tree drift and report nothing about having stopped. A status-check function | |
| # in the `if` is what drops the implicit success(), so the event test below decides alone. | |
| # `!cancelled()` rather than `always()` because `cancel-in-progress` above is a real path and a | |
| # superseded run should not finish this job. | |
| # | |
| # Pull request behaviour is deliberately unchanged: on a PR a failed `changes` leaves | |
| # `needs.changes.outputs.package` empty, so the corpus still skips. The nightly is the backstop | |
| # for that, which is the same trade the paths gate already makes for routes-only pull requests. | |
| if: >- | |
| !cancelled() && | |
| (github.event_name != 'pull_request' || needs.changes.outputs.package == 'true') | |
| runs-on: warp-ubuntu-latest-x64-4x | |
| steps: | |
| - name: ⬇️ Checkout repo | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| fetch-depth: 1 | |
| persist-credentials: false | |
| - name: ⎔ Setup pnpm | |
| uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 | |
| with: | |
| version: 10.33.2 | |
| - name: ⎔ Setup node | |
| uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 | |
| with: | |
| node-version: 24.18.0 | |
| cache: "pnpm" | |
| - name: 📥 Download deps | |
| run: pnpm install --frozen-lockfile | |
| - name: 🧬 Run the corpus | |
| env: | |
| OBS_MAP_MUTATION_CORPUS: "1" | |
| run: | | |
| pnpm --filter @internal/observability-map exec vitest run \ | |
| src/mutationCorpus.test.ts --disable-console-intercept | |
| # The package's own tests are NOT run here. They gate through pr_checks.yml, which is the only | |
| # workflow the all-checks aggregate can see, so a job in this file would report a result nobody | |
| # is required to wait for. See unit-tests-observability-map.yml and the obsmap filter. | |
| report: | |
| runs-on: warp-ubuntu-latest-x64-4x | |
| # Only this job comments, so only this job gets the write. | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| # Fork PRs get a read-only token, so the comment cannot post. Skipping the job beats a red x. | |
| # The event test is what keeps this job off the nightly, which has no pull request to comment on | |
| # and only exists for the corpus job above. | |
| if: >- | |
| github.event_name == 'pull_request' && | |
| github.event.pull_request.head.repo.full_name == github.repository | |
| steps: | |
| - name: ⬇️ Checkout repo | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - name: ⎔ Setup pnpm | |
| uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 | |
| with: | |
| version: 10.33.2 | |
| - name: ⎔ Setup node | |
| uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 | |
| with: | |
| node-version: 24.18.0 | |
| cache: "pnpm" | |
| - name: 📥 Download deps | |
| run: pnpm install --frozen-lockfile | |
| # Guarded rather than allowed to fail: this job must never block a pull request. The failure | |
| # is not swallowed either, the render step below turns a missing head report into a comment | |
| # saying so, because a swallowed failure with no comment is the outcome nobody wants. | |
| - name: 🔎 Scan head | |
| run: | | |
| if pnpm --filter @internal/observability-map exec tsx src/cli.ts --json --no-write \ | |
| > /tmp/head.json.partial; then | |
| mv /tmp/head.json.partial /tmp/head.json | |
| else | |
| rm -f /tmp/head.json /tmp/head.json.partial | |
| echo "head scan failed; the comment will say the report is stale for this run" >&2 | |
| fi | |
| # base.sha, not a merge base, and two reviewers have now read that as a bug. The checkout | |
| # above is the default for a pull_request event, so the working tree is GitHub's test merge | |
| # commit, whose parents are base.sha and the PR head. The head tree therefore already contains | |
| # the base branch up to base.sha, and diffing it against base.sha is what isolates this pull | |
| # request's own work. A merge base would leave the intervening base-branch commits in the head | |
| # tree and out of the base tree, and blame the pull request for all of them. | |
| - name: 🔎 Scan base with the head's scanner | |
| run: | | |
| if git worktree add /tmp/base-tree ${{ github.event.pull_request.base.sha }} \ | |
| && pnpm --filter @internal/observability-map exec tsx src/cli.ts --json --no-write \ | |
| --routes=/tmp/base-tree/apps/webapp/app/routes > /tmp/base.json; then | |
| : | |
| else | |
| echo "-" > /tmp/base.json || true | |
| echo "base scan failed or the worktree could not be added; falling back to no base" >&2 | |
| fi | |
| # Looked up before the render step because the render decision needs it: with no delta to | |
| # report, a pull request that already has a comment gets a resolved state rather than being | |
| # left with findings that no longer exist, and one that does not gets nothing at all. The | |
| # upsert step reuses the id rather than asking twice. | |
| # | |
| # On a failure that outlasts the retries, both steps below do nothing. Guessing is worse than | |
| # silence here: this step is the only thing that knows which comment to PATCH, so a guess of | |
| # "a comment exists" still reaches an upsert with no id to patch, which POSTs. That either | |
| # adds a second marker comment beside the stale one, or says "the findings an earlier push | |
| # reported are gone" on a pull request that never had findings. Worst case now is no comment | |
| # this run, which the next push fixes. | |
| - name: 🔍 Look for a comment from an earlier push | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | |
| rm -f /tmp/existing-comment-id /tmp/comment-lookup-failed | |
| found="" | |
| ok="" | |
| for attempt in 1 2 3; do | |
| if found=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ | |
| --jq '[.[] | select(.body | startswith("<!-- observability-map-report -->"))][0].id // empty'); then | |
| ok=1 | |
| break | |
| fi | |
| echo "comment lookup attempt ${attempt} failed" >&2 | |
| sleep $((attempt * 5)) | |
| done | |
| if [ -z "$ok" ]; then | |
| touch /tmp/comment-lookup-failed | |
| echo "comment lookup failed after 3 attempts; this run posts nothing" >&2 | |
| exit 0 | |
| fi | |
| # --paginate runs the jq once per page, so a marker comment on more than one page yields | |
| # one id per page. Unhandled, that puts a newline in the PATCH url and the step dies under | |
| # continue-on-error. The oldest wins: it is the one the upsert has been updating. | |
| count=$(printf '%s\n' "$found" | grep -c '[0-9]' || true) | |
| if [ "$count" -gt 1 ]; then | |
| echo "warning: ${count} marker comments on this pull request; updating the oldest" >&2 | |
| fi | |
| printf '%s\n' "$found" | awk 'NF { print $1; exit }' > /tmp/existing-comment-id | |
| # continue-on-error for the same reason as the scan: a rendering bug must not turn the job | |
| # red. An empty /tmp/comment.md means there is nothing to post, which is a decision | |
| # prCommentCli makes, not this shell. | |
| - name: 📝 Render comment | |
| continue-on-error: true | |
| run: | | |
| rm -f /tmp/comment.md | |
| render() { pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts "$@"; } | |
| # Every write goes through this, so a renderer that exits non-zero never leaves a 0-byte | |
| # comment.md for the upsert to skip in silence. | |
| emit() { | |
| if render "$@" > /tmp/comment.md.partial; then | |
| mv /tmp/comment.md.partial /tmp/comment.md | |
| return 0 | |
| fi | |
| rm -f /tmp/comment.md.partial | |
| return 1 | |
| } | |
| if [ -f /tmp/comment-lookup-failed ]; then | |
| echo "the comment lookup failed, so this run posts nothing" >&2 | |
| exit 0 | |
| fi | |
| if [ ! -s /tmp/head.json ]; then | |
| emit --scan-failed || echo "could not render the stale-report comment either" >&2 | |
| exit 0 | |
| fi | |
| base=/tmp/base.json | |
| if [ ! -s /tmp/base.json ] || [ "$(cat /tmp/base.json)" = "-" ]; then | |
| base="-" | |
| fi | |
| flags=() | |
| if [ -s /tmp/existing-comment-id ]; then | |
| flags=(--existing-comment) | |
| fi | |
| if ! emit /tmp/head.json "$base" "${flags[@]}"; then | |
| echo "render failed; falling back to the stale-report comment" >&2 | |
| emit --scan-failed || echo "could not render the stale-report comment either" >&2 | |
| fi | |
| # continue-on-error for the same reason: a transient gh api failure (rate limit, network) | |
| # must not fail the job either. Worst case, the PR gets no comment this run. | |
| - name: 💬 Upsert PR comment | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | |
| # The same sentinel the render step reads, so the two cannot disagree about what a failed | |
| # lookup means. Without it this step reads a missing id as "no comment exists" and POSTs. | |
| if [ -f /tmp/comment-lookup-failed ]; then | |
| echo "the comment lookup failed, so this run posts nothing" | |
| exit 0 | |
| fi | |
| if [ ! -s /tmp/comment.md ]; then | |
| echo "nothing to post: this pull request does not move the report" | |
| exit 0 | |
| fi | |
| existing="" | |
| if [ -f /tmp/existing-comment-id ]; then | |
| existing=$(cat /tmp/existing-comment-id) | |
| fi | |
| if [ -n "$existing" ]; then | |
| gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${existing}" -F body=@/tmp/comment.md | |
| else | |
| gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@/tmp/comment.md | |
| fi |