From 2bd2d6bb1b720a2bf57eedd109bd66ad70ec6aed Mon Sep 17 00:00:00 2001 From: os-bill Date: Tue, 8 Sep 2026 16:13:17 +0000 Subject: [PATCH 1/3] ci: smoke-check a local preview of the packaged Worker before merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 09-04 outage was detected in production. #269 built detection and recovery around the deploy, but the only environment in which a rendering defect is DETECTED is still production, and the chain `deploy succeeds -> smoke fails -> rollback fires` has never executed end to end. Runs `.github/scripts/smoke-docs.mjs` — the same script `deploy-docs.yml` runs against the live site, unmodified — with `--base` pointing at a local `opennextjs-cloudflare preview` of the `.open-next` package the `build` job already produces on every pull request since #262. No second build, no Cloudflare credentials. Readiness is asserted from wrangler's own `Ready on` line before anything is judged, because "no findings" from a server that never started is indistinguishable from a rendered site. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ChPQM8jamxLUfUAxwFpJ8S --- .github/workflows/ci.yml | 152 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ab69c3..84a4b1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -312,6 +312,158 @@ jobs: if-no-files-found: error retention-days: 3 + # #274. The 09-04 outage was found by a human looking at the live site. + # #269 answered that with detection and recovery — the deploy is gated on + # CI, the published artifact is the one CI tested, the live site is + # smoke-checked after deploying, and a bad deploy auto-rolls back — but + # the only environment in which a rendering defect is DETECTED is still + # production, and the chain `deploy succeeds -> smoke fails -> rollback + # fires` has never once executed end to end. A check that fires before + # merge costs a red pull request; the same check firing after merge costs + # a live outage plus a recovery path nobody has ever seen run. + # + # ## The same script, pointed at a different base + # + # `.github/scripts/smoke-docs.mjs` is the post-deploy check + # `deploy-docs.yml` runs against `https://docs.objectos.ai`. It is + # invoked here unmodified, with `--base` pointing at a local preview. + # NOT a second implementation of "does the site render": two copies of + # those rules drift, and the copy that drifts is the one nobody watches. + # Its live negative control — a `/docs/` slug no page claims, which must + # produce findings or the run fails on `negative-control-passed` — comes + # along with it, which is what makes a green here worth reading. + # + # ## This is not a second build + # + # `opennextjs-cloudflare preview` does not build. It populates the + # incremental cache (for this app, a copy of `.open-next/cache` into the + # Workers static assets) and then runs `wrangler dev` on the `.open-next` + # package the step above produced with `--skipNextBuild`. Since #262 + # removed the `main`-only condition from that packaging step, that + # package exists on every pull request, so this step adds a preview boot + # and four fetches and nothing else. Measured in this repo's container, + # against the package already sitting in the tree: `Ready on` at 41 s, + # the smoke run itself 1 s. No `opennextjs-cloudflare build`, no + # `next build`, no Cloudflare credentials — `wrangler dev` serves the + # Worker locally under real workerd. + # + # ## Why it runs LAST, after the artifact upload + # + # `preview` copies `.open-next/cache` into `.open-next/assets/cdn-cgi`, + # which for this app is 268 MB: measured, `.open-next` goes from 386 MB + # to 653 MB the moment the preview boots. `opennextjs-cloudflare deploy` + # makes that same copy in the deploy job from `.open-next/cache`, which + # the artifact already carries — so running this before the upload would + # add 268 MB to every `main` artifact, both ways across the wire, to + # ship a copy the deploy remakes anyway. Placed here, the uploaded bundle + # is byte-for-byte what it was before this step existed, and the gating + # is unchanged: `deploy-docs` needs the whole `build` job, so a red here + # keeps a bad render off production just as a red anywhere above it does. + # + # ## A dead server must not read as a pass + # + # "No findings" from a preview that never started is indistinguishable + # from "the site renders", and this lane logged four probes of exactly + # that shape in a single day. So readiness is asserted from wrangler's + # own `Ready on` line before anything is judged, with the preview log + # printed into the step summary when it does not arrive, and the step + # exits 1 rather than reporting a measurement it never took. + # + # `setsid` is load-bearing, not tidiness. The preview is a chain of six + # processes — pnpm, node, sh, pnpm, wrangler, workerd — and a SIGTERM to + # the pnpm wrapper at the top leaves workerd running and holding the + # port. Measured here: the cleanup `wait` never returned and the whole + # thing hung. Starting it in its own process group lets the trap signal + # the GROUP and take the chain with it; the `$$` comparison is there so + # that a `setsid` which did not take effect can never turn that into the + # step killing itself. + - name: The docs site renders — smoke-check a local preview + working-directory: apps/docs + shell: bash + env: + PREVIEW_PORT: '8792' + # Boot budget for the preview. Measured at 41 s in this repo's + # container; the margin is for a cold runner, and overrunning it is + # a finding (NOT MEASURED), never a skip. + PREVIEW_READY_TIMEOUT_S: '180' + WRANGLER_SEND_METRICS: 'false' + run: | + set -euo pipefail + + BASE="http://127.0.0.1:${PREVIEW_PORT}" + PREVIEW_LOG="$RUNNER_TEMP/preview.log" + SMOKE_LOG="$RUNNER_TEMP/smoke.log" + : > "$PREVIEW_LOG" + + setsid pnpm exec opennextjs-cloudflare preview -- \ + --port "$PREVIEW_PORT" --ip 127.0.0.1 > "$PREVIEW_LOG" 2>&1 & + PREVIEW_PID=$! + PREVIEW_PGID="$(ps -o pgid= -p "$PREVIEW_PID" | tr -d ' ' || true)" + SELF_PGID="$(ps -o pgid= -p $$ | tr -d ' ')" + + cleanup() { + if [ -n "${PREVIEW_PGID:-}" ] && [ "$PREVIEW_PGID" != "$SELF_PGID" ]; then + kill -TERM "-$PREVIEW_PGID" 2>/dev/null || true + sleep 2 + kill -KILL "-$PREVIEW_PGID" 2>/dev/null || true + else + kill -TERM "$PREVIEW_PID" 2>/dev/null || true + fi + } + trap cleanup EXIT + + READY=0 + for _ in $(seq 1 "$PREVIEW_READY_TIMEOUT_S"); do + if grep -q 'Ready on http' "$PREVIEW_LOG"; then READY=1; break; fi + if ! kill -0 "$PREVIEW_PID" 2>/dev/null; then break; fi + sleep 1 + done + + if [ "$READY" -ne 1 ]; then + { + echo "### Pre-merge render check — NOT MEASURED" + echo + echo "The local preview never printed \`Ready on\` within ${PREVIEW_READY_TIMEOUT_S}s, so" + echo "nothing was checked. Failing rather than passing: \"no findings\" from a server" + echo "that never started is indistinguishable from a rendered site." + echo + echo '```' + tail -n 40 "$PREVIEW_LOG" + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + echo "::error::Local preview never became ready — the render check measured nothing." + exit 1 + fi + + echo "preview ready: $(grep -m1 'Ready on http' "$PREVIEW_LOG" || true)" + + # Exit code captured before anything pipes it. `cmd | tee` hands back + # tee's status, and this step's verdict is the script's. + set +e + node "$GITHUB_WORKSPACE/.github/scripts/smoke-docs.mjs" --base "$BASE" \ + > "$SMOKE_LOG" 2>&1 + SMOKE_EXIT=$? + set -e + + cat "$SMOKE_LOG" + { + if [ "$SMOKE_EXIT" -eq 0 ]; then + echo "### Pre-merge render check — the site renders" + else + echo "### Pre-merge render check — FINDINGS" + fi + echo + echo "Ran \`.github/scripts/smoke-docs.mjs\` — the same script \`deploy-docs.yml\` runs" + echo "against the live site — against a local \`opennextjs-cloudflare preview\` of the" + echo "Worker this job packaged, at \`$BASE\`." + echo + echo '```' + cat "$SMOKE_LOG" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + exit "$SMOKE_EXIT" + # Defect 1 of #269: `deploy-docs.yml` used to hang off `push: branches: # [main]` exactly as this workflow does, so the two ran in PARALLEL and a # commit that failed any gate above still deployed. There was no `needs:` and From 8266e6bbec12dbae15257163bd99f5f79e66f36c Mon Sep 17 00:00:00 2001 From: os-bill Date: Tue, 8 Sep 2026 16:18:15 +0000 Subject: [PATCH 2/3] ci: name which way the preview failed when the render check is NOT MEASURED MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A crashed preview and an exhausted boot budget call for different fixes — a broken bundle versus a slow runner — and the log printed underneath is the same either way. Both demonstrated: with the budget cut to 5s the step reports the budget ran out; with the compiled OpenNext config removed it reports the process exited. Both exit 1. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ChPQM8jamxLUfUAxwFpJ8S --- .github/workflows/ci.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84a4b1f..161cbe3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -420,18 +420,27 @@ jobs: done if [ "$READY" -ne 1 ]; then + # Which of the two shapes it was. They call for different fixes — + # a crashed preview is a broken bundle, an exhausted budget is a + # slow runner — and the log below is the same either way, so the + # sentence has to say which one the reader is looking at. + if kill -0 "$PREVIEW_PID" 2>/dev/null; then + WHY="the ${PREVIEW_READY_TIMEOUT_S}s boot budget ran out with the preview still starting" + else + WHY="the preview process exited before it was ready" + fi { echo "### Pre-merge render check — NOT MEASURED" echo - echo "The local preview never printed \`Ready on\` within ${PREVIEW_READY_TIMEOUT_S}s, so" - echo "nothing was checked. Failing rather than passing: \"no findings\" from a server" - echo "that never started is indistinguishable from a rendered site." + echo "The local preview never printed \`Ready on\`: ${WHY}. Nothing was checked." + echo "Failing rather than passing: \"no findings\" from a server that never started" + echo "is indistinguishable from a rendered site." echo echo '```' tail -n 40 "$PREVIEW_LOG" echo '```' } | tee -a "$GITHUB_STEP_SUMMARY" - echo "::error::Local preview never became ready — the render check measured nothing." + echo "::error::Local preview never became ready (${WHY}) — the render check measured nothing." exit 1 fi From 815e881323b9319f59443448cd37333ea88883aa Mon Sep 17 00:00:00 2001 From: os-bill Date: Tue, 8 Sep 2026 16:25:24 +0000 Subject: [PATCH 3/3] ci: resolve the preview's process group at kill time, not at launch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `setsid` only moves the child into its own group once the forked child has exec'd it, so reading the group with `ps` immediately after `&` is a race that can still return the step's own group — and the guard that stops the step from signalling itself then falls back to signalling the pnpm wrapper alone, which ignores SIGTERM. Measured on run 34250422860: the step went green and the runner's own orphan sweeper had to terminate esbuild and two workerd processes after the job. The identical code cleaned up completely in this repo's container on every run, which is how a race presents. Resolving the group inside the cleanup function removes the window. A preview that genuinely shares the step's group now says so as a warning instead of leaving a silent orphan, and the ready line records both group ids. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ChPQM8jamxLUfUAxwFpJ8S --- .github/workflows/ci.yml | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 161cbe3..08faeb5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -398,16 +398,36 @@ jobs: setsid pnpm exec opennextjs-cloudflare preview -- \ --port "$PREVIEW_PORT" --ip 127.0.0.1 > "$PREVIEW_LOG" 2>&1 & PREVIEW_PID=$! - PREVIEW_PGID="$(ps -o pgid= -p "$PREVIEW_PID" | tr -d ' ' || true)" SELF_PGID="$(ps -o pgid= -p $$ | tr -d ' ')" + # The preview's process group is read HERE, at kill time, and never + # cached at launch. `setsid` only changes the group once the forked + # child has exec'd it, so a `ps` issued straight after `&` is a race + # that can still see the STEP's own group. Measured on a GitHub + # runner, run 34250422860: the cached read lost that race, the guard + # below fell back to signalling the pnpm wrapper alone, and the + # runner's own orphan sweeper had to terminate esbuild and two + # workerd processes after the job. The identical code cleaned up + # correctly in this repo's container every time — which is exactly + # how a race presents, and why the group is resolved at use. cleanup() { - if [ -n "${PREVIEW_PGID:-}" ] && [ "$PREVIEW_PGID" != "$SELF_PGID" ]; then + PREVIEW_PGID="$(ps -o pgid= -p "$PREVIEW_PID" 2>/dev/null | tr -d ' ' || true)" + if [ -z "$PREVIEW_PGID" ]; then + : # already gone — nothing to signal + elif [ "$PREVIEW_PGID" != "$SELF_PGID" ]; then kill -TERM "-$PREVIEW_PGID" 2>/dev/null || true - sleep 2 + for _ in 1 2 3 4 5; do + pgrep -g "$PREVIEW_PGID" >/dev/null 2>&1 || break + sleep 1 + done kill -KILL "-$PREVIEW_PGID" 2>/dev/null || true else + # `setsid` did not take effect and the preview is sharing this + # step's group, which must NEVER be signalled as a group or the + # step kills itself. Signal the pid and say so out loud rather + # than leaving a silent orphan for the runner to sweep. kill -TERM "$PREVIEW_PID" 2>/dev/null || true + echo "::warning::preview pid $PREVIEW_PID is in this step's own process group — signalled the pid alone, its descendants may survive." fi } trap cleanup EXIT @@ -445,6 +465,7 @@ jobs: fi echo "preview ready: $(grep -m1 'Ready on http' "$PREVIEW_LOG" || true)" + echo "preview pid $PREVIEW_PID in process group $(ps -o pgid= -p "$PREVIEW_PID" | tr -d ' '), step in $SELF_PGID" # Exit code captured before anything pipes it. `cmd | tee` hands back # tee's status, and this step's verdict is the script's.