From 7c8394c35e357c8b8e1bd50ed30a66636a7d7bae Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 15 Oct 2026 16:56:25 -0400 Subject: [PATCH 1/4] fix(probe): kill the app's process group when TERM leaves it running MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Next.js's `next dev` keeps its `start-server` worker bound to the port after the parent handles SIGTERM, so stopBackgroundApp returned with the old server still answering. The next probe then booted on another port while the stale server took the probe's requests and answered 500 for routes it had never compiled — the two-product eval's oracle passed the login probe and failed every probe after it on Next.js. The stop now gives the group five seconds after TERM and then SIGKILLs it. --- src/probe.ts | 12 +++++++-- tests/probe-stop.test.ts | 55 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 tests/probe-stop.test.ts diff --git a/src/probe.ts b/src/probe.ts index 5702163..5fbc012 100644 --- a/src/probe.ts +++ b/src/probe.ts @@ -197,13 +197,21 @@ export async function readAppLogTail(exec: ProbeExecFn, lines = 40): Promise { + // SIGTERM the group, give it five seconds, then SIGKILL whatever is left. + // Dev servers that fork a worker outlive a TERM: Next.js's `start-server` + // child stayed bound to the port after `next dev` handled the signal, so + // the next probe booted on another port while the stale server answered + // the probe's requests (500s on routes it had never compiled). await exec( [ "sh", "-c", - `pid="$(cat ${APP_PID_PATH} 2>/dev/null)"; [ -z "$pid" ] || kill -TERM -- "-$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true`, + `pid="$(cat ${APP_PID_PATH} 2>/dev/null)"; [ -n "$pid" ] || exit 0; ` + + `kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true; ` + + `i=0; while [ "$i" -lt 10 ] && kill -0 -- "-$pid" 2>/dev/null; do sleep 0.5; i=$((i + 1)); done; ` + + `kill -KILL -- "-$pid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null || true`, ], - { timeoutMs: 10_000 }, + { timeoutMs: 20_000 }, ); } diff --git a/tests/probe-stop.test.ts b/tests/probe-stop.test.ts new file mode 100644 index 0000000..de60fa1 --- /dev/null +++ b/tests/probe-stop.test.ts @@ -0,0 +1,55 @@ +/** + * The probe runner's app lifecycle: a dev server that shrugs off SIGTERM + * must still be gone after stopBackgroundApp, or the next probe boots on + * another port while the stale server keeps answering. The shell contract + * is pinned offline; the behavioural case runs where `setsid` exists (Linux, + * the sandbox images) and skips elsewhere. + */ + +import { describe, expect, test } from "bun:test"; +import { APP_PID_PATH, startAppInBackground, stopBackgroundApp, type ProbeExecFn } from "../src/probe.ts"; + +const realExec: ProbeExecFn = async (cmd, opts) => { + const proc = Bun.spawn(cmd, { stdout: "pipe", stderr: "pipe" }); + const timer = setTimeout(() => proc.kill(), opts?.timeoutMs ?? 30_000); + const [stdout, stderr] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text()]); + const exitCode = await proc.exited; + clearTimeout(timer); + return { exitCode, stdout, stderr, timedOut: false }; +}; + +describe("stopBackgroundApp", () => { + test("terms the group, waits, then kills what is left — never only a polite TERM", async () => { + const commands: string[][] = []; + await stopBackgroundApp(async (cmd) => { + commands.push(cmd); + return { exitCode: 0, stdout: "", stderr: "", timedOut: false }; + }); + expect(commands).toHaveLength(1); + const script = commands[0]![2]!; + expect(script).toContain(`cat ${APP_PID_PATH}`); + expect(script).toContain('kill -TERM -- "-$pid"'); + expect(script).toContain('kill -0 -- "-$pid"'); + expect(script).toContain('kill -KILL -- "-$pid"'); + expect(script.indexOf("kill -TERM")).toBeLessThan(script.indexOf("kill -KILL")); + }); + + test.skipIf(Bun.which("setsid") === null)( + "a server that ignores SIGTERM is gone after stop", + async () => { + // A process group that traps TERM and keeps a child alive, like a dev + // server whose worker outlives the parent. + await startAppInBackground(realExec, `sh -c 'trap "" TERM; sh -c "trap \\"\\" TERM; sleep 300" & wait'`); + const pid = (await Bun.file(APP_PID_PATH).text()).trim(); + expect(pid).toMatch(/^\d+$/); + await new Promise((resolve) => setTimeout(resolve, 300)); + expect((await realExec(["sh", "-c", `kill -0 -- "-${pid}"`])).exitCode).toBe(0); + + const started = Date.now(); + await stopBackgroundApp(realExec); + expect(Date.now() - started).toBeLessThan(15_000); + expect((await realExec(["sh", "-c", `kill -0 -- "-${pid}"`])).exitCode).not.toBe(0); + }, + 30_000, + ); +}); From d0fd3f04c5a845be530d822ffa8df4e6fabb6579 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 15 Oct 2026 16:58:44 -0400 Subject: [PATCH 2/4] test(probe): start the stubborn stand-in from a script file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inline start command carried single quotes that the launcher's own sh -c '…' wrapper cut short, so the background job kept the test's stdout pipe open and the test timed out on Linux. --- tests/probe-stop.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/probe-stop.test.ts b/tests/probe-stop.test.ts index de60fa1..668495c 100644 --- a/tests/probe-stop.test.ts +++ b/tests/probe-stop.test.ts @@ -38,8 +38,11 @@ describe("stopBackgroundApp", () => { "a server that ignores SIGTERM is gone after stop", async () => { // A process group that traps TERM and keeps a child alive, like a dev - // server whose worker outlives the parent. - await startAppInBackground(realExec, `sh -c 'trap "" TERM; sh -c "trap \\"\\" TERM; sleep 300" & wait'`); + // server whose worker outlives the parent. A script file keeps the + // start command free of quotes (startAppInBackground wraps it in `sh -c '…'`). + const script = `/tmp/quickstudy-stubborn-${process.pid}.sh`; + await Bun.write(script, 'trap "" TERM\nsh -c \'trap "" TERM; sleep 300\' &\nwait\n'); + await startAppInBackground(realExec, `sh ${script}`); const pid = (await Bun.file(APP_PID_PATH).text()).trim(); expect(pid).toMatch(/^\d+$/); await new Promise((resolve) => setTimeout(resolve, 300)); From e030edadb567c521af354c35fc0d8cff801a7c13 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 15 Oct 2026 17:02:13 -0400 Subject: [PATCH 3/4] fix(probe): signal the app's group through /proc, not a negative pid dash's builtin kill rejects a negative pid ("Illegal number"), so the group signal `kill -- -` never worked on the Debian-based sandbox images: the fallback terminated the group leader alone and a dev server's worker lived on. The stop now enumerates the group from /proc and signals each member, TERM first and KILL five seconds later, keeping the negative-pid form only where /proc is absent. --- src/probe.ts | 29 +++++++++++++++++++++++------ tests/probe-stop.test.ts | 17 +++++++++++------ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/probe.ts b/src/probe.ts index 5fbc012..3bb240e 100644 --- a/src/probe.ts +++ b/src/probe.ts @@ -196,20 +196,37 @@ export async function readAppLogTail(exec: ProbeExecFn, lines = 40): Promise` only ever worked + * under bash and busybox; on the Debian-based sandbox images it failed and + * the fallback signalled the group leader alone. + */ +const GROUP_MEMBERS_SH = + 'members() { for d in /proc/[0-9]*; do [ -r "$d/stat" ] || continue; ' + + 'pg=$(sed "s/^.*) //" "$d/stat" 2>/dev/null | cut -d" " -f3); ' + + '[ "$pg" = "$pid" ] && echo "${d#/proc/}"; done; }; ' + + // Without /proc (a host runtime on macOS) fall back to the shell's group kill. + 'signal() { if [ -d /proc/self ]; then m=$(members); [ -n "$m" ] || return 1; kill -s "$1" $m 2>/dev/null; ' + + 'else kill -s "$1" -- "-$pid" 2>/dev/null; fi; }; ' + + 'alive() { if [ -d /proc/self ]; then [ -n "$(members)" ]; else kill -0 -- "-$pid" 2>/dev/null; fi; }; '; + export async function stopBackgroundApp(exec: ProbeExecFn): Promise { // SIGTERM the group, give it five seconds, then SIGKILL whatever is left. - // Dev servers that fork a worker outlive a TERM: Next.js's `start-server` - // child stayed bound to the port after `next dev` handled the signal, so - // the next probe booted on another port while the stale server answered + // Dev servers that fork a worker outlive a TERM to the leader: Next.js's + // `start-server` child stayed bound to the port after `npm run dev` exited, + // so the next probe booted on another port while the stale server answered // the probe's requests (500s on routes it had never compiled). await exec( [ "sh", "-c", `pid="$(cat ${APP_PID_PATH} 2>/dev/null)"; [ -n "$pid" ] || exit 0; ` + - `kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true; ` + - `i=0; while [ "$i" -lt 10 ] && kill -0 -- "-$pid" 2>/dev/null; do sleep 0.5; i=$((i + 1)); done; ` + - `kill -KILL -- "-$pid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null || true`, + GROUP_MEMBERS_SH + + 'signal TERM || kill -s TERM "$pid" 2>/dev/null || true; ' + + 'i=0; while [ "$i" -lt 10 ] && alive; do sleep 0.5; i=$((i + 1)); done; ' + + 'signal KILL || kill -s KILL "$pid" 2>/dev/null || true', ], { timeoutMs: 20_000 }, ); diff --git a/tests/probe-stop.test.ts b/tests/probe-stop.test.ts index 668495c..d1b9252 100644 --- a/tests/probe-stop.test.ts +++ b/tests/probe-stop.test.ts @@ -28,10 +28,12 @@ describe("stopBackgroundApp", () => { expect(commands).toHaveLength(1); const script = commands[0]![2]!; expect(script).toContain(`cat ${APP_PID_PATH}`); - expect(script).toContain('kill -TERM -- "-$pid"'); - expect(script).toContain('kill -0 -- "-$pid"'); - expect(script).toContain('kill -KILL -- "-$pid"'); - expect(script.indexOf("kill -TERM")).toBeLessThan(script.indexOf("kill -KILL")); + // The group is enumerated from /proc: dash's builtin kill cannot signal a + // negative pid, so `kill -- -` is only the no-/proc fallback. + expect(script).toContain("/proc/[0-9]*"); + expect(script).toContain("signal TERM"); + expect(script).toContain("signal KILL"); + expect(script.indexOf("signal TERM")).toBeLessThan(script.indexOf("signal KILL")); }); test.skipIf(Bun.which("setsid") === null)( @@ -46,12 +48,15 @@ describe("stopBackgroundApp", () => { const pid = (await Bun.file(APP_PID_PATH).text()).trim(); expect(pid).toMatch(/^\d+$/); await new Promise((resolve) => setTimeout(resolve, 300)); - expect((await realExec(["sh", "-c", `kill -0 -- "-${pid}"`])).exitCode).toBe(0); + // Group members by /proc, the way the stop itself finds them. + const members = async () => + (await realExec(["sh", "-c", `for d in /proc/[0-9]*; do pg=$(sed "s/^.*) //" "$d/stat" 2>/dev/null | cut -d" " -f3); [ "$pg" = "${pid}" ] && echo "\${d#/proc/}"; done`])).stdout.trim().split("\n").filter(Boolean); + expect((await members()).length).toBeGreaterThanOrEqual(2); const started = Date.now(); await stopBackgroundApp(realExec); expect(Date.now() - started).toBeLessThan(15_000); - expect((await realExec(["sh", "-c", `kill -0 -- "-${pid}"`])).exitCode).not.toBe(0); + expect(await members()).toEqual([]); }, 30_000, ); From 78323bcf5e399d419b0368ad8ae3e89f254efe75 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 15 Oct 2026 17:06:38 -0400 Subject: [PATCH 4/4] test(probe): clean up the stubborn group and settle before asserting The behavioural test now kills its own group and removes its script in a finally block, polls up to five seconds for the group to vanish after SIGKILL instead of asserting at once, and both it and the stop ignore zombies: a killed process can sit unreaped in /proc for a moment, and an unreaped process is not a running server. --- src/probe.ts | 6 ++++-- tests/probe-stop.test.ts | 44 ++++++++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/probe.ts b/src/probe.ts index 3bb240e..f1b0806 100644 --- a/src/probe.ts +++ b/src/probe.ts @@ -204,9 +204,11 @@ export async function readAppLogTail(exec: ProbeExecFn, lines = 40): Promise/stat reads + // " …"; a zombie (state Z) is dead and only unreaped. 'members() { for d in /proc/[0-9]*; do [ -r "$d/stat" ] || continue; ' + - 'pg=$(sed "s/^.*) //" "$d/stat" 2>/dev/null | cut -d" " -f3); ' + - '[ "$pg" = "$pid" ] && echo "${d#/proc/}"; done; }; ' + + 'set -- $(sed "s/^.*) //" "$d/stat" 2>/dev/null); ' + + '[ "$3" = "$pid" ] && [ "$1" != "Z" ] && echo "${d#/proc/}"; done; }; ' + // Without /proc (a host runtime on macOS) fall back to the shell's group kill. 'signal() { if [ -d /proc/self ]; then m=$(members); [ -n "$m" ] || return 1; kill -s "$1" $m 2>/dev/null; ' + 'else kill -s "$1" -- "-$pid" 2>/dev/null; fi; }; ' + diff --git a/tests/probe-stop.test.ts b/tests/probe-stop.test.ts index d1b9252..c765015 100644 --- a/tests/probe-stop.test.ts +++ b/tests/probe-stop.test.ts @@ -46,17 +46,43 @@ describe("stopBackgroundApp", () => { await Bun.write(script, 'trap "" TERM\nsh -c \'trap "" TERM; sleep 300\' &\nwait\n'); await startAppInBackground(realExec, `sh ${script}`); const pid = (await Bun.file(APP_PID_PATH).text()).trim(); - expect(pid).toMatch(/^\d+$/); - await new Promise((resolve) => setTimeout(resolve, 300)); - // Group members by /proc, the way the stop itself finds them. + // Live group members by /proc, the way the stop itself finds them (zombies excluded). const members = async () => - (await realExec(["sh", "-c", `for d in /proc/[0-9]*; do pg=$(sed "s/^.*) //" "$d/stat" 2>/dev/null | cut -d" " -f3); [ "$pg" = "${pid}" ] && echo "\${d#/proc/}"; done`])).stdout.trim().split("\n").filter(Boolean); - expect((await members()).length).toBeGreaterThanOrEqual(2); + ( + await realExec([ + "sh", + "-c", + `for d in /proc/[0-9]*; do set -- $(sed "s/^.*) //" "$d/stat" 2>/dev/null); [ "$3" = "${pid}" ] && [ "$1" != "Z" ] && echo "\${d#/proc/}"; done; true`, + ]) + ).stdout + .trim() + .split("\n") + .filter(Boolean); + const settle = async (): Promise => { + const deadline = Date.now() + 5_000; + let left = await members(); + while (left.length > 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 200)); + left = await members(); + } + return left; + }; + try { + expect(pid).toMatch(/^\d+$/); + await new Promise((resolve) => setTimeout(resolve, 300)); + expect((await members()).length).toBeGreaterThanOrEqual(2); - const started = Date.now(); - await stopBackgroundApp(realExec); - expect(Date.now() - started).toBeLessThan(15_000); - expect(await members()).toEqual([]); + const started = Date.now(); + await stopBackgroundApp(realExec); + expect(Date.now() - started).toBeLessThan(15_000); + // SIGKILL does not wait for the exit to land; give the kernel a moment. + expect(await settle()).toEqual([]); + } finally { + // Whatever the assertions did, leave no stubborn group or script behind. + const left = await members(); + if (left.length > 0) await realExec(["sh", "-c", `kill -s KILL ${left.join(" ")} 2>/dev/null; true`]); + await realExec(["rm", "-f", script, APP_PID_PATH]); + } }, 30_000, );