diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 60b026e5f..ba69744d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,6 +43,7 @@ The preview opens under `/docs/`. Read [Documentation contributions](docs/contri - **TypeScript must be erasable**, because Node strips the types at runtime. That means no `enum`, `namespace`, parameter properties or decorators. Relative imports include the `.ts` extension. - **Tests sit next to the code** as `*.test.ts` and test behaviour through the public interface, with injected fakes rather than mocks. - **A test waits for the event that shows the state it needs, never for the clock,** and the one-minute ceiling in `vitest.shared.ts` only guards against a hang. +- **A test removes what it makes.** Each run of a package's tests works in a temporary folder of its own, which `tmpdir()` answers in every worker, and its teardown fails the run, naming what a test left there, before it removes the folder. ## Commits and pull requests diff --git a/packages/identity/src/keys/api-keys-setting.test.ts b/packages/identity/src/keys/api-keys-setting.test.ts index c45a16a03..391b6cefc 100644 --- a/packages/identity/src/keys/api-keys-setting.test.ts +++ b/packages/identity/src/keys/api-keys-setting.test.ts @@ -1,9 +1,9 @@ import { spawnSync } from 'node:child_process'; -import { mkdtempSync, writeFileSync } from 'node:fs'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, onTestFinished } from 'vitest'; import { InvalidApiKeysError, createApiKey, readApiKeys, type ApiKey } from '../index.ts'; @@ -59,6 +59,9 @@ describe('readApiKeys', () => { it('reads an unquoted compact JSON line from an env file unchanged', () => { const directory = mkdtempSync(join(tmpdir(), 'api-keys-')); + onTestFinished(() => { + rmSync(directory, { recursive: true, force: true }); + }); const line = JSON.stringify([entry, everything]); writeFileSync(join(directory, 'keys.env'), `API_KEYS=${line}\n`); diff --git a/packages/operations/README.md b/packages/operations/README.md index 9b76ed70c..97a3f1deb 100644 --- a/packages/operations/README.md +++ b/packages/operations/README.md @@ -215,3 +215,5 @@ The digest is Node's, behind the one module of the repository for it: the digest `recordedContentBehaviour(entry)` is the behaviour suite of the content port, in `src/content-testing`: a content of no bytes to 64 MiB kept and streamed back, a text kept once with its first media type in its playbook alone, a content put again whole or with its digest, the refusals of a wrong digest, a wrong size and a stream that fails, wrong bytes put with the digest of a content held, a wrong put racing a right one, and staging, read through a second client, ended, refused and raced. An adapter runs it with a `ContentEntry` whose `aStore()` answers `{ content, elsewhere, unnamedLeft }`: the port, the port of a second client of the same store, and how many pieces of puts no kept content names the store holds, which is 0 after every test. This package runs it on `memoryRecordedContent`, the ledger on SQLite and PostgreSQL, and `@on.auto/object-storage` on its fake bucket and on a real S3-compatible server. The samples it is made of are exported with it: `generated(bytes, seed)` and `generatedStream`, bytes made the same for a seed, `answerOfAbout(bytes)`, a JSON answer of about that size, `readBack` and `readThrough`, which digest what a read streams, `deliveredBeforeFailing(stream)`, how many bytes a failing stream handed over, `brokenAfter(bytes)` and `unread()`, streams that fail or are never read, and `resultOf`, `refusalOf` and `present`, which run an effect for its value, its failure or its value present. `spawnForTheTest(command, args, options)` (`src/process-testing/test-processes.ts`) is how a test of the repository starts a process it runs itself. It starts it as `spawn` of `node:child_process` does and registers its end with `onTestFinished` of Vitest, the hook that runs once the test finished, whether it passed, failed or timed out: a process still there is sent SIGTERM, on which the server stops, and SIGKILL if it is still there `stoppedWithinMs`, 10 s, later, a bound the server's test of its longest stop holds that stop under, and well within the 60 s a hook may take. `endWithTheTest(pid)` ends in the same way, by its pid, a process the test did not start itself, such as a server started in the background, and signals no pid that `kill(pid, 0)` finds gone or another user's; it cannot tell a pid the system gave to another process of the same user since. `endWithTheTest(reader)` takes a `PidReader` in place of the pid, registered before the process is started: once the test finished, it reads the pid with `read()` again and again while `stillToCome()` holds, for at most `readWithinMs`, and ends the pid it read, if any. A worker of the test runner that dies runs no hook, so it ends nothing. + +`aFolderOfTheTestRun(runningIn)` (`src/process-testing/test-run-folder.ts`) is the global setup of the tests of every package, which `vitest.shared.ts` names with its default, `aFolderOfTheTestRun()`. Each run makes a folder of its own in the temporary folder, `auto-runtime-test-run--`, and sets `TMPDIR` to it before any worker starts, so `tmpdir()` of `node:os` answers it in every worker and in every process a test starts with the environment it inherits, and two runs at once have two folders. Its teardown, which a worker that dies, a hook that times out or a child that is killed does not keep from running, lists what the tests left there, ends each `RunningProcess` that `runningIn(entry)` finds still running in an entry, with SIGTERM and with SIGKILL if `isStillRunning()` still holds `stoppedWithinMs` later, sending a signal only while it holds, fails the run naming each entry and each process it ended, and removes the folder whatever happened. A package whose own processes may outlive their test gives it a `runningIn` of its own, as the server does for the runtimes its tests start in the background, through `@on.auto/operations/test-run-folder`. That is an entry of its own because a global setup runs in Vitest's own process and not in a worker, so neither it nor anything it imports takes Vitest, with whose hooks `@on.auto/operations/testing` registers what it ends; `endedWhileItRuns(running)` of `@on.auto/operations/testing` is the same end for the hook of a test, over the same `RunningProcess`. diff --git a/packages/operations/package.json b/packages/operations/package.json index e15609621..41a1650e9 100644 --- a/packages/operations/package.json +++ b/packages/operations/package.json @@ -6,6 +6,7 @@ "type": "module", "exports": { ".": "./src/index.ts", + "./test-run-folder": "./src/process-testing/test-run-folder.ts", "./testing": "./src/testing/index.ts" }, "scripts": { diff --git a/packages/operations/src/process-testing/process-endings.ts b/packages/operations/src/process-testing/process-endings.ts new file mode 100644 index 000000000..427a0c2d3 --- /dev/null +++ b/packages/operations/src/process-testing/process-endings.ts @@ -0,0 +1,48 @@ +import { setTimeout } from 'node:timers/promises'; + +export interface Ending { + readonly isGone: () => boolean; + readonly signal: (name: NodeJS.Signals) => unknown; +} + +export interface RunningProcess { + readonly name: string; + readonly pid: number; + readonly isStillRunning: () => boolean; +} + +export const stoppedWithinMs = 10_000; + +export const lookEveryMs = 50; + +export function signalled(pid: number, signal: NodeJS.Signals | 0): boolean { + try { + process.kill(pid, signal); + return true; + } catch { + return false; + } +} + +async function goneBy(ending: Ending, deadline: number): Promise { + if (ending.isGone() || performance.now() >= deadline) { + return ending.isGone(); + } + await setTimeout(lookEveryMs); + return goneBy(ending, deadline); +} + +export async function endedWithin(ending: Ending, withinMs: number): Promise { + if (ending.isGone()) { + return; + } + ending.signal('SIGTERM'); + if (!(await goneBy(ending, performance.now() + withinMs))) { + ending.signal('SIGKILL'); + await goneBy(ending, Number.POSITIVE_INFINITY); + } +} + +export function endedWhileItRuns({ pid, isStillRunning }: RunningProcess): Promise { + return endedWithin({ isGone: () => !isStillRunning(), signal: (name) => signalled(pid, name) }, stoppedWithinMs); +} diff --git a/packages/operations/src/process-testing/test-processes.test.ts b/packages/operations/src/process-testing/test-processes.test.ts index 0ef4c2895..d33168882 100644 --- a/packages/operations/src/process-testing/test-processes.test.ts +++ b/packages/operations/src/process-testing/test-processes.test.ts @@ -5,7 +5,8 @@ import { setTimeout } from 'node:timers/promises'; import { describe, expect, it, onTestFinished } from 'vitest'; -import { endWithTheTest, spawnForTheTest, stoppedWithinMs, type PidReader, type TestFinish } from './test-processes.ts'; +import { stoppedWithinMs } from './process-endings.ts'; +import { endWithTheTest, spawnForTheTest, type PidReader, type TestFinish } from './test-processes.ts'; const stopsWhenTold = [ "process.on('SIGTERM', () => { process.stdout.write('stopped'); process.exit(0); });", diff --git a/packages/operations/src/process-testing/test-processes.ts b/packages/operations/src/process-testing/test-processes.ts index e77d0938f..323dcf062 100644 --- a/packages/operations/src/process-testing/test-processes.ts +++ b/packages/operations/src/process-testing/test-processes.ts @@ -3,6 +3,8 @@ import { setTimeout } from 'node:timers/promises'; import { onTestFinished } from 'vitest'; +import { endedWithin, lookEveryMs, signalled, stoppedWithinMs, type Ending } from './process-endings.ts'; + export interface TestProcessOptions { readonly cwd?: string; readonly env?: Readonly>; @@ -23,15 +25,6 @@ export interface PidReader { readonly readWithinMs: number; } -interface Ending { - readonly isGone: () => boolean; - readonly signal: (name: NodeJS.Signals) => unknown; -} - -export const stoppedWithinMs = 10_000; - -const lookEveryMs = 50; - const theTestsFinish: TestFinish = { registered: (end) => { onTestFinished(end); @@ -39,34 +32,6 @@ const theTestsFinish: TestFinish = { stoppedWithinMs, }; -function signalled(pid: number, signal: NodeJS.Signals | 0): boolean { - try { - process.kill(pid, signal); - return true; - } catch { - return false; - } -} - -async function goneBy(ending: Ending, deadline: number): Promise { - if (ending.isGone() || performance.now() >= deadline) { - return ending.isGone(); - } - await setTimeout(lookEveryMs); - return goneBy(ending, deadline); -} - -async function ended(ending: Ending, withinMs: number): Promise { - if (ending.isGone()) { - return; - } - ending.signal('SIGTERM'); - if (!(await goneBy(ending, performance.now() + withinMs))) { - ending.signal('SIGKILL'); - await goneBy(ending, Number.POSITIVE_INFINITY); - } -} - function endingOfPid(pid: number): Ending { return { isGone: () => !signalled(pid, 0), signal: (name) => signalled(pid, name) }; } @@ -90,7 +55,7 @@ export function endWithTheTest(pidOrReader: number | PidReader, finish: TestFini finish.registered(async () => { const pid = await pidOf(pidOrReader); if (pid !== undefined) { - await ended(endingOfPid(pid), finish.stoppedWithinMs); + await endedWithin(endingOfPid(pid), finish.stoppedWithinMs); } }); } @@ -114,6 +79,6 @@ export function spawnForTheTest( isGone: () => child.pid === undefined || child.exitCode !== null || child.signalCode !== null, signal: (name) => child.kill(name), }; - finish.registered(() => ended(ending, finish.stoppedWithinMs)); + finish.registered(() => endedWithin(ending, finish.stoppedWithinMs)); return child; } diff --git a/packages/operations/src/process-testing/test-run-folder.test.ts b/packages/operations/src/process-testing/test-run-folder.test.ts new file mode 100644 index 000000000..b9476e905 --- /dev/null +++ b/packages/operations/src/process-testing/test-run-folder.test.ts @@ -0,0 +1,130 @@ +import { spawnSync } from 'node:child_process'; +import { once } from 'node:events'; +import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +import type { RunningProcess } from './process-endings.ts'; +import { spawnForTheTest } from './test-processes.ts'; +import theFolderOfTheTestRun, { aFolderOfTheTestRun, type RunningIn } from './test-run-folder.ts'; + +const run = { name: 'probe' }; + +function tmpdirOfAProcessItStarts(): string { + return spawnSync(process.execPath, ['--eval', "process.stdout.write(require('node:os').tmpdir())"], { + encoding: 'utf8', + }).stdout; +} + +function leftIn(folder: string, ...names: readonly string[]): void { + for (const name of names) { + mkdirSync(join(folder, name)); + } +} + +describe('the temporary folder of a test run', () => { + it('is made in the temporary folder it is set up in, named for the run, and is the temporary folder of the run and of what it starts until its teardown removes it', async () => { + const outside = tmpdir(); + const teardown = theFolderOfTheTestRun(run); + const folder = tmpdir(); + const ofAProcessItStarts = tmpdirOfAProcessItStarts(); + + await teardown(); + + expect([dirname(folder), basename(folder).startsWith('auto-runtime-test-run-probe-')]).toEqual([outside, true]); + expect([ofAProcessItStarts, existsSync(folder), tmpdir()]).toEqual([folder, false, outside]); + }); + + it('fails the run, naming in order what its tests left there, and is removed all the same', async () => { + const teardown = theFolderOfTheTestRun(run); + const folder = tmpdir(); + leftIn(folder, 'left-second'); + writeFileSync(join(folder, 'left-first'), ''); + + await expect(teardown()).rejects.toThrow( + new Error('The tests of the run did not remove what they left in its temporary folder: left-first, left-second.'), + ); + expect(existsSync(folder)).toBe(false); + }); + + it('is its own in each of two runs set up at once, so the teardown of one names nothing the other left', async () => { + const outside = tmpdir(); + const first = theFolderOfTheTestRun(run); + const ofTheFirst = tmpdir(); + process.env['TMPDIR'] = outside; + const second = theFolderOfTheTestRun(run); + leftIn(ofTheFirst, 'left-by-the-first'); + + await second(); + + await expect(first()).rejects.toThrow(/: left-by-the-first\.$/u); + }); +}); + +const stopsWhenTold = [ + "process.on('SIGTERM', () => { process.stdout.write('stopped'); process.exit(0); });", + 'setInterval(() => {}, 1000);', + "process.stdout.write('ready ');", +].join('\n'); + +interface Probe { + readonly ready: Promise; + readonly printed: () => string; + readonly closed: Promise; + readonly runningIn: RunningIn; + readonly lookedIn: () => readonly (readonly [string, boolean])[]; +} + +function aProbeRunningIn(home: string): Probe { + const child = spawnForTheTest(process.execPath, ['--eval', stopsWhenTold], { + stdio: ['ignore', 'pipe', 'inherit'], + }); + const printed = { text: '' }; + const ready = Promise.withResolvers(); + child.stdout?.setEncoding('utf8').on('data', (chunk: string) => { + printed.text += chunk; + ready.resolve(); + }); + const running: RunningProcess = { + name: 'a probe', + pid: Number(child.pid), + isStillRunning: () => child.exitCode === null && child.signalCode === null, + }; + const lookedIn: (readonly [string, boolean])[] = []; + return { + ready: ready.promise, + printed: () => printed.text, + closed: once(child, 'close'), + runningIn: (entry) => { + lookedIn.push([basename(entry), existsSync(entry)]); + return basename(entry) === home ? [running] : []; + }, + lookedIn: () => lookedIn, + }; +} + +describe('a process still running in what the tests of a run left in its folder', () => { + it('is ended as its teardown looks, before the folder is removed, and named with what it ran in', async () => { + const probe = aProbeRunningIn('a-home'); + const teardown = aFolderOfTheTestRun(probe.runningIn)(run); + leftIn(tmpdir(), 'a-home', 'a-folder'); + await probe.ready; + + await expect(teardown()).rejects.toThrow( + new Error( + 'The tests of the run did not remove what they left in its temporary folder: a-folder, a-home. The teardown ended a probe, which still ran in a-home.', + ), + ); + + expect(await probe.closed).toEqual([0, null]); + expect([probe.printed(), probe.lookedIn()]).toEqual([ + 'ready stopped', + [ + ['a-folder', true], + ['a-home', true], + ], + ]); + }); +}); diff --git a/packages/operations/src/process-testing/test-run-folder.ts b/packages/operations/src/process-testing/test-run-folder.ts new file mode 100644 index 000000000..963d8c8ab --- /dev/null +++ b/packages/operations/src/process-testing/test-run-folder.ts @@ -0,0 +1,56 @@ +import { mkdtempSync, readdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { endedWhileItRuns, type RunningProcess } from './process-endings.ts'; + +export type RunningIn = (entry: string) => readonly RunningProcess[]; + +interface TestRun { + readonly name: string; +} + +type TestRunTeardown = () => Promise; + +interface RunningInAnEntry { + readonly entry: string; + readonly running: RunningProcess; +} + +const nothingRuns: RunningIn = () => []; + +function leftWords(left: readonly string[], ended: readonly RunningInAnEntry[]): string { + return [ + `The tests of the run did not remove what they left in its temporary folder: ${left.join(', ')}.`, + ...ended.map(({ entry, running }) => `The teardown ended ${running.name}, which still ran in ${entry}.`), + ].join(' '); +} + +async function checkedAndEnded(folder: string, runningIn: RunningIn): Promise { + const left = readdirSync(folder).toSorted(); + const ended = left.flatMap((entry) => + runningIn(join(folder, entry)).map((running): RunningInAnEntry => ({ entry, running })), + ); + await Promise.all(ended.map(({ running }) => endedWhileItRuns(running))); + if (left.length > 0) { + throw new Error(leftWords(left, ended)); + } +} + +export function aFolderOfTheTestRun(runningIn: RunningIn = nothingRuns): (run: TestRun) => TestRunTeardown { + return ({ name }) => { + const madeIn = tmpdir(); + const folder = mkdtempSync(join(madeIn, `auto-runtime-test-run-${name}-`)); + process.env['TMPDIR'] = folder; + return async () => { + try { + await checkedAndEnded(folder, runningIn); + } finally { + rmSync(folder, { recursive: true, force: true }); + process.env['TMPDIR'] = madeIn; + } + }; + }; +} + +export default aFolderOfTheTestRun(); diff --git a/packages/operations/src/testing/index.ts b/packages/operations/src/testing/index.ts index daec52a56..4fdf3817c 100644 --- a/packages/operations/src/testing/index.ts +++ b/packages/operations/src/testing/index.ts @@ -33,4 +33,5 @@ export { type Unread, } from '../content-testing/content-samples.ts'; export { internalTerms, internalTermsIn } from '../plain-language/internal-terms.ts'; -export { endWithTheTest, spawnForTheTest, stoppedWithinMs, type PidReader } from '../process-testing/test-processes.ts'; +export { endedWhileItRuns, stoppedWithinMs, type RunningProcess } from '../process-testing/process-endings.ts'; +export { endWithTheTest, spawnForTheTest, type PidReader } from '../process-testing/test-processes.ts'; diff --git a/packages/server/README.md b/packages/server/README.md index 71d1d74aa..f647af354 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -22,6 +22,6 @@ A test that waits for what a workflow's step waits for before the step's timeout The composition of main, started in the process with a shutdown timeout longer than the test, stops with an idle keep-alive connection open after a workflow ran, and while a function a workflow called still runs, without waiting for that timeout (`src/lifecycle/composed-stop.test.ts`); so no test of a spawned process holds it to how long it takes: `src/main.test.ts` holds main to its exit code alone, `src/workflows/workflow-lifecycle.test.ts` to its exit code and what it says at its start, and `src/workflows/workflow-shutdown.test.ts` to its exit code, to leaving nothing running and to taking the call it was stopped in up again once started again. -The tests start the servers, the `auto-runtime` command and the development runner they run through `spawnForTheTest` of `@on.auto/operations/testing` (`src/testing/processes/spawned-server.ts`, `runtime-command.ts` and `development-process.ts`), which ends each once its test finished, whether it passed, failed or timed out: with SIGTERM, on which the server stops, and with SIGKILL if it is still there `stoppedWithinMs`, 10 s, later; `src/lifecycle/shutdown.test.ts` holds the server's longest stop, `longestStopMs`, under that bound. Two tests start their processes through the code they test instead: `src/development/children.test.ts` through the runner's `startChild`, which starts each child in a process group of its own, and ends each with SIGKILL once its test finished, and `src/background-start/started-in-the-background.test.ts` through `startInBackgroundOf`, whose recording entry ends itself once it has written what the test reads, and whose other entry, `src/testing/entries/stay-until-signalled.ts`, runs until the test signals it through the handle the start hands back, with `endWithTheTest` registered for its pid. The processes the helper starts are attached to the test's worker, in its process group, so a signal to that group, as Ctrl+C in a terminal sends, stops them too; a worker that dies alone runs no hook, and leaves them running until they next write to the output it held. A runtime started in the background, `start --background --instance `, runs in a process group of its own and writes to its log, so nothing of the test's ends it with the test. `src/agent-relay/agent-relay.test.ts` starts one alone, in the test that proves it runs on outside the process group of the agent that started it, and starts a second one the same way once the first was stopped; another has the relay start one as it starts, and a second once the test stopped the first and the relay's grace passed; two of its other tests start the runtime in a terminal, attached, and another starts none, since another program holds its port. `src/agent-relay/relay-replacement.test.ts` starts one in place of a runtime in a terminal that the test stops, and one through the relay of `src/testing/entries/relay-with-short-bounds.ts`, which the test freezes with SIGSTOP and the relay stops and replaces; its third test freezes a runtime in a terminal, which the relay leaves alone, and lets it go on with SIGCONT before it stops it with Ctrl+C. Each test that lets a relay start a runtime registers, before anything starts, `runtimesStartedInTheBackgroundEndedWithTheTest` of `src/testing/processes/background-runtimes.ts`. Once the test finished, it reads the lines the relay writes to the `runtime.log` of the test's home as it acts, `Auto Runtime was started in the background as process , instance ` and `Auto Runtime, process , ended`, and ends each process started and not ended: with SIGTERM, and with SIGKILL if it is still there `stoppedWithinMs` later, each sent only while its command line, as `ps` reads it, still carries that instance, and the process counts as gone once it no longer does. It asks the port nothing, so a runtime that answers slower than a probe's 2 s, or one whose relay was ended while it held the starting lock, is ended all the same, and a pid used again by another process is not signalled. +The tests start the servers, the `auto-runtime` command and the development runner they run through `spawnForTheTest` of `@on.auto/operations/testing` (`src/testing/processes/spawned-server.ts`, `runtime-command.ts` and `development-process.ts`), which ends each once its test finished, whether it passed, failed or timed out: with SIGTERM, on which the server stops, and with SIGKILL if it is still there `stoppedWithinMs`, 10 s, later; `src/lifecycle/shutdown.test.ts` holds the server's longest stop, `longestStopMs`, under that bound. Two tests start their processes through the code they test instead: `src/development/children.test.ts` through the runner's `startChild`, which starts each child in a process group of its own, and ends each with SIGKILL once its test finished, and `src/background-start/started-in-the-background.test.ts` through `startInBackgroundOf`, whose recording entry ends itself once it has written what the test reads, and whose other entry, `src/testing/entries/stay-until-signalled.ts`, runs until the test signals it through the handle the start hands back, with `endWithTheTest` registered for its pid. The processes the helper starts are attached to the test's worker, in its process group, so a signal to that group, as Ctrl+C in a terminal sends, stops them too; a worker that dies alone runs no hook, and leaves them running until they next write to the output it held. A runtime started in the background, `start --background --instance `, runs in a process group of its own and writes to its log, so nothing of the test's ends it with the test. `src/agent-relay/agent-relay.test.ts` starts one alone, in the test that proves it runs on outside the process group of the agent that started it, and starts a second one the same way once the first was stopped; another has the relay start one as it starts, and a second once the test stopped the first and the relay's grace passed; two of its other tests start the runtime in a terminal, attached, and another starts none, since another program holds its port. `src/agent-relay/relay-replacement.test.ts` starts one in place of a runtime in a terminal that the test stops, and one through the relay of `src/testing/entries/relay-with-short-bounds.ts`, which the test freezes with SIGSTOP and the relay stops and replaces; its third test freezes a runtime in a terminal, which the relay leaves alone, and lets it go on with SIGCONT before it stops it with Ctrl+C. Each test that lets a relay start a runtime registers, before anything starts, `runtimesStartedInTheBackgroundEndedWithTheTest` of `src/testing/processes/background-runtimes.ts`. Once the test finished, it reads the lines the relay writes to the `runtime.log` of the test's home as it acts, `Auto Runtime was started in the background as process , instance ` and `Auto Runtime, process , ended`, and ends each process started and not ended: with SIGTERM, and with SIGKILL if it is still there `stoppedWithinMs` later, each sent only while its command line, as `ps` reads it, still carries that instance, and the process counts as gone once it no longer does. It asks the port nothing, so a runtime that answers slower than a probe's 2 s, or one whose relay was ended while it held the starting lock, is ended all the same, and a pid used again by another process is not signalled. The suite's teardown does the same with what a test left in the temporary folder of the run, as when its worker died or its hook timed out: `vitest.config.ts` names `src/testing/test-run/test-run-folder.ts` in place of the shared global setup, `aFolderOfTheTestRun` of `@on.auto/operations/test-run-folder` over `runtimesStillRunningInTheBackgroundIn` of `src/testing/processes/runtimes-in-the-background.ts`, which reads the `runtime.log` of each home left there, so a runtime still running in one is ended once the run ends and named, with its home, in the failure of the run. The relay of `auto-runtime mcp` holds its bounds on Effect's test clock, in `src/agent-relay/relay-watch.test.ts`, `src/agent-relay/relay-losses.test.ts` and `src/runtime-recovery/runtime-stop.test.ts`; `src/agent-relay/relay-replacement.test.ts` runs a relay on real time with a silence of 1 s and a stop of 0.5 s in place of 15 s and 9.1 s, and holds what happens, never how long it took. The waits of its start, for a runtime it starts to answer, for an older one to stop and for one that restarts to come back, probe the port every 100 ms on Effect's clock and end with the relay: a start the relay makes while it relays ends at once when the agent leaves, aborts the probe in flight and frees the starting lock. `src/background-start/background-start.test.ts` holds them on the test clock, by the probes its fake answers and by the time they take, and `src/local-runtime/runtime-commands.test.ts` holds the waits of `stop` and `restart`, which run the same way. Some tests keep a bound on real time on purpose. `src/lifecycle/shutdown.test.ts` holds a request still running to being cut off no sooner than the shutdown timeout less 50 ms, a lower bound a slow machine only lengthens. `src/computation/computation-deadline.test.ts` holds a run that never ends by itself to being refused no sooner than its 10 s deadline less the slack of a timer, and a read of the definition and a run on another worker to answering before that deadline, a race the server wins by the pool's 10 s on the real clock, which the pool keeps and no clock of the server reaches; its saves check within 30 s through `checkingWithin`. `src/reasoning/large-answers-on-stores.test.ts` holds a reasoning run whose 25 calls each answer 1 MB to reading every answer back within `toolBounds.runMs`, the 600 s a run that calls tools may take, on the real clock on purpose, since that is the bound such a run meets in the server. `src/telemetry/start-telemetry.test.ts` holds the shutdown of telemetry whose collector cannot be reached to within 1.5 s, since that shutdown races its exports against a timer of 1 s of the platform, for which the exporters take no clock. `src/development/source-changes.test.ts` saves a file again every 500 ms until the watcher reports it, and `src/workflows/workflow-lifecycle.test.ts` waits until the timers of a stopped server's runs are due before it starts the next process, waits a slow machine only lengthens. diff --git a/packages/server/src/agent-relay/agent-relay.test.ts b/packages/server/src/agent-relay/agent-relay.test.ts index f8bbc57d7..de154ae6e 100644 --- a/packages/server/src/agent-relay/agent-relay.test.ts +++ b/packages/server/src/agent-relay/agent-relay.test.ts @@ -1,15 +1,12 @@ import { spawnSync } from 'node:child_process'; -import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { existsSync } from 'node:fs'; import { createServer } from 'node:http'; import { join } from 'node:path'; import { describe, expect, it, onTestFinished } from 'vitest'; import { restartMarkIn } from '../local-runtime/local-control.ts'; -import { - runtimesStartedInTheBackgroundEndedWithTheTest, - startedInTheBackgroundIn, -} from '../testing/processes/background-runtimes.ts'; +import { runtimesStartedInTheBackgroundEndedWithTheTest } from '../testing/processes/background-runtimes.ts'; import { aPortNothingListensOn } from '../testing/processes/free-port.ts'; import { homeOfTheTest, @@ -219,34 +216,3 @@ describe('an agent whose runtime port another program holds', () => { expect(await relay.exited).toBe(0); }); }); - -describe('the end of the runtimes a relay started in the home of a test', () => { - it('leaves alone a process the log of that home says has ended, and one whose command is no longer the runtime started', async () => { - const home = homeOfTheTest(homePrefix); - const another = spawnRuntimeCommand([], { home: homeOfTheTest(homePrefix), port: await aPortNothingListensOn() }); - await until(() => { - expect(readiesIn(another.stdout())).toBe(1); - }); - const instance = '0199a3c4-7d2e-7c1a-9b3f-2f1e0d9c8b7a'; - mkdirSync(join(home, '.auto-runtime'), { recursive: true }); - writeFileSync( - join(home, '.auto-runtime', 'runtime.log'), - [ - `Auto Runtime was started in the background as process ${String(another.pid)}, instance ${instance}`, - `Auto Runtime was started in the background as process ${process.pid}, instance ${instance}`, - `Auto Runtime, process ${process.pid}, ended`, - '', - ].join('\n'), - ); - onTestFinished(() => { - expect(() => process.kill(Number(another.pid), 0)).not.toThrow(); - }); - - runtimesStartedInTheBackgroundEndedWithTheTest(home); - - expect(startedInTheBackgroundIn(home)).toEqual([ - { pid: another.pid, instance }, - { pid: process.pid, instance }, - ]); - }); -}); diff --git a/packages/server/src/computation/computation-over-mcp.test.ts b/packages/server/src/computation/computation-over-mcp.test.ts index fca47ee44..d980ab494 100644 --- a/packages/server/src/computation/computation-over-mcp.test.ts +++ b/packages/server/src/computation/computation-over-mcp.test.ts @@ -1,9 +1,9 @@ import { listedTools, plainTextIn, problemIn, textOf, withMcpSession, type McpSession } from '@on.auto/api/testing'; import { campaignPace, campaignRows } from '@on.auto/computation/testing'; import { internalTermsIn } from '@on.auto/operations/testing'; -import { afterEach, describe, expect, it } from 'vitest'; +import { describe, expect, it, onTestFinished } from 'vitest'; -import { servingReasoning, type ReasoningServer } from '../testing/servers/reasoning-server.ts'; +import { servingReasoning } from '../testing/servers/reasoning-server.ts'; const dated = [ '---', @@ -23,14 +23,9 @@ const total = [ '}', ].join('\n'); -let server: ReasoningServer; - -afterEach(async () => { - await server.stop(); -}); - async function onAlpha(use: (session: McpSession) => Promise): Promise { - server = await servingReasoning([]); + const server = await servingReasoning([]); + onTestFinished(server.stop); await withMcpSession('current revision', { url: `${server.origin}/orgs/acme/mcp`, headers: {} }, (session) => session.callTool('create_playbook', { playbook: 'alpha', name: 'Alpha' }), ); diff --git a/packages/server/src/testing/processes/background-runtimes.ts b/packages/server/src/testing/processes/background-runtimes.ts index f376b9471..ffa5b81ca 100644 --- a/packages/server/src/testing/processes/background-runtimes.ts +++ b/packages/server/src/testing/processes/background-runtimes.ts @@ -1,74 +1,16 @@ -import { spawnSync } from 'node:child_process'; -import { readFileSync } from 'node:fs'; -import { setTimeout } from 'node:timers/promises'; - -import { stoppedWithinMs } from '@on.auto/operations/testing'; +import { endedWhileItRuns } from '@on.auto/operations/testing'; import { onTestFinished } from 'vitest'; -import { logFileIn } from '../../local-runtime/local-control.ts'; import { until } from '../waiting/until.ts'; +import { runtimesStillRunningInTheBackgroundIn, startedInTheBackgroundIn } from './runtimes-in-the-background.ts'; export interface StartedInTheBackground { readonly pid: () => Promise; } -function logIn(home: string): string { - try { - return readFileSync(logFileIn(home), 'utf8'); - } catch { - return ''; - } -} - -interface Started { - readonly pid: number; - readonly instance: string; -} - -export function startedInTheBackgroundIn(home: string): readonly Started[] { - return [ - ...logIn(home).matchAll(/Auto Runtime was started in the background as process (\d+), instance ([\da-f-]+)/gu), - ].map(([, pid, instance]: readonly string[]) => ({ pid: Number(pid), instance: String(instance) })); -} - -function endedIn(home: string): readonly number[] { - return [...logIn(home).matchAll(/Auto Runtime, process (\d+), ended/gu)].map(([, pid]: readonly string[]) => - Number(pid), - ); -} - -const lookEveryMs = 50; - -function stillTheOneStarted({ pid, instance }: Started): boolean { - return spawnSync('ps', ['-o', 'command=', '-p', String(pid)], { encoding: 'utf8' }).stdout.includes(instance); -} - -function signalledWhileItRuns(started: Started, signal: NodeJS.Signals): void { - if (stillTheOneStarted(started)) { - process.kill(started.pid, signal); - } -} - -async function goneBy(started: Started, deadline: number): Promise { - if (!stillTheOneStarted(started) || performance.now() >= deadline) { - return; - } - await setTimeout(lookEveryMs); - await goneBy(started, deadline); -} - -async function endedWhileItRuns(started: Started): Promise { - signalledWhileItRuns(started, 'SIGTERM'); - await goneBy(started, performance.now() + stoppedWithinMs); - signalledWhileItRuns(started, 'SIGKILL'); - await goneBy(started, performance.now() + stoppedWithinMs); -} - export function runtimesStartedInTheBackgroundEndedWithTheTest(home: string): StartedInTheBackground { onTestFinished(async () => { - const ended = endedIn(home); - const living = startedInTheBackgroundIn(home).filter(({ pid }) => !ended.includes(pid)); - await Promise.all(living.map((started) => endedWhileItRuns(started))); + await Promise.all(runtimesStillRunningInTheBackgroundIn(home).map((running) => endedWhileItRuns(running))); }); const read = { starts: 0 }; return { diff --git a/packages/server/src/testing/processes/runtimes-in-the-background.ts b/packages/server/src/testing/processes/runtimes-in-the-background.ts new file mode 100644 index 000000000..8a215d74c --- /dev/null +++ b/packages/server/src/testing/processes/runtimes-in-the-background.ts @@ -0,0 +1,46 @@ +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; + +import type { RunningProcess } from '@on.auto/operations/testing'; + +import { logFileIn } from '../../local-runtime/local-control.ts'; + +interface Started { + readonly pid: number; + readonly instance: string; +} + +function logIn(home: string): string { + try { + return readFileSync(logFileIn(home), 'utf8'); + } catch { + return ''; + } +} + +export function startedInTheBackgroundIn(home: string): readonly Started[] { + return [ + ...logIn(home).matchAll(/Auto Runtime was started in the background as process (\d+), instance ([\da-f-]+)/gu), + ].map(([, pid, instance]: readonly string[]) => ({ pid: Number(pid), instance: String(instance) })); +} + +function endedIn(home: string): readonly number[] { + return [...logIn(home).matchAll(/Auto Runtime, process (\d+), ended/gu)].map(([, pid]: readonly string[]) => + Number(pid), + ); +} + +function stillTheOneStarted({ pid, instance }: Started): boolean { + return spawnSync('ps', ['-o', 'command=', '-p', String(pid)], { encoding: 'utf8' }).stdout.includes(instance); +} + +export function runtimesStillRunningInTheBackgroundIn(home: string): readonly RunningProcess[] { + const ended = endedIn(home); + return startedInTheBackgroundIn(home) + .filter((started) => !ended.includes(started.pid) && stillTheOneStarted(started)) + .map((started) => ({ + name: `Auto Runtime, process ${started.pid}, instance ${started.instance}`, + pid: started.pid, + isStillRunning: () => stillTheOneStarted(started), + })); +} diff --git a/packages/server/src/testing/test-run/test-run-folder.test.ts b/packages/server/src/testing/test-run/test-run-folder.test.ts new file mode 100644 index 000000000..77b91d7c3 --- /dev/null +++ b/packages/server/src/testing/test-run/test-run-folder.test.ts @@ -0,0 +1,74 @@ +import { existsSync, mkdirSync, mkdtempSync, realpathSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; + +import { endWithTheTest, spawnForTheTest } from '@on.auto/operations/testing'; +import { Function } from 'effect'; +import { describe, expect, it } from 'vitest'; + +import { loggedTo, startInBackgroundOf } from '../../background-start/background-process.ts'; +import { logFileIn, runtimeHomeIn } from '../../local-runtime/local-control.ts'; +import { until } from '../waiting/until.ts'; +import theFolderOfTheServersTestRun from './test-run-folder.ts'; + +const stayingEntry = join(import.meta.dirname, '../entries/stay-until-signalled.ts'); + +const run = { name: 'probe' }; + +const anInstanceThatEnded = '5c8e2a0f-3d14-4b6e-9a7c-1f2b3c4d5e6f'; + +const anInstanceNoProcessRunsAs = '9d0c1b2a-4e3f-4a5b-8c6d-7e8f9a0b1c2d'; + +const leftWords = 'The tests of the run did not remove what they left in its temporary folder:'; + +function aHomeLeftIn(folder: string): string { + return realpathSync(mkdtempSync(join(folder, 'auto-runtime-left-'))); +} + +describe("the teardown of the temporary folder of a run of the server's tests", () => { + it('ends a runtime a test left running in the background in a home it left there, and names the runtime with the home', async () => { + const teardown = theFolderOfTheServersTestRun(run); + const home = aHomeLeftIn(tmpdir()); + const started = startInBackgroundOf({ + execPath: process.execPath, + execArgv: [], + entry: stayingEntry, + home, + environment: { PATH: process.env['PATH'], NODE_V8_COVERAGE: process.env['NODE_V8_COVERAGE'] }, + report: Function.constVoid, + log: loggedTo(home, Function.constVoid), + })(); + const pid = Number(started.pid); + endWithTheTest(pid); + await until(() => existsSync(join(runtimeHomeIn(home), 'started.json')), Boolean); + + await expect(teardown()).rejects.toThrow( + new Error( + `${leftWords} ${basename(home)}. The teardown ended Auto Runtime, process ${pid}, instance ${started.instance}, which still ran in ${basename(home)}.`, + ), + ); + await started.ended; + + expect(started.endedAt()).toBeGreaterThanOrEqual(started.startedAt); + }); + + it('signals no process that a log names as ended, or that runs as another than the instance it names, and names what was left alone', async () => { + const another = spawnForTheTest(process.execPath, ['--eval', 'setInterval(() => {}, 1000);']); + const teardown = theFolderOfTheServersTestRun(run); + const home = aHomeLeftIn(tmpdir()); + mkdirSync(runtimeHomeIn(home)); + writeFileSync( + logFileIn(home), + [ + `Auto Runtime was started in the background as process ${process.pid}, instance ${anInstanceThatEnded}`, + `Auto Runtime, process ${process.pid}, ended`, + `Auto Runtime was started in the background as process ${Number(another.pid)}, instance ${anInstanceNoProcessRunsAs}`, + ].join('\n'), + ); + writeFileSync(join(tmpdir(), 'a-file-left'), ''); + + await expect(teardown()).rejects.toThrow(new Error(`${leftWords} a-file-left, ${basename(home)}.`)); + + expect([another.exitCode, another.signalCode]).toEqual([null, null]); + }); +}); diff --git a/packages/server/src/testing/test-run/test-run-folder.ts b/packages/server/src/testing/test-run/test-run-folder.ts new file mode 100644 index 000000000..85b8cbc28 --- /dev/null +++ b/packages/server/src/testing/test-run/test-run-folder.ts @@ -0,0 +1,5 @@ +import { aFolderOfTheTestRun } from '@on.auto/operations/test-run-folder'; + +import { runtimesStillRunningInTheBackgroundIn } from '../processes/runtimes-in-the-background.ts'; + +export default aFolderOfTheTestRun(runtimesStillRunningInTheBackgroundIn); diff --git a/packages/server/vitest.config.ts b/packages/server/vitest.config.ts index 7908619cd..e878eaf57 100644 --- a/packages/server/vitest.config.ts +++ b/packages/server/vitest.config.ts @@ -1,5 +1,8 @@ import { defineConfig, mergeConfig } from 'vitest/config'; -import { sharedConfig } from '../../vitest.shared.ts'; +import { sharedConfigSetUpBy } from '../../vitest.shared.ts'; -export default mergeConfig(sharedConfig, defineConfig({ test: { name: 'server' } })); +export default mergeConfig( + sharedConfigSetUpBy('src/testing/test-run/test-run-folder.ts'), + defineConfig({ test: { name: 'server' } }), +); diff --git a/turbo.json b/turbo.json index bad1476f9..2111f9fcc 100644 --- a/turbo.json +++ b/turbo.json @@ -8,6 +8,8 @@ "tsconfig.base.json", ".oxlintrc.json", "vitest.shared.ts", + "packages/operations/src/process-testing/test-run-folder.ts", + "packages/operations/src/process-testing/process-endings.ts", ".nvmrc", "pnpm-lock.yaml" ], diff --git a/vitest.shared.ts b/vitest.shared.ts index cacaaf0dd..730d7a920 100644 --- a/vitest.shared.ts +++ b/vitest.shared.ts @@ -1,19 +1,28 @@ +import { join } from 'node:path'; + import { defineConfig } from 'vitest/config'; -export const sharedConfig = defineConfig({ - test: { - testTimeout: 60_000, - hookTimeout: 60_000, - experimental: { - viteModuleRunner: false, - }, - coverage: { - provider: 'v8', - include: ['src/**/*.ts'], - exclude: ['src/**/*.test.ts'], - reporter: ['text', 'json-summary'], - autoAttachSubprocess: true, - thresholds: { 100: true, perFile: true }, +const theFolderOfTheTestRun = join(import.meta.dirname, 'packages/operations/src/process-testing/test-run-folder.ts'); + +export function sharedConfigSetUpBy(globalSetup: string) { + return defineConfig({ + test: { + globalSetup, + testTimeout: 60_000, + hookTimeout: 60_000, + experimental: { + viteModuleRunner: false, + }, + coverage: { + provider: 'v8', + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts'], + reporter: ['text', 'json-summary'], + autoAttachSubprocess: true, + thresholds: { 100: true, perFile: true }, + }, }, - }, -}); + }); +} + +export const sharedConfig = sharedConfigSetUpBy(theFolderOfTheTestRun);