diff --git a/sdk/typescript/src/multiscan.ts b/sdk/typescript/src/multiscan.ts index 534715a..a82b270 100644 --- a/sdk/typescript/src/multiscan.ts +++ b/sdk/typescript/src/multiscan.ts @@ -28,6 +28,7 @@ const REQUIRED_ARTIFACTS = [ "coverage.json", "report.md", ]; +const INCOMPLETE_LOCK_MILLISECONDS = 30_000; interface MultiscanTask { id: string; @@ -268,29 +269,71 @@ async function appendReceipt(path: string, receipt: string): Promise { async function acquireLock(output: string): Promise<() => Promise> { const path = join(output, ".lock"); const ownerPath = join(path, "owner.json"); - try { - await mkdir(path, { mode: 0o700 }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - const { pid } = JSON.parse(await readFile(ownerPath, "utf8")) as { - pid: number; - }; + for (;;) { + try { + await mkdir(path, { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + if (!(await recoverStaleLock(output, path, ownerPath))) { + throw new Error("A multiscan supervisor is already running."); + } + continue; + } + await writeFile(ownerPath, `${JSON.stringify({ pid: process.pid })}\n`, { + flag: "wx", + mode: 0o600, + }); + return async () => rm(path, { recursive: true }); + } +} + +async function recoverStaleLock( + output: string, + path: string, + ownerPath: string, +): Promise { + const metadata = await lstat(path).catch((error: unknown) => { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + }); + if (metadata === null) return true; + + let owner: unknown; + if (metadata.isDirectory()) { try { - process.kill(pid, 0); - throw new Error("A multiscan supervisor is already running."); + owner = JSON.parse(await readFile(ownerPath, "utf8")); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "ENOENT" && !(error instanceof SyntaxError)) throw error; + } + } + + if (isRecord(owner) && typeof owner["pid"] === "number") { + try { + process.kill(owner["pid"], 0); + return false; } catch (failure) { - if ((failure as NodeJS.ErrnoException).code !== "ESRCH") throw failure; + const code = (failure as NodeJS.ErrnoException).code; + if (code === "EPERM") return false; + if (code !== "ESRCH") throw failure; } - const stale = join(output, `.lock.stale-${randomUUID()}`); + } else if (Date.now() - metadata.mtimeMs < INCOMPLETE_LOCK_MILLISECONDS) { + return false; + } + + const stale = join(output, `.lock.stale-${randomUUID()}`); + try { await rename(path, stale); - await rm(stale, { recursive: true }); - return await acquireLock(output); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return true; + throw error; } - await writeFile(ownerPath, `${JSON.stringify({ pid: process.pid })}\n`, { - flag: "wx", - mode: 0o600, - }); - return async () => rm(path, { recursive: true }); + await rm(stale, { recursive: true, force: true }); + return true; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); } async function ensureManifest( diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.ts index f357fac..7b7ed3e 100644 --- a/sdk/typescript/tests-ts/multiscan.test.ts +++ b/sdk/typescript/tests-ts/multiscan.test.ts @@ -8,6 +8,7 @@ import { readdir, rm, symlink, + utimes, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -440,6 +441,59 @@ describe("multiscan", () => { await expect(access(lock)).rejects.toThrow(); }); + test("recovers locks with missing or malformed owner metadata", async () => { + const paths = await fixture(); + const source = await repository(paths.root, "lock-recovery"); + await writeFile( + paths.input, + `id,repository,revision\nlock-recovery,${source.path},${source.revision}\n`, + ); + await mkdir(paths.output); + const lock = join(paths.output, ".lock"); + const aged = new Date(Date.now() - 2 * 60_000); + let scans = 0; + const security = client(async (_repository, scanOptions = {}) => { + scans += 1; + return await completedScan(scanOptions.outputDir!); + }); + + for (const owner of [undefined, '{"pid":', '{"pid":"invalid"}']) { + await mkdir(lock); + if (owner !== undefined) { + await writeFile(join(lock, "owner.json"), owner); + } + await utimes(lock, aged, aged); + const result = await runMultiscan(options(paths, security)); + expect(result).toMatchObject({ completed: 1, failed: 0 }); + await expect(access(lock)).rejects.toThrow(); + } + + expect(scans).toBe(1); + expect( + (await readdir(paths.output)).some((name) => + name.startsWith(".lock.stale-"), + ), + ).toBe(false); + }); + + test("treats a fresh lock without owner metadata as an active supervisor", async () => { + const paths = await fixture(); + const source = await repository(paths.root, "lock-fresh"); + await writeFile( + paths.input, + `id,repository,revision\nlock-fresh,${source.path},${source.revision}\n`, + ); + await mkdir(paths.output); + await mkdir(join(paths.output, ".lock")); + const security = client(async (_repository, scanOptions = {}) => { + return await completedScan(scanOptions.outputDir!); + }); + + await expect(runMultiscan(options(paths, security))).rejects.toThrow( + /running|locked|supervisor/iu, + ); + }); + test("retries a failed attempt and records both durable receipts", async () => { const paths = await fixture(); const source = await repository(paths.root, "retry");