From 561fa2c6991e5d8525f88d997bae05a69b800644 Mon Sep 17 00:00:00 2001 From: alarcritty Date: Fri, 31 Jul 2026 13:13:04 +0530 Subject: [PATCH] fix: recover multiscan locks with missing or malformed owner metadata Bulk scan creates its lock in two steps: create the .lock directory, then write owner.json inside it. A crash between those steps left a lock that every later run tried to read unconditionally, so recovery failed forever with ENOENT until the directory was deleted by hand. Truncated or structurally invalid owner files failed the same way. Treat missing, malformed, or invalid owner metadata as a stale lock and reclaim it through the existing rename-aside path, mirroring the credential-home lock recovery in runtime.ts. A lock modified within the last 30 seconds is still treated as an active supervisor so a concurrent process that is mid-publication is not stolen from, and EPERM from the liveness probe now reports an active supervisor instead of crashing. Fixes #102 --- sdk/typescript/src/multiscan.ts | 79 +++++++++++++++++------ sdk/typescript/tests-ts/multiscan.test.ts | 54 ++++++++++++++++ 2 files changed, 115 insertions(+), 18 deletions(-) diff --git a/sdk/typescript/src/multiscan.ts b/sdk/typescript/src/multiscan.ts index 534715a6..a82b2706 100644 --- a/sdk/typescript/src/multiscan.ts +++ b/sdk/typescript/src/multiscan.ts @@ -28,6 +28,7 @@ const REQUIRED_ARTIFACTS = [ "coverage.json", "report.md", ]; +const INCOMPLETE_LOCK_MILLISECONDS = 30_000; interface MultiscanTask { id: string; @@ -268,29 +269,71 @@ async function appendReceipt(path: string, receipt: string): Promise { async function acquireLock(output: string): Promise<() => Promise> { const path = join(output, ".lock"); const ownerPath = join(path, "owner.json"); - try { - await mkdir(path, { mode: 0o700 }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - const { pid } = JSON.parse(await readFile(ownerPath, "utf8")) as { - pid: number; - }; + for (;;) { + try { + await mkdir(path, { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + if (!(await recoverStaleLock(output, path, ownerPath))) { + throw new Error("A multiscan supervisor is already running."); + } + continue; + } + await writeFile(ownerPath, `${JSON.stringify({ pid: process.pid })}\n`, { + flag: "wx", + mode: 0o600, + }); + return async () => rm(path, { recursive: true }); + } +} + +async function recoverStaleLock( + output: string, + path: string, + ownerPath: string, +): Promise { + const metadata = await lstat(path).catch((error: unknown) => { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + }); + if (metadata === null) return true; + + let owner: unknown; + if (metadata.isDirectory()) { try { - process.kill(pid, 0); - throw new Error("A multiscan supervisor is already running."); + owner = JSON.parse(await readFile(ownerPath, "utf8")); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "ENOENT" && !(error instanceof SyntaxError)) throw error; + } + } + + if (isRecord(owner) && typeof owner["pid"] === "number") { + try { + process.kill(owner["pid"], 0); + return false; } catch (failure) { - if ((failure as NodeJS.ErrnoException).code !== "ESRCH") throw failure; + const code = (failure as NodeJS.ErrnoException).code; + if (code === "EPERM") return false; + if (code !== "ESRCH") throw failure; } - const stale = join(output, `.lock.stale-${randomUUID()}`); + } else if (Date.now() - metadata.mtimeMs < INCOMPLETE_LOCK_MILLISECONDS) { + return false; + } + + const stale = join(output, `.lock.stale-${randomUUID()}`); + try { await rename(path, stale); - await rm(stale, { recursive: true }); - return await acquireLock(output); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return true; + throw error; } - await writeFile(ownerPath, `${JSON.stringify({ pid: process.pid })}\n`, { - flag: "wx", - mode: 0o600, - }); - return async () => rm(path, { recursive: true }); + await rm(stale, { recursive: true, force: true }); + return true; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); } async function ensureManifest( diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.ts index f357fac9..7b7ed3e1 100644 --- a/sdk/typescript/tests-ts/multiscan.test.ts +++ b/sdk/typescript/tests-ts/multiscan.test.ts @@ -8,6 +8,7 @@ import { readdir, rm, symlink, + utimes, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -440,6 +441,59 @@ describe("multiscan", () => { await expect(access(lock)).rejects.toThrow(); }); + test("recovers locks with missing or malformed owner metadata", async () => { + const paths = await fixture(); + const source = await repository(paths.root, "lock-recovery"); + await writeFile( + paths.input, + `id,repository,revision\nlock-recovery,${source.path},${source.revision}\n`, + ); + await mkdir(paths.output); + const lock = join(paths.output, ".lock"); + const aged = new Date(Date.now() - 2 * 60_000); + let scans = 0; + const security = client(async (_repository, scanOptions = {}) => { + scans += 1; + return await completedScan(scanOptions.outputDir!); + }); + + for (const owner of [undefined, '{"pid":', '{"pid":"invalid"}']) { + await mkdir(lock); + if (owner !== undefined) { + await writeFile(join(lock, "owner.json"), owner); + } + await utimes(lock, aged, aged); + const result = await runMultiscan(options(paths, security)); + expect(result).toMatchObject({ completed: 1, failed: 0 }); + await expect(access(lock)).rejects.toThrow(); + } + + expect(scans).toBe(1); + expect( + (await readdir(paths.output)).some((name) => + name.startsWith(".lock.stale-"), + ), + ).toBe(false); + }); + + test("treats a fresh lock without owner metadata as an active supervisor", async () => { + const paths = await fixture(); + const source = await repository(paths.root, "lock-fresh"); + await writeFile( + paths.input, + `id,repository,revision\nlock-fresh,${source.path},${source.revision}\n`, + ); + await mkdir(paths.output); + await mkdir(join(paths.output, ".lock")); + const security = client(async (_repository, scanOptions = {}) => { + return await completedScan(scanOptions.outputDir!); + }); + + await expect(runMultiscan(options(paths, security))).rejects.toThrow( + /running|locked|supervisor/iu, + ); + }); + test("retries a failed attempt and records both durable receipts", async () => { const paths = await fixture(); const source = await repository(paths.root, "retry");