fix: recover multiscan locks with missing or malformed owner metadata

sdk/typescript/src/multiscan.ts

@@ -28,6 +28,7 @@ const REQUIRED_ARTIFACTS = [
   "coverage.json",
   "report.md",
 ];
+const INCOMPLETE_LOCK_MILLISECONDS = 30_000;
 
 interface MultiscanTask {
   id: string;
@@ -268,29 +269,71 @@ async function appendReceipt(path: string, receipt: string): Promise<void> {
 async function acquireLock(output: string): Promise<() => Promise<void>> {
   const path = join(output, ".lock");
   const ownerPath = join(path, "owner.json");
-  try {
-    await mkdir(path, { mode: 0o700 });
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
-    const { pid } = JSON.parse(await readFile(ownerPath, "utf8")) as {
-      pid: number;
-    };
+  for (;;) {
+    try {
+      await mkdir(path, { mode: 0o700 });
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+      if (!(await recoverStaleLock(output, path, ownerPath))) {
+        throw new Error("A multiscan supervisor is already running.");
+      }
+      continue;
+    }
+    await writeFile(ownerPath, `${JSON.stringify({ pid: process.pid })}\n`, {
+      flag: "wx",
+      mode: 0o600,
+    });
+    return async () => rm(path, { recursive: true });
+  }
+}
+
+async function recoverStaleLock(
+  output: string,
+  path: string,
+  ownerPath: string,
+): Promise<boolean> {
+  const metadata = await lstat(path).catch((error: unknown) => {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw error;
+  });
+  if (metadata === null) return true;
+
+  let owner: unknown;
+  if (metadata.isDirectory()) {
     try {
-      process.kill(pid, 0);
-      throw new Error("A multiscan supervisor is already running.");
+      owner = JSON.parse(await readFile(ownerPath, "utf8"));
+    } catch (error) {
+      const code = (error as NodeJS.ErrnoException).code;
+      if (code !== "ENOENT" && !(error instanceof SyntaxError)) throw error;
+    }
+  }
+
+  if (isRecord(owner) && typeof owner["pid"] === "number") {
+    try {
+      process.kill(owner["pid"], 0);
+      return false;
     } catch (failure) {
-      if ((failure as NodeJS.ErrnoException).code !== "ESRCH") throw failure;
+      const code = (failure as NodeJS.ErrnoException).code;
+      if (code === "EPERM") return false;
+      if (code !== "ESRCH") throw failure;
     }
-    const stale = join(output, `.lock.stale-${randomUUID()}`);
+  } else if (Date.now() - metadata.mtimeMs < INCOMPLETE_LOCK_MILLISECONDS) {
+    return false;
+  }
+
+  const stale = join(output, `.lock.stale-${randomUUID()}`);
+  try {
     await rename(path, stale);
-    await rm(stale, { recursive: true });
-    return await acquireLock(output);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return true;
+    throw error;
   }
-  await writeFile(ownerPath, `${JSON.stringify({ pid: process.pid })}\n`, {
-    flag: "wx",
-    mode: 0o600,
-  });
-  return async () => rm(path, { recursive: true });
+  await rm(stale, { recursive: true, force: true });
+  return true;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 
 async function ensureManifest(

sdk/typescript/tests-ts/multiscan.test.ts

@@ -8,6 +8,7 @@ import {
   readdir,
   rm,
   symlink,
+  utimes,
   writeFile,
 } from "node:fs/promises";
 import { tmpdir } from "node:os";
@@ -440,6 +441,59 @@ describe("multiscan", () => {
     await expect(access(lock)).rejects.toThrow();
   });
 
+  test("recovers locks with missing or malformed owner metadata", async () => {
+    const paths = await fixture();
+    const source = await repository(paths.root, "lock-recovery");
+    await writeFile(
+      paths.input,
+      `id,repository,revision\nlock-recovery,${source.path},${source.revision}\n`,
+    );
+    await mkdir(paths.output);
+    const lock = join(paths.output, ".lock");
+    const aged = new Date(Date.now() - 2 * 60_000);
+    let scans = 0;
+    const security = client(async (_repository, scanOptions = {}) => {
+      scans += 1;
+      return await completedScan(scanOptions.outputDir!);
+    });
+
+    for (const owner of [undefined, '{"pid":', '{"pid":"invalid"}']) {
+      await mkdir(lock);
+      if (owner !== undefined) {
+        await writeFile(join(lock, "owner.json"), owner);
+      }
+      await utimes(lock, aged, aged);
+      const result = await runMultiscan(options(paths, security));
+      expect(result).toMatchObject({ completed: 1, failed: 0 });
+      await expect(access(lock)).rejects.toThrow();
+    }
+
+    expect(scans).toBe(1);
+    expect(
+      (await readdir(paths.output)).some((name) =>
+        name.startsWith(".lock.stale-"),
+      ),
+    ).toBe(false);
+  });
+
+  test("treats a fresh lock without owner metadata as an active supervisor", async () => {
+    const paths = await fixture();
+    const source = await repository(paths.root, "lock-fresh");
+    await writeFile(
+      paths.input,
+      `id,repository,revision\nlock-fresh,${source.path},${source.revision}\n`,
+    );
+    await mkdir(paths.output);
+    await mkdir(join(paths.output, ".lock"));
+    const security = client(async (_repository, scanOptions = {}) => {
+      return await completedScan(scanOptions.outputDir!);
+    });
+
+    await expect(runMultiscan(options(paths, security))).rejects.toThrow(
+      /running|locked|supervisor/iu,
+    );
+  });
+
   test("retries a failed attempt and records both durable receipts", async () => {
     const paths = await fixture();
     const source = await repository(paths.root, "retry");