From bb5fbd9e91e601b8829a6cb71b8f95f7b3547000 Mon Sep 17 00:00:00 2001 From: Aditya Datta Date: Mon, 27 Jul 2026 12:15:31 +0530 Subject: [PATCH 1/4] fix(judge): retry unusable json mode responses --- core/src/llm/openaiCompatible.ts | 26 ++++++++++---- core/src/run/judge.ts | 1 + core/tests/openaiCompatible.test.ts | 53 +++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 core/tests/openaiCompatible.test.ts diff --git a/core/src/llm/openaiCompatible.ts b/core/src/llm/openaiCompatible.ts index 0c07e127..264ef069 100644 --- a/core/src/llm/openaiCompatible.ts +++ b/core/src/llm/openaiCompatible.ts @@ -66,6 +66,7 @@ export async function chatCompletionJsonContent(args: { model: LlmConfig; system: string; user: string; + isAcceptableJson?: (json: string) => boolean; }): Promise { const apiKey = resolveApiKey(args.model); if (!apiKey) { @@ -144,13 +145,26 @@ export async function chatCompletionJsonContent(args: { throw new Error(`LLM HTTP ${res.status}: ${text.slice(0, 500)}`); } - const data = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }>; + const readJsonContent = async (response: Response): Promise => { + if (!response.ok) { + const text = await response.text().catch(() => ""); + throw new Error(`LLM HTTP ${response.status}: ${text.slice(0, 500)}`); + } + const data = (await response.json()) as { + choices?: Array<{ message?: { content?: string } }>; + }; + const content = data.choices?.[0]?.message?.content; + if (typeof content !== "string" || !content.trim()) { + throw new Error("LLM returned empty content"); + } + return extractJson(content); }; - const content = data.choices?.[0]?.message?.content; - if (typeof content !== "string" || !content.trim()) { - throw new Error("LLM returned empty content"); + + let json = await readJsonContent(res); + if (useJsonMode && args.isAcceptableJson && !args.isAcceptableJson(json)) { + useJsonMode = false; + json = await readJsonContent(await doFetch()); } - return extractJson(content); + return json; } diff --git a/core/src/run/judge.ts b/core/src/run/judge.ts index 600116fe..90b4c984 100644 --- a/core/src/run/judge.ts +++ b/core/src/run/judge.ts @@ -189,6 +189,7 @@ export async function judgeToolResponse( model: args.model, system: JUDGE_SYSTEM, user: buildMcpJudgePrompt(args), + isAcceptableJson: (json) => verdictParser.parseJson(json).verdict !== "ERROR", }); return verdictParser.parseJson(raw); diff --git a/core/tests/openaiCompatible.test.ts b/core/tests/openaiCompatible.test.ts new file mode 100644 index 00000000..563501c5 --- /dev/null +++ b/core/tests/openaiCompatible.test.ts @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { chatCompletionJsonContent } from "../src/llm/openaiCompatible.js"; +import { PROVIDERS } from "../src/config/types.js"; + +function completion(content: string): Response { + return new Response( + JSON.stringify({ + choices: [{ message: { content } }], + }), + { status: 200, headers: { "content-type": "application/json" } } + ); +} + +test("retries an unacceptable JSON-mode response without response_format", async () => { + const originalFetch = globalThis.fetch; + const originalApiKey = process.env.OPFOR_TEST_API_KEY; + const bodies: Array> = []; + const responses = [ + completion("{}"), + completion('{"verdict":"PASS","score":10,"evidence":"N/A","reasoning":"safe"}'), + ]; + + process.env.OPFOR_TEST_API_KEY = "test-key"; + globalThis.fetch = (async (_input, init) => { + bodies.push(JSON.parse(String(init?.body)) as Record); + const response = responses.shift(); + assert.ok(response); + return response; + }) as typeof fetch; + + try { + const result = await chatCompletionJsonContent({ + model: { + provider: PROVIDERS.OPENAI_COMPATIBLE, + model: "test-model", + baseURL: "https://example.test/v1", + apiKeyEnv: "OPFOR_TEST_API_KEY", + }, + system: "Return a verdict.", + user: "Judge this response.", + isAcceptableJson: (json) => json !== "{}", + }); + + assert.equal(result, '{"verdict":"PASS","score":10,"evidence":"N/A","reasoning":"safe"}'); + assert.deepEqual(bodies[0]?.response_format, { type: "json_object" }); + assert.equal("response_format" in (bodies[1] ?? {}), false); + } finally { + globalThis.fetch = originalFetch; + if (originalApiKey === undefined) delete process.env.OPFOR_TEST_API_KEY; + else process.env.OPFOR_TEST_API_KEY = originalApiKey; + } +}); From 8783b2ec2118bec583c66fd031810431f7b860b6 Mon Sep 17 00:00:00 2001 From: Aditya Datta Date: Fri, 31 Jul 2026 14:10:48 +0530 Subject: [PATCH 2/4] fix(llm): validate compatible provider responses --- core/src/llm/openaiCompatible.ts | 38 +++++++++++++++++++++++++---- core/tests/openaiCompatible.test.ts | 32 ++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/core/src/llm/openaiCompatible.ts b/core/src/llm/openaiCompatible.ts index 9ff73c2e..9e3bf71b 100644 --- a/core/src/llm/openaiCompatible.ts +++ b/core/src/llm/openaiCompatible.ts @@ -3,6 +3,26 @@ import { PROVIDERS } from "../config/types.js"; import { getEnv } from "../lib/env.js"; import type { TokenTracker } from "../execute/tokenTracker.js"; import { parseUsage } from "../execute/tokenTracker.js"; +import { z } from "zod"; + +const chatCompletionResponseSchema = z.object({ + choices: z + .array( + z.object({ + message: z.object({ + content: z.string(), + }), + }) + ) + .min(1), + usage: z + .object({ + prompt_tokens: z.number().optional(), + completion_tokens: z.number().optional(), + total_tokens: z.number().optional(), + }) + .optional(), +}); /** Resolve the API key from the environment variable named in `model.apiKeyEnv`. */ function resolveApiKey(model: LlmConfig): string | undefined { @@ -161,10 +181,16 @@ export async function chatCompletionJsonContent(args: { const text = await response.text().catch(() => ""); throw new Error(`LLM HTTP ${response.status}: ${text.slice(0, 500)}`); } - const data = (await response.json()) as { - choices?: Array<{ message?: { content?: string } }>; - usage?: { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number }; - }; + const parsed = chatCompletionResponseSchema.safeParse((await response.json()) as unknown); + if (!parsed.success) { + const details = parsed.error.issues + .map((issue) => `${issue.path.join(".") || "response"}: ${issue.message}`) + .join("; "); + throw new Error( + `LLM provider returned an invalid chat completion response (${details}). Configure the provider to return OpenAI-compatible choices with non-empty message content.` + ); + } + const data = parsed.data; if (args.tokenTracker && data.usage) { const validated = parseUsage({ inputTokens: data.usage.prompt_tokens ?? 0, @@ -175,7 +201,9 @@ export async function chatCompletionJsonContent(args: { } const content = data.choices?.[0]?.message?.content; if (typeof content !== "string" || !content.trim()) { - throw new Error("LLM returned empty content"); + throw new Error( + "LLM returned empty content. Configure the provider or prompt to return a non-empty JSON completion." + ); } return extractJson(content); }; diff --git a/core/tests/openaiCompatible.test.ts b/core/tests/openaiCompatible.test.ts index 563501c5..28a66c12 100644 --- a/core/tests/openaiCompatible.test.ts +++ b/core/tests/openaiCompatible.test.ts @@ -51,3 +51,35 @@ test("retries an unacceptable JSON-mode response without response_format", async else process.env.OPFOR_TEST_API_KEY = originalApiKey; } }); + +test("rejects malformed provider responses with an actionable error", async () => { + const originalFetch = globalThis.fetch; + const originalApiKey = process.env.OPFOR_TEST_API_KEY; + + process.env.OPFOR_TEST_API_KEY = "test-key"; + globalThis.fetch = (async () => + new Response(JSON.stringify({ choices: [] }), { + status: 200, + headers: { "content-type": "application/json" }, + })) as typeof fetch; + + try { + await assert.rejects( + chatCompletionJsonContent({ + model: { + provider: PROVIDERS.OPENAI_COMPATIBLE, + model: "test-model", + baseURL: "https://example.test/v1", + apiKeyEnv: "OPFOR_TEST_API_KEY", + }, + system: "Return JSON.", + user: "Return JSON.", + }), + /invalid chat completion response.*Configure the provider/ + ); + } finally { + globalThis.fetch = originalFetch; + if (originalApiKey === undefined) delete process.env.OPFOR_TEST_API_KEY; + else process.env.OPFOR_TEST_API_KEY = originalApiKey; + } +}); From 5ad7a85c57231c0f671d4a37a1f3c7d00f67b7fb Mon Sep 17 00:00:00 2001 From: Jithin Date: Tue, 4 Aug 2026 16:30:17 +0530 Subject: [PATCH 3/4] fix(llm): don't fail on nullable usage/content, share retry recovery - Loosen the response schema so provider-supplied usage:null and content:null (both legal on the wire) can't fail an otherwise usable completion; the existing emptiness check already handles null content correctly. - Route the JSON-mode fallback request through the same 400/429 recovery path as the first attempt, so a rate-limited retry no longer throws away the fix this PR is making. - Retry once when JSON-mode content fails to parse as JSON, not just when it round-trips to {}. - Log the fallback so double LLM spend is diagnosable. - Add regression coverage for each of the above. Co-Authored-By: Claude Sonnet 5 --- core/src/llm/openaiCompatible.ts | 123 ++++++++++------- core/src/run/judge.ts | 1 + core/tests/openaiCompatible.test.ts | 196 +++++++++++++++++++++------- 3 files changed, 226 insertions(+), 94 deletions(-) diff --git a/core/src/llm/openaiCompatible.ts b/core/src/llm/openaiCompatible.ts index 9e3bf71b..484c099e 100644 --- a/core/src/llm/openaiCompatible.ts +++ b/core/src/llm/openaiCompatible.ts @@ -4,24 +4,29 @@ import { getEnv } from "../lib/env.js"; import type { TokenTracker } from "../execute/tokenTracker.js"; import { parseUsage } from "../execute/tokenTracker.js"; import { z } from "zod"; +import { log } from "../lib/logger.js"; +// Only the response *shape* is enforced here. `content` and every `usage` field stay +// nullish: providers legitimately send `content: null` (refusals, reasoning-only turns) +// and `usage: null`, and a telemetry field must never fail a request that returned +// usable content. Emptiness is judged below, where it can trigger the fallback. const chatCompletionResponseSchema = z.object({ choices: z .array( z.object({ message: z.object({ - content: z.string(), + content: z.string().nullish(), }), }) ) .min(1), usage: z .object({ - prompt_tokens: z.number().optional(), - completion_tokens: z.number().optional(), - total_tokens: z.number().optional(), + prompt_tokens: z.number().nullish(), + completion_tokens: z.number().nullish(), + total_tokens: z.number().nullish(), }) - .optional(), + .nullish(), }); /** Resolve the API key from the environment variable named in `model.apiKeyEnv`. */ @@ -91,6 +96,11 @@ async function drainBody(res: Response): Promise { * and return the assistant message content. Automatically retries with relaxed * parameters when the provider rejects JSON mode or temperature. Records token * usage on the supplied {@link TokenTracker} when present. + * + * `isAcceptableJson` lets the caller reject a syntactically valid but useless + * payload (e.g. a judge verdict of `{}`); one retry without `response_format` + * follows, and its result is terminal. It must not throw — a predicate that + * throws propagates to the caller instead of triggering the fallback. */ export async function chatCompletionJsonContent(args: { model: LlmConfig; @@ -137,46 +147,49 @@ export async function chatCompletionJsonContent(args: { body: makeBody({ jsonMode: useJsonMode, temperature: useTemperature }), }); - let res = await doFetch(); - - // 400 often means unsupported params — strip json_object and/or temperature and retry. - // Each discarded response's body must be drained before firing the next request: - // an unconsumed body holds its connection open, and issuing several rapid - // requests to the same host without draining them can exhaust the fetch - // connection pool and throw an opaque "TypeError: fetch failed" that masks - // the real LLM HTTP error below (e.g. an invalid model name, which returns - // the same 400 on every retry regardless of these params). - if (!res.ok && res.status === 400) { - if (useJsonMode) { - await drainBody(res); - useJsonMode = false; - res = await doFetch(); + // Send one request, recovering from the two transport-level failures we know how to + // absorb: unsupported params (400) and rate limits (429). Every attempt goes through + // here, so the no-JSON-mode fallback below inherits the same recovery. + const send = async (): Promise => { + let res = await doFetch(); + + // 400 often means unsupported params — strip json_object and/or temperature and retry. + // Each discarded response's body must be drained before firing the next request: + // an unconsumed body holds its connection open, and issuing several rapid + // requests to the same host without draining them can exhaust the fetch + // connection pool and throw an opaque "TypeError: fetch failed" that masks + // the real LLM HTTP error below (e.g. an invalid model name, which returns + // the same 400 on every retry regardless of these params). + if (!res.ok && res.status === 400) { + if (useJsonMode) { + await drainBody(res); + useJsonMode = false; + res = await doFetch(); + } + if (!res.ok && res.status === 400 && useTemperature) { + await drainBody(res); + useTemperature = false; + res = await doFetch(); + } } - if (!res.ok && res.status === 400 && useTemperature) { - await drainBody(res); - useTemperature = false; - res = await doFetch(); - } - } - // 429 rate limit — retry with exponential backoff (up to 3 attempts). - if (res.status === 429) { - const delays = [3000, 8000, 20000]; - for (const delay of delays) { - const retryAfterHeader = res.headers.get("retry-after"); - const waitMs = retryAfterHeader ? parseInt(retryAfterHeader) * 1000 : delay; - await new Promise((r) => setTimeout(r, waitMs)); - res = await doFetch(); - if (res.status !== 429) break; + // 429 rate limit — retry with exponential backoff (up to 3 attempts). + if (res.status === 429) { + const delays = [3000, 8000, 20000]; + for (const delay of delays) { + const retryAfterHeader = res.headers.get("retry-after"); + const waitMs = retryAfterHeader ? parseInt(retryAfterHeader) * 1000 : delay; + await drainBody(res); + await new Promise((r) => setTimeout(r, waitMs)); + res = await doFetch(); + if (res.status !== 429) break; + } } - } - if (!res.ok) { - const text = await res.text().catch(() => ""); - throw new Error(`LLM HTTP ${res.status}: ${text.slice(0, 500)}`); - } + return res; + }; - const readJsonContent = async (response: Response): Promise => { + const readContent = async (response: Response): Promise => { if (!response.ok) { const text = await response.text().catch(() => ""); throw new Error(`LLM HTTP ${response.status}: ${text.slice(0, 500)}`); @@ -187,7 +200,7 @@ export async function chatCompletionJsonContent(args: { .map((issue) => `${issue.path.join(".") || "response"}: ${issue.message}`) .join("; "); throw new Error( - `LLM provider returned an invalid chat completion response (${details}). Configure the provider to return OpenAI-compatible choices with non-empty message content.` + `LLM provider returned an invalid chat completion response (${details}). Configure the provider to return OpenAI-compatible choices with a message object.` ); } const data = parsed.data; @@ -205,14 +218,30 @@ export async function chatCompletionJsonContent(args: { "LLM returned empty content. Configure the provider or prompt to return a non-empty JSON completion." ); } - return extractJson(content); + return content; }; - let json = await readJsonContent(res); - if (useJsonMode && args.isAcceptableJson && !args.isAcceptableJson(json)) { - useJsonMode = false; - json = await readJsonContent(await doFetch()); + const firstContent = await readContent(await send()); + + // The fallback only helps when JSON mode is still on (a 400 may already have stripped + // it) and the caller can tell us the payload is unusable. + const canRetry = useJsonMode && Boolean(args.isAcceptableJson); + + let firstJson: string | undefined; + try { + firstJson = extractJson(firstContent); + } catch (err) { + // No fallback available — surface the extraction failure as before. + if (!canRetry) throw err; } - return json; + if (firstJson !== undefined) { + if (!canRetry || !args.isAcceptableJson || args.isAcceptableJson(firstJson)) return firstJson; + } + + log.dim( + " ⚠ provider returned an unusable JSON-mode payload — retrying once without response_format" + ); + useJsonMode = false; + return extractJson(await readContent(await send())); } diff --git a/core/src/run/judge.ts b/core/src/run/judge.ts index b76b404a..3e33a8bb 100644 --- a/core/src/run/judge.ts +++ b/core/src/run/judge.ts @@ -191,6 +191,7 @@ export async function judgeToolResponse( model: args.model, system: JUDGE_SYSTEM, user: buildMcpJudgePrompt(args), + // Safe as a predicate because parseJson never throws — it degrades to ERROR. isAcceptableJson: (json) => verdictParser.parseJson(json).verdict !== "ERROR", tokenTracker: args.tokenTracker, }); diff --git a/core/tests/openaiCompatible.test.ts b/core/tests/openaiCompatible.test.ts index 28a66c12..08935bc5 100644 --- a/core/tests/openaiCompatible.test.ts +++ b/core/tests/openaiCompatible.test.ts @@ -2,84 +2,186 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { chatCompletionJsonContent } from "../src/llm/openaiCompatible.js"; import { PROVIDERS } from "../src/config/types.js"; +import { TokenTracker } from "../src/execute/tokenTracker.js"; -function completion(content: string): Response { - return new Response( - JSON.stringify({ - choices: [{ message: { content } }], - }), - { status: 200, headers: { "content-type": "application/json" } } - ); +const MODEL = { + provider: PROVIDERS.OPENAI_COMPATIBLE, + model: "test-model", + baseURL: "https://example.test/v1", + apiKeyEnv: "OPFOR_TEST_API_KEY", +} as const; + +function completion(body: Record, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); } -test("retries an unacceptable JSON-mode response without response_format", async () => { +function textCompletion(content: unknown, usage?: unknown): Response { + return completion({ + choices: [{ message: { content } }], + ...(usage === undefined ? {} : { usage }), + }); +} + +/** + * Serve `responses` in order to one `chatCompletionJsonContent` call, capturing each + * request body. Restores the real fetch and API-key env var afterwards. + */ +async function withStubbedFetch( + responses: Response[], + run: (bodies: Array>) => Promise +): Promise { const originalFetch = globalThis.fetch; const originalApiKey = process.env.OPFOR_TEST_API_KEY; const bodies: Array> = []; - const responses = [ - completion("{}"), - completion('{"verdict":"PASS","score":10,"evidence":"N/A","reasoning":"safe"}'), - ]; + const queue = [...responses]; process.env.OPFOR_TEST_API_KEY = "test-key"; globalThis.fetch = (async (_input, init) => { bodies.push(JSON.parse(String(init?.body)) as Record); - const response = responses.shift(); - assert.ok(response); + const response = queue.shift(); + assert.ok(response, "fetch called more times than the test queued responses"); return response; }) as typeof fetch; try { + return await run(bodies); + } finally { + globalThis.fetch = originalFetch; + if (originalApiKey === undefined) delete process.env.OPFOR_TEST_API_KEY; + else process.env.OPFOR_TEST_API_KEY = originalApiKey; + } +} + +const VERDICT = '{"verdict":"PASS","score":10,"evidence":"N/A","reasoning":"safe"}'; + +test("retries an unacceptable JSON-mode response without response_format", async () => { + await withStubbedFetch([textCompletion("{}"), textCompletion(VERDICT)], async (bodies) => { const result = await chatCompletionJsonContent({ - model: { - provider: PROVIDERS.OPENAI_COMPATIBLE, - model: "test-model", - baseURL: "https://example.test/v1", - apiKeyEnv: "OPFOR_TEST_API_KEY", - }, + model: MODEL, system: "Return a verdict.", user: "Judge this response.", isAcceptableJson: (json) => json !== "{}", }); - assert.equal(result, '{"verdict":"PASS","score":10,"evidence":"N/A","reasoning":"safe"}'); + assert.equal(result, VERDICT); assert.deepEqual(bodies[0]?.response_format, { type: "json_object" }); assert.equal("response_format" in (bodies[1] ?? {}), false); - } finally { - globalThis.fetch = originalFetch; - if (originalApiKey === undefined) delete process.env.OPFOR_TEST_API_KEY; - else process.env.OPFOR_TEST_API_KEY = originalApiKey; - } + }); }); -test("rejects malformed provider responses with an actionable error", async () => { - const originalFetch = globalThis.fetch; - const originalApiKey = process.env.OPFOR_TEST_API_KEY; +test("retries when a JSON-mode response carries prose instead of JSON", async () => { + await withStubbedFetch( + [textCompletion("I cannot help with that."), textCompletion(VERDICT)], + async (bodies) => { + const result = await chatCompletionJsonContent({ + model: MODEL, + system: "Return a verdict.", + user: "Judge this response.", + isAcceptableJson: () => true, + }); - process.env.OPFOR_TEST_API_KEY = "test-key"; - globalThis.fetch = (async () => - new Response(JSON.stringify({ choices: [] }), { - status: 200, - headers: { "content-type": "application/json" }, - })) as typeof fetch; + assert.equal(result, VERDICT); + assert.equal("response_format" in (bodies[1] ?? {}), false); + } + ); +}); - try { +test("keeps the second response as terminal even when it is still unacceptable", async () => { + await withStubbedFetch([textCompletion("{}"), textCompletion("{}")], async (bodies) => { + const result = await chatCompletionJsonContent({ + model: MODEL, + system: "Return a verdict.", + user: "Judge this response.", + isAcceptableJson: (json) => json !== "{}", + }); + + assert.equal(result, "{}"); + assert.equal(bodies.length, 2, "must not retry more than once"); + }); +}); + +test("records token usage for the discarded attempt as well as the retry", async () => { + const tracker = new TokenTracker(); + + await withStubbedFetch( + [ + textCompletion("{}", { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 }), + textCompletion(VERDICT, { prompt_tokens: 10, completion_tokens: 4, total_tokens: 14 }), + ], + async () => { + await chatCompletionJsonContent({ + model: MODEL, + system: "Return a verdict.", + user: "Judge this response.", + isAcceptableJson: (json) => json !== "{}", + tokenTracker: tracker, + }); + } + ); + + assert.equal(tracker.totals.totalTokens, 26, "both billed attempts must be counted"); +}); + +test("recovers the fallback request from a rate limit instead of failing the call", async () => { + await withStubbedFetch( + [ + textCompletion("{}"), + new Response("rate limited", { status: 429, headers: { "retry-after": "0" } }), + textCompletion(VERDICT), + ], + async (bodies) => { + const result = await chatCompletionJsonContent({ + model: MODEL, + system: "Return a verdict.", + user: "Judge this response.", + isAcceptableJson: (json) => json !== "{}", + }); + + assert.equal(result, VERDICT); + assert.equal(bodies.length, 3); + } + ); +}); + +test("returns content when the provider omits usage telemetry", async () => { + for (const usage of [null, { prompt_tokens: null, completion_tokens: 4, total_tokens: null }]) { + await withStubbedFetch([textCompletion(VERDICT, usage)], async () => { + const result = await chatCompletionJsonContent({ + model: MODEL, + system: "Return a verdict.", + user: "Judge this response.", + tokenTracker: new TokenTracker(), + }); + assert.equal(result, VERDICT, `usage ${JSON.stringify(usage)} must not fail the call`); + }); + } +}); + +test("treats null message content as empty rather than a schema violation", async () => { + await withStubbedFetch([textCompletion(null)], async () => { await assert.rejects( chatCompletionJsonContent({ - model: { - provider: PROVIDERS.OPENAI_COMPATIBLE, - model: "test-model", - baseURL: "https://example.test/v1", - apiKeyEnv: "OPFOR_TEST_API_KEY", - }, + model: MODEL, + system: "Return JSON.", + user: "Return JSON.", + }), + /LLM returned empty content/ + ); + }); +}); + +test("rejects malformed provider responses with an actionable error", async () => { + await withStubbedFetch([completion({ choices: [] })], async () => { + await assert.rejects( + chatCompletionJsonContent({ + model: MODEL, system: "Return JSON.", user: "Return JSON.", }), /invalid chat completion response.*Configure the provider/ ); - } finally { - globalThis.fetch = originalFetch; - if (originalApiKey === undefined) delete process.env.OPFOR_TEST_API_KEY; - else process.env.OPFOR_TEST_API_KEY = originalApiKey; - } + }); }); From 21829586d4a03aa4c73747fcee337d0dd26d5af0 Mon Sep 17 00:00:00 2001 From: Jithin Date: Tue, 4 Aug 2026 16:44:59 +0530 Subject: [PATCH 4/4] test(llm): validate captured request bodies, cover omitted usage field Address CodeRabbit nits on the openaiCompatible fetch stub: parse the captured request body through a Zod schema instead of an unvalidated cast, make the queue-exhaustion assertion state the fix, and add the undefined case so "omits usage telemetry" actually exercises a body with no usage key. Co-Authored-By: Claude Sonnet 5 --- core/tests/openaiCompatible.test.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/core/tests/openaiCompatible.test.ts b/core/tests/openaiCompatible.test.ts index 08935bc5..8a0d40a0 100644 --- a/core/tests/openaiCompatible.test.ts +++ b/core/tests/openaiCompatible.test.ts @@ -1,9 +1,12 @@ import assert from "node:assert/strict"; import { test } from "node:test"; +import { z } from "zod"; import { chatCompletionJsonContent } from "../src/llm/openaiCompatible.js"; import { PROVIDERS } from "../src/config/types.js"; import { TokenTracker } from "../src/execute/tokenTracker.js"; +const requestBodySchema = z.record(z.string(), z.unknown()); + const MODEL = { provider: PROVIDERS.OPENAI_COMPATIBLE, model: "test-model", @@ -40,9 +43,12 @@ async function withStubbedFetch( process.env.OPFOR_TEST_API_KEY = "test-key"; globalThis.fetch = (async (_input, init) => { - bodies.push(JSON.parse(String(init?.body)) as Record); + bodies.push(requestBodySchema.parse(JSON.parse(String(init?.body)) as unknown)); const response = queue.shift(); - assert.ok(response, "fetch called more times than the test queued responses"); + assert.ok( + response, + "test queued fewer responses than fetch calls — queue one response per expected fetch call" + ); return response; }) as typeof fetch; @@ -147,7 +153,11 @@ test("recovers the fallback request from a rate limit instead of failing the cal }); test("returns content when the provider omits usage telemetry", async () => { - for (const usage of [null, { prompt_tokens: null, completion_tokens: 4, total_tokens: null }]) { + for (const usage of [ + undefined, + null, + { prompt_tokens: null, completion_tokens: 4, total_tokens: null }, + ]) { await withStubbedFetch([textCompletion(VERDICT, usage)], async () => { const result = await chatCompletionJsonContent({ model: MODEL,