From 42006e16956ecd16a72b89a8b826b3a15aba6de2 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 15 Jul 2026 15:47:08 -0700 Subject: [PATCH 1/3] ci: add format:check and test:exports to release gate The release script published without verifying formatting or the package export map, so unformatted code or a broken export map could reach npm (recurring failure, see CHANGELOG #263). Add format:check before build (cheap, fail fast) and test:exports after build (needs the built dist), with npm publish last. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8fd650c2..00831188 100644 --- a/package.json +++ b/package.json @@ -169,7 +169,7 @@ "test:exports": "publint && attw --pack .", "lint": "eslint .", "format:check": "prettier --check .", - "release": "pnpm build && pnpm test && pnpm lint && npm publish", + "release": "pnpm format:check && pnpm build && pnpm test && pnpm lint && pnpm test:exports && npm publish", "prepare": "husky || true" }, "lint-staged": { From ab9335da5171724654ed402ea39edf46e275367f Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 15 Jul 2026 15:49:25 -0700 Subject: [PATCH 2/3] fix: guard recordedTimings.interChunkDelaysMs against non-array values Untrusted fixture JSON may set interChunkDelaysMs to null/missing/non-array. The sanitizer called .filter with no Array.isArray guard, throwing a TypeError inside entryToFixture that aborted the entire fixture file's load and silently dropped every fixture in it. Coerce non-array values to [] like sibling guards. --- src/__tests__/fixture-loader.test.ts | 26 ++++++++++++++++++++++++++ src/fixture-loader.ts | 4 +++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/__tests__/fixture-loader.test.ts b/src/__tests__/fixture-loader.test.ts index ca4719c1..ecab90da 100644 --- a/src/__tests__/fixture-loader.test.ts +++ b/src/__tests__/fixture-loader.test.ts @@ -316,6 +316,32 @@ describe("loadFixtureFile", () => { expect(fixtures[0].disconnectAfterMs).toBeUndefined(); }); + it("loads sibling fixtures when one entry has a non-array interChunkDelaysMs", () => { + // Untrusted fixture JSON: interChunkDelaysMs is typed number[] but may be + // null/missing/non-array. Without an Array.isArray guard, .filter throws a + // TypeError inside entryToFixture, aborting the entire file's load and + // silently dropping every fixture in it. + const filePath = writeJson(tmpDir, "bad-timings.json", { + fixtures: [ + { + match: { userMessage: "malformed" }, + response: { content: "bad" }, + recordedTimings: { ttftMs: 0, interChunkDelaysMs: null, totalDurationMs: 0 }, + }, + { + match: { userMessage: "valid" }, + response: { content: "good" }, + }, + ], + }); + + const fixtures = loadFixtureFile(filePath); + expect(fixtures).toHaveLength(2); + expect(fixtures[1].match.userMessage).toBe("valid"); + // The malformed entry is sanitized: non-array interChunkDelaysMs -> []. + expect(fixtures[0].recordedTimings?.interChunkDelaysMs).toEqual([]); + }); + it("warns and returns empty array for invalid JSON", () => { const filePath = join(tmpDir, "bad.json"); writeFileSync(filePath, "{ not valid json", "utf-8"); diff --git a/src/fixture-loader.ts b/src/fixture-loader.ts index 9d911fcb..7a89915d 100644 --- a/src/fixture-loader.ts +++ b/src/fixture-loader.ts @@ -107,7 +107,9 @@ export function entryToFixture(entry: FixtureFileEntry, logger?: Logger): Fixtur if (fixture.recordedTimings) { const rt = fixture.recordedTimings; if (!Number.isFinite(rt.ttftMs) || rt.ttftMs < 0) rt.ttftMs = 0; - rt.interChunkDelaysMs = rt.interChunkDelaysMs.filter((d) => Number.isFinite(d) && d >= 0); + rt.interChunkDelaysMs = Array.isArray(rt.interChunkDelaysMs) + ? rt.interChunkDelaysMs.filter((d) => Number.isFinite(d) && d >= 0) + : []; if (!Number.isFinite(rt.totalDurationMs) || rt.totalDurationMs < 0) rt.totalDurationMs = 0; } From 87dcf798015e65a6bab4b8d8e5d145f18e67f3f8 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 15 Jul 2026 15:48:31 -0700 Subject: [PATCH 3/3] docs: fix E2E fixture lifecycle to use resetMatchCounts The E2E setup examples loaded fixtures once then called full reset() in afterEach, which wipes the in-memory fixture pool and 404s every test after the first. Switch between-test cleanup to resetMatchCounts() (suite.llm.resetMatchCounts() for MockSuite) and correct the prose so it leads with resetMatchCounts() and warns reset() also clears fixtures. --- skills/write-fixtures/SKILL.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/skills/write-fixtures/SKILL.md b/skills/write-fixtures/SKILL.md index 8c3de1a4..05099d8e 100644 --- a/skills/write-fixtures/SKILL.md +++ b/skills/write-fixtures/SKILL.md @@ -255,7 +255,7 @@ mock.on( mock.on({ userMessage: "status", sequenceIndex: 1 }, { content: "All systems operational." }); ``` -Match counts are tracked per fixture group and reset with `reset()` or `resetMatchCounts()`. +Match counts are tracked per fixture group. Use `resetMatchCounts()` between tests to reset counts while keeping loaded fixtures. `reset()` also clears the fixture pool, so avoid it between tests that share a loaded fixture set. ### Streaming physics (realistic timing) @@ -500,7 +500,7 @@ These fields map correctly across all provider formats — for example, `finishR 10. **Embeddings auto-generate if no fixture matches** — deterministic vectors are generated from the input text hash. You don't need a catch-all for embedding requests. -11. **Sequential response counts are tracked per fixture** — counts reset with `reset()` or `resetMatchCounts()`. The count increments after each match of that fixture group (all fixtures sharing the same non-`sequenceIndex` match fields). +11. **Sequential response counts are tracked per fixture** — use `resetMatchCounts()` between tests to reset counts while keeping loaded fixtures; `reset()` also clears the fixture pool, so don't use it between tests that share a loaded fixture set. The count increments after each match of that fixture group (all fixtures sharing the same non-`sequenceIndex` match fields). 12. **Bedrock uses Anthropic Messages format internally** — the adapter normalizes Bedrock requests to `ChatCompletionRequest`, so the same fixtures work. Bedrock supports both non-streaming (`/invoke`, `/converse`) and streaming (`/invoke-with-response-stream`, `/converse-stream`) endpoints. @@ -549,7 +549,7 @@ await suite.start(); // suite.llm — the LLMock instance // suite.url — base URL -afterEach(() => suite.reset()); // resets everything +afterEach(() => suite.llm.resetMatchCounts()); // reset sequence counts, keep fixtures afterAll(() => suite.stop()); ``` @@ -684,8 +684,8 @@ mock.loadFixtureDir("./fixtures"); await mock.start(); process.env.OPENAI_BASE_URL = `${mock.url}/v1`; -// Per-test cleanup -afterEach(() => mock.reset()); // clears fixtures AND journal +// Per-test cleanup — reset sequence match counts, keep the loaded fixtures +afterEach(() => mock.resetMatchCounts()); // Teardown afterAll(async () => await mock.stop()); @@ -736,6 +736,8 @@ const mock = await LLMock.create({ port: 0 }); // creates + starts in one call | `url` / `baseUrl` | Server URL (throws if not started) | | `port` | Server port number | +Between tests that share a loaded fixture set, use `resetMatchCounts()` (not `reset()`, which also clears fixtures). For a `MockSuite`, call `suite.llm.resetMatchCounts()` — the suite itself has no `resetMatchCounts()`. + Sequential responses use `on()` with `sequenceIndex` in the match — there is no dedicated convenience method. ## Record-and-Replay (VCR Mode)