From 9986edb951715b622ed3ab1f412c73ad9c8100de Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sat, 19 Sep 2026 18:49:52 -0400 Subject: [PATCH 01/13] fix: bound session-shard memory for ranged queries Session month shards decode incrementally (one turn at a time, keeping only in-range turns with the exact kept/dropped/carry contract); shard loads run serially with the pre-lock snapshot released before the canonical reload. Out-of-range turns contribute raw dedup keys to cross-file suppression markers; retained strings detach from tokenizer buffers. The codex result cache rewrite, digest wiring, and aggregate parse mode follow as stacked PRs. --- CHANGELOG.md | 3 + package-lock.json | 25 + package.json | 1 + src/content-utils.ts | 44 ++ src/parser.ts | 66 +- src/providers/antigravity.ts | 4 +- src/providers/codex.ts | 14 +- src/providers/cursor.ts | 8 +- src/providers/types.ts | 4 +- src/session-cache.ts | 889 +++++++++++++++++++++-- src/shard-stream.ts | 342 +++++++++ tests/cache-directory-switch.test.ts | 4 + tests/content-utils.test.ts | 36 + tests/providers/antigravity.test.ts | 2 +- tests/session-cache-range-filter.test.ts | 675 +++++++++++++++++ tests/session-cache-shards.test.ts | 82 +++ tests/shard-stream-turns.test.ts | 152 ++++ 17 files changed, 2256 insertions(+), 95 deletions(-) create mode 100644 src/shard-stream.ts create mode 100644 tests/session-cache-range-filter.test.ts create mode 100644 tests/shard-stream-turns.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 7685d278b..ff3b09ed5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +### Fixed +- **Ranged commands decode cache shards incrementally instead of assembling whole multi-hundred-megabyte files.** Session month shards now decode one turn at a time and keep only in-range turns (with the exact same kept/dropped/carry contract as before, verified by the range-filter suite, including a 30k-turn record test); shard loads run serially and the pre-lock snapshot is released before the canonical reload instead of overlapping it. All retained strings are detached from tokenizer buffers (sliced views pinned whole input chunks: ~1.2GB unexplained heap on this corpus). Out-of-range turns contribute their raw dedup keys to cross-file suppression markers, and provider-scoped queries skip unrelated sections. + ### Fixed (desktop) - **The Models table shows every model that ran, including the ones under a cent.** The CLI's `models` command defaults `minCost` to $0.01, and the desktop bridge passed neither `--min-cost` nor `--unpriced`, so the table silently dropped every row below a cent — which by construction excluded every unpriced row too (a `0 >= 0.01` filter), leaving #1443's dimming and add-alias affordances unreachable in the shipped app. `codeburn:getModels` now passes `--min-cost 0` (and the demo bridge mirrors it), so sub-cent and unpriced rows arrive and render with their existing dim treatment; on a real lifetime corpus that recovers 10 rows and 2,160 calls the default filter hid (43 → 53 rows, verified in both themes). A row priced between $0.00 and $0.01 renders as "$0.00" without dimming — it is genuinely priced, just below the display floor. Fixes #1465. - **Packaging no longer aborts on a checkout whose path contains a UUID.** `app/scripts/stage-cli.mjs` resolved the CLI's production dependency closure by matching each `npm ls --parseable` line against the checkout's absolute `/node_modules/` prefix, and npm 11 and later redact UUID-shaped path segments to `***` in that output — so on CI runners and scratch worktrees under a UUID directory the match found nothing and packaging died with the misleading "is the root npm installed?". The top-level package name is now read by position (after the N-th `/node_modules` occurrence, N counted off the real root path, which the redaction cannot move) instead of by absolute-prefix match, so redacted, Windows-separated and nested-checkout shapes all resolve; a missing `npm_execpath` also fails with its own message instead of degrading into the empty-closure one. Fixes #1466. diff --git a/package-lock.json b/package-lock.json index f54e42604..12f3f275f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "ink": "^7.0.0", "react": "^19.2.5", "selfsigned": "^5.5.0", + "stream-json": "^3.6.0", "strip-ansi": "^7.2.0", "undici": "^7.27.2", "zod": "^3.25.76" @@ -3391,6 +3392,30 @@ "dev": true, "license": "MIT" }, + "node_modules/stream-chain": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-4.2.5.tgz", + "integrity": "sha512-Wtyq3bNE3ggLR0v2vftqvuhltym3WbZAkZpfIrkr5F/6vpeUmWmwTgXa16zD87gpahwJ/Qulq3zVfUlgIc0J2A==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=22" + }, + "funding": { + "url": "https://github.com/sponsors/uhop" + } + }, + "node_modules/stream-json": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-3.6.0.tgz", + "integrity": "sha512-NiJdqxKyau579z/E8vfqcjWfSDWxW/AT99javFXdPXF147Z5za85LRXSHEmSX9TKOakB7gaIccfD0fOIctb7KQ==", + "license": "BSD-3-Clause", + "dependencies": { + "stream-chain": "^4.2.5" + }, + "funding": { + "url": "https://github.com/sponsors/uhop" + } + }, "node_modules/string-width": { "version": "8.2.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.0.tgz", diff --git a/package.json b/package.json index 97004fc67..8e4112c31 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "ink": "^7.0.0", "react": "^19.2.5", "selfsigned": "^5.5.0", + "stream-json": "^3.6.0", "strip-ansi": "^7.2.0", "undici": "^7.27.2", "zod": "^3.25.76" diff --git a/src/content-utils.ts b/src/content-utils.ts index 0c41afb67..7a5766d43 100644 --- a/src/content-utils.ts +++ b/src/content-utils.ts @@ -56,3 +56,47 @@ export function flatSlice(s: string, max: number): string { export function flatString(s: string): string { return Buffer.from(s, 'utf16le').toString('utf16le') } +/// Flatten every string token in a decoded JSON graph IN PLACE (see +/// `flatString`, including object keys) and return the same graph. +/// +/// The streaming decoders hand values whose strings are slices of +/// multi-hundred-MB input chunks; structures memoized long-term must not pin +/// those chunks. The decoded graph is decoder-owned (fresh containers per +/// record, discarded by the pipeline after the callback), so mutating it is +/// safe — and unlike a deep copy it adds no parallel-graph transient. +/// Re-keying deletes and re-inserts every key in snapshot order, so relative +/// key order is unchanged; duplicate-flattened keys cannot occur (object +/// keys are already unique by content). +export function flattenJsonStrings(value: T): T { + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + const element: unknown = value[i] + value[i] = typeof element === 'string' ? flatString(element) : flattenJsonStrings(element) + } + return value + } + if (value !== null && typeof value === 'object') { + const record = value as Record + for (const key of Object.keys(record)) { + const field: unknown = record[key] + const flatField = typeof field === 'string' ? flatString(field) : flattenJsonStrings(field) + const flatKey = flatString(key) + delete record[key] + // `__proto__` assignment would invoke the prototype setter (losing the + // own key and changing the prototype); valid JSON can carry it as an + // own key (JSON.parse preserves it), so reinsert declaratively. + if (flatKey === '__proto__') { + Object.defineProperty(record, '__proto__', { + value: flatField, + enumerable: true, + writable: true, + configurable: true, + }) + } else { + record[flatKey] = flatField + } + } + return value + } + return typeof value === 'string' ? (flatString(value) as T) : value +} diff --git a/src/parser.ts b/src/parser.ts index 9fbf624b3..51685a91f 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -24,12 +24,16 @@ import { type CachedCall, type CachedFile, type CachedTurn, + type LoadCacheOptions, type ProviderSection, type SessionCache, beginColdHydration, cleanupOrphanedTempFiles, computeEnvFingerprint, DURABLE_PROVIDER_NAMES, + emptyCache, + fileFirstTurnProject, + fileNewestCallMs, fingerprintFile, isCacheComplete, isCacheDirty, @@ -39,6 +43,7 @@ import { monthScopeForRange, reconcileFile, saveCache, + seedDroppedKeys, sourcePathStatCandidates, } from './session-cache.js' import { acquireCacheRefreshLock, type RefreshLockHandle, type RefreshLockOutcome } from './cache-refresh-lock.js' @@ -2057,13 +2062,15 @@ async function scanProjectDirs( if (allDiscoveredFiles.has(filePath)) continue if (!readOnly && !cached.prLinks?.length) continue const dirName = cached.canonicalProjectName - ?? cached.turns[0]?.calls[0]?.project + ?? fileFirstTurnProject(cached) ?? basename(dirname(filePath)) unchangedFiles.push({ filePath, dirName, cached }) } // Pre-seed dedup set from cached (unchanged) files for (const { cached } of unchangedFiles) { + // Dropped turns contribute only dedup keys (see RangeFilteredMeta). + seedDroppedKeys(seenMsgIds, cached) for (const turn of cached.turns) { for (const call of turn.calls) { seenMsgIds.add(call.deduplicationKey) @@ -2350,12 +2357,12 @@ async function scanProjectDirs( // stores a turn's branch only when it changes, so resolving here (over the // full ordered turn list) means a later date slice can drop the anchor turn // without the surviving turns losing their branch. - let carriedBranch: string | undefined + let carriedBranch: string | undefined = cachedFile.rangeFiltered?.carryBranch // The PR set active going into the report range: carried across the FULL turn // list, frozen the moment the first in-range turn is reached. Lets per-turn PR // attribution seed from a reference made before the window (see // attributeSessionPrSpend); the branch carry above solves the same problem. - let carriedPrRefs: string[] | undefined + let carriedPrRefs: string[] | undefined = cachedFile.rangeFiltered?.carryPrRefs let prRefsAtRangeStart: string[] | undefined let frozePrRefs = !dateRange // The keep/drop decision is taken on the RAW turn, before classifying it: @@ -2385,7 +2392,9 @@ async function scanProjectDirs( // Captured from the FULL turn list, which the date slice above can strip of // the turn a branch was first seen on. Lets the by-branch report keep this // session's in-range unbranched spend as `null` instead of discarding it. - const everHadBranch = carriedBranch !== undefined + // A filtered load initializes the carries below from the dropped prefix; + // droppedHadBranch covers branches that only ever appear out of range. + const everHadBranch = carriedBranch !== undefined || (cachedFile.rangeFiltered?.droppedHadBranch ?? false) // Built from the FULL (pre-slice) turn list: each subagent-spawn tool_use id -> // the PR set active at the turn that emitted it. Lets a subagent fold into the @@ -3293,6 +3302,14 @@ function turnSlicedToRange(turn: CachedTurn, dateRange: DateRange): CachedTurn | return { ...turn, calls: inRangeCalls, timestamp: inRangeCalls[0]!.timestamp } } +/// Load-time keep/drop rule for one cached turn under a date range: true when +/// any call intersects (the same null-rule as turnSlicedToRange, which the +/// serve loops still apply themselves for call trimming and classification). +/// Exported so range-filtered loads project exactly what serving would keep. +export function turnIntersectsRange(turn: CachedTurn, dateRange: DateRange): boolean { + return turnSlicedToRange(turn, dateRange) !== null +} + // Same slice, applied post-classification (scanProjectDirs classifies each // surviving turn from its FULL call list, before date filtering — see the // carriedBranch/carriedPrRefs comments in scanProjectDirs — so this only @@ -3433,7 +3450,7 @@ export async function parseProviderSources( servedSources.push({ provider: providerName, path, - project: cached.turns[0]?.calls[0]?.project ?? providerName, + project: fileFirstTurnProject(cached) ?? providerName, }) allDiscoveredFiles.add(path) unchangedSources.push({ source: servedSources[servedSources.length - 1]!, cached }) @@ -3457,8 +3474,11 @@ export async function parseProviderSources( // Parser dedup: cross-provider keys + cached file keys. // Separate from seenKeys so parsing doesn't suppress query-time output. - const parserDedup = new Set(seenKeys) + const parserDedup = new Set() + for (const key of seenKeys) parserDedup.add(key) for (const { cached } of unchangedSources) { + // Dropped turns contribute only dedup keys (see RangeFilteredMeta). + seedDroppedKeys(parserDedup, cached) for (const turn of cached.turns) { for (const call of turn.calls) { parserDedup.add(call.deduplicationKey) @@ -3483,7 +3503,9 @@ export async function parseProviderSources( if (providerName === 'codex' && !readOnly) { for (const { source, fp } of changedSources) { if (dateRange && fp.mtimeMs < dateRange.start.getTime()) continue - if (await readCachedCodexResults(source.path)) continue + // PR-A scope: unfiltered codex-result lookup (range-filtered serve of + // the codex cache rides the stacked codex-cache rewrite). + if (await readCachedCodexResults(source.path)) continue workerJobs.push({ kind: 'codex', source }) workerPaths.add(source.path) pendingBytes += fp.sizeBytes @@ -3752,11 +3774,7 @@ export async function parseProviderSources( const cutoffMs = Date.now() - 90 * 24 * 60 * 60 * 1000 for (const [cachedPath, cachedFile] of Object.entries(section.files)) { if (retainPaths.has(cachedPath)) continue - const newestTs = cachedFile.turns - .flatMap(t => t.calls) - .map(c => new Date(c.timestamp).getTime()) - .filter(ts => !isNaN(ts)) - .reduce((max, ts) => Math.max(max, ts), 0) + const newestTs = fileNewestCallMs(cachedFile) if (!allDiscoveredFiles.has(cachedPath) && newestTs > 0 && newestTs < cutoffMs) { delete section.files[cachedPath] markCacheDirty(diskCache, providerName, cachedPath) @@ -4006,7 +4024,10 @@ export async function parseProviderSources( for (const source of servedSources) { const cachedFile = section.files[source.path] if (!cachedFile) continue - + // Dropped turns contribute only dedup keys (see RangeFilteredMeta): the + // full walk would have added them at this same file position, so later + // files suppress identically. + seedDroppedKeys(seenKeys, cachedFile) for (const rawTurn of cachedFile.turns) { const turn = reconcileCopilotCalls(rawTurn) if (!turn) continue @@ -4082,7 +4103,9 @@ export async function parseProviderSources( if (provider.durableSources) { for (const [cachedPath, cachedFile] of Object.entries(section.files)) { if (allDiscoveredFiles.has(cachedPath)) continue // already counted above - + // Dropped turns contribute only dedup keys (see RangeFilteredMeta): the + // full walk would have added them at this same file position. + seedDroppedKeys(seenKeys, cachedFile) for (const rawTurn of cachedFile.turns) { const turn = reconcileCopilotCalls(rawTurn) if (!turn) continue @@ -5450,6 +5473,19 @@ export async function isCompleteSessionSnapshotAvailable(dateRange: DateRange, p return canServeCompleteSnapshot(diskCache, providerFilter, dateRange.start.getTime()) } +/// Projection policy for ranged session-cache loads (see TurnFilter): one +/// shared predicate feeds the initial load, the hydration reload and every +/// refresh/lock retry in this parse, so all of them see the identical slice. +/// Undefined without a range AND without a provider selection; +/// snapshot-completeness checks bypass turn projection (never provider scope). +function cacheLoadOpts(dateRange: DateRange | undefined, providerFilter?: string): LoadCacheOptions | undefined { + if (!dateRange) return providerFilter === undefined ? undefined : { providerFilter } + return { + turnFilter: turn => turnIntersectsRange(turn, dateRange), + ...(providerFilter !== undefined ? { providerFilter } : {}), + } +} + async function parseAllSessionsInCacheScope(dateRange?: DateRange, providerFilter?: string): Promise { // Anchor freshness before any config, cache, or session input is read. A // watched-root event that lands while this parse is in flight must remain @@ -5500,7 +5536,7 @@ async function parseAllSessionsInCacheScope(dateRange?: DateRange, providerFilte const loadScope = dateRange ? monthScopeForRange(dateRange.start, dateRange.end) : undefined const rangeStartMs = dateRange?.start.getTime() const cacheLoadStarted = performance.now() - let diskCache = await loadCache(loadScope) + let diskCache = await loadCache(loadScope, cacheLoadOpts(dateRange, providerFilter)) await cleanupOrphanedTempFiles() if (process.env['CODEBURN_VERBOSE'] === '1') { process.stderr.write(`codeburn: startup timing cache-load=${(performance.now() - cacheLoadStarted).toFixed(1)}ms complete=${isCacheComplete(diskCache, providerFilter, rangeStartMs)}\n`) diff --git a/src/providers/antigravity.ts b/src/providers/antigravity.ts index ede51db66..09f2acbf2 100644 --- a/src/providers/antigravity.ts +++ b/src/providers/antigravity.ts @@ -1246,6 +1246,8 @@ function parseStatusLineEvent(input: unknown): StatusLineEvent | null { } } +/// True when the shared dedup set already holds an RPC-cache entry for this +/// conversation (raw prefix scan over the exact keys). function hasRpcCacheForConversation(seenKeys: Set, conversationId: string): boolean { const prefix = `antigravity:${conversationId}:` for (const key of seenKeys) { @@ -1254,10 +1256,10 @@ function hasRpcCacheForConversation(seenKeys: Set, conversationId: strin return false } + async function parseStatusLineCalls(source: SessionSource, seenKeys: Set): Promise { const raw = await readFile(source.path, 'utf-8').catch(() => '') const runsByConversation = new Map>() - for (const line of raw.split(/\r?\n/)) { if (!line.trim()) continue let parsed: unknown diff --git a/src/providers/codex.ts b/src/providers/codex.ts index 64dfe4484..b8fe57ef4 100644 --- a/src/providers/codex.ts +++ b/src/providers/codex.ts @@ -701,10 +701,12 @@ export type CodexCacheWrite = { // written comes back through `capture` for the caller to install. That is what // lets a worker thread run this exact decode without owning the cache module's // per-directory state. -function createParser(source: SessionSource, seenKeys: Set, capture?: { write?: CodexCacheWrite }): SessionParser { +function createParser(source: SessionSource, seenKeys: Set, capture?: { write?: CodexCacheWrite }, rangeStartMs?: number): SessionParser { return { async *parse(): AsyncGenerator { - const hit = capture ? null : await readCachedCodexResults(source.path) + // PR-A scope: unfiltered lookup (range-filtered serve rides the stacked + // codex-cache rewrite); downstream turn slicing still bounds the report. + const hit = capture ? null : await readCachedCodexResults(source.path) if (hit?.kind === 'exact') { for (const call of hit.calls) { if (seenKeys.has(call.deduplicationKey)) continue @@ -1438,8 +1440,12 @@ export function createCodexProvider( return dropOverlappingNestSources(sources, primaryDir) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { - return createParser(source, seenKeys) + createSessionParser( + source: SessionSource, + seenKeys: Set, + dateRange?: { start: Date; end: Date }, + ): SessionParser { + return createParser(source, seenKeys, undefined, dateRange?.start.getTime()) }, } } diff --git a/src/providers/cursor.ts b/src/providers/cursor.ts index b38aee54f..ad0248373 100644 --- a/src/providers/cursor.ts +++ b/src/providers/cursor.ts @@ -1008,10 +1008,10 @@ function createParser( process.stderr.write('codeburn: Cursor storage format not recognized. You may need to update CodeBurn.\n') return } - // Use a fresh local Set for intra-parse dedup so the global - // seenKeys is not mutated by calls that the workspace filter is - // about to drop. Cross-source dedup happens at yield time. - const localSeen = new Set() + // Use a fresh local Set for intra-parse dedup so the global + // seenKeys is not mutated by calls that the workspace filter is + // about to drop. Cross-source dedup happens at yield time. + const localSeen = new Set() // agentKv rows carry no timestamps; sessions found only there get // the DB's last-write time. let agentKvTimestamp: string diff --git a/src/providers/types.ts b/src/providers/types.ts index 742340b40..aae70a391 100644 --- a/src/providers/types.ts +++ b/src/providers/types.ts @@ -1,4 +1,4 @@ -import type { DateRange, ToolCall } from '../types.js' + import type { DateRange, ToolCall } from '../types.js' export type SessionSource = { path: string @@ -129,7 +129,7 @@ export type Provider = { // Report once per excluded session, independently of deduplicated warnings. // The callback belongs to this scan, avoiding stale/shared diagnostic counts. discoverSessions(onSkippedVersion?: (version: number) => void): Promise - createSessionParser(source: SessionSource, seenKeys: Set, dateRange?: DateRange): SessionParser + createSessionParser(source: SessionSource, seenKeys: Set, dateRange?: DateRange): SessionParser // The exact directories/dbs discoverSessions() scans, resolved the same way. // Optional: providers that implement it let `codeburn doctor` show and // existence-check the probed paths even when zero sessions are found (so diff --git a/src/session-cache.ts b/src/session-cache.ts index e491fe9ba..8309d568a 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -4,8 +4,10 @@ import { createHash, randomBytes } from 'crypto' import { join } from 'path' import { getCodeburnCacheDir } from './cache-dir.js' +import { flattenJsonStrings } from './content-utils.js' import { acquireCacheRefreshLock, releaseOwnedRefreshLocksForExit } from './cache-refresh-lock.js' import type { ToolCall } from './types.js' +import { streamShardArrayField, streamShardEntries } from './shard-stream.js' // ── Types ────────────────────────────────────────────────────────────── @@ -105,6 +107,65 @@ export type FileFingerprint = { sizeBytes: number } +/// Keep-or-drop predicate over a cached turn, decided by the caller that owns +/// the query range. Dropping is turn-granular only: a kept turn is always the +/// WHOLE turn (all calls), because `cachedTurnToClassified` classifies from +/// the full call list and the serve loops trim calls to the range themselves. +/// A predicate that trims calls would silently change activity/retry/tool +/// categories on midnight-straddling turns. +export type TurnFilter = (turn: CachedTurn) => boolean + +/// Options for loadCache. `turnFilter` is the ranged-query projection: when +/// present (and the provider is neither durable nor fingerprint-mismatched), +/// each shard is streamed and only turns the predicate keeps are retained. +/// `providerFilter` narrows the load to one provider (`'all'` or absent loads +/// everything): other sections come back as empty shells — no shards read — +/// so a provider-scoped query never decodes unrelated corpora. Shells still +/// carry their envelope refs through the next save untouched (see the +/// unloaded-months carry in saveCache). Cross-provider dedup seeding and +/// cross-provider PR correlation only see loaded providers, the same class of +/// weakening month scoping already accepts; unscoped runs are unaffected. +export type LoadCacheOptions = { + turnFilter?: TurnFilter + providerFilter?: string +} + +/// Metadata carried on a file whose turns were narrowed by a TurnFilter at +/// load. The on-disk shard bytes stay authoritative for everything dropped: +/// records carrying this marker must never be serialized back into a shard +/// (see saveCache), and a hypothetical on-disk record carrying it fails +/// validation (see validateCachedFile), so the marker is memory-only in both +/// directions. Every read path below consults these scalars instead of the +/// absent turns. +export type RangeFilteredMeta = { + /// Pre-filter month span: `bucket` is the oldest turn's month (the shard the + /// file lives in), `until` the newest. Appends never move a file, so the + /// span recorded at decode stays correct while the file is unchanged. + span: { bucket: string; until: string } + /// Newest call timestamp in the FULL pre-filter turn list (for the 90-day + /// durable age-out, which reads past the retained turns). + newestCallMs: number + /// `turns[0].calls[0].project` of the FULL pre-filter list (for orphan + /// identity fallbacks that read past the retained turns). + firstTurnProject?: string + /// Deduplication keys of the dropped turns, in file walk order. Added to + /// the shared dedup sets wherever the serve loops open the file. Exact if + /// and only if no key appears in both a kept and a dropped turn of this + /// file — the decoder keeps the whole file (unflagged) on any such + /// overlap, so a retained turn can never be suppressed by a dropped later + /// duplicate (or vice versa) differently than the full walk would. + droppedKeys: string[] + /// Branch/PR state carried into the first kept turn, walked from the dropped + /// Exact on append-only transcripts, where kept turns always form a suffix; + /// files whose dropped turns also carry branch/PR state between kept turns + /// (intra-file time disorder) are kept whole instead (see above). + carryBranch?: string + carryPrRefs?: string[] + /// Any dropped turn carried a git branch: feeds `everHadBranch` exactly, + /// since that judgment is order-independent (anywhere in the file counts). + droppedHadBranch?: boolean +} + export type CachedFile = { fingerprint: FileFingerprint lastCompleteLineOffset?: number @@ -114,6 +175,12 @@ export type CachedFile = { canonicalProjectName?: string mcpInventory: string[] turns: CachedTurn[] + // Set only by a range-filtered load (see TurnFilter): the turns list holds + // just the in-range turns, and this marker carries everything the dropped + // turns contributed (dedup keys, span, scalars). Memory-only: the on-disk + // shard bytes stay authoritative, so save paths must exclude flagged + // records from every write set (see saveCache) or history is truncated. + rangeFiltered?: RangeFilteredMeta // Claude Code only: for a subagent transcript (`subagents/.../agent-*.jsonl`), // the `agentType` from its sibling `.meta.json` (e.g. `workflow-subagent`, // `Explore`, `general-purpose`). Drives the Claude-scoped agent-type breakdown. @@ -529,6 +596,46 @@ export function cacheBucketMonth(file: CachedFile): string { return cacheFileSpan(file).bucket } +/// Month span honoring a range-filtered record's carried span. Filtered files +/// keep only in-range turns, so recomputing from `turns` would misbucket them; +/// the span recorded at decode (over the full pre-filter list) stays correct +/// while the file is unchanged, and a re-parse replaces the record wholesale. +export function fileSpan(file: CachedFile): { bucket: string; until: string } { + return file.rangeFiltered?.span ?? cacheFileSpan(file) +} + +/// Newest call timestamp in milliseconds, mirroring the 90-day durable age-out +/// computation exactly (NaN stamps ignored, 0 when none) for unfiltered files. +export function fileNewestCallMs(file: CachedFile): number { + if (file.rangeFiltered) return file.rangeFiltered.newestCallMs + let max = 0 + for (const turn of file.turns) { + for (const call of turn.calls) { + const ts = new Date(call.timestamp).getTime() + if (!Number.isNaN(ts) && ts > max) max = ts + } + } + return max +} + +/// First turn's first-call project of the FULL pre-filter list. Serve paths +/// fall back to it when the retained turns start later (or are gone). +export function fileFirstTurnProject(file: CachedFile): string | undefined { + return file.rangeFiltered?.firstTurnProject ?? file.turns[0]?.calls[0]?.project +} + +/// Add a filtered file's dropped dedup keys to a shared set. Call sites must +/// invoke this when the serve walk OPENS the file (before its turns), in walk +/// order: that reproduces cross-file suppression exactly, because the full +/// walk adds an out-of-range turn's keys at the same file position. Within a +/// file the decoder keeps the record whole on any kept/dropped key overlap, +/// so no reordering hazard remains. No-op for unfiltered files. +export function seedDroppedKeys(seen: Set, file: CachedFile): void { + const marker = file.rangeFiltered + if (!marker) return + for (const key of marker.droppedKeys) seen.add(key) +} + // Save bookkeeping, held beside the cache rather than on it so it never lands in // a shard's JSON or in a caller's deep-equality. type CacheState = { @@ -546,6 +653,8 @@ type CacheState = { bucketOf: Map /** The load scope this cache was read under, for the cross-request memo. */ scope: string + /** The provider this cache was narrowed to (`'all'` when unscoped). */ + scopeProvider: string } const cacheStates = new WeakMap() @@ -560,6 +669,7 @@ function stateOf(cache: SessionCache): CacheState { fingerprints: new Map(), bucketOf: new Map(), scope: 'all', + scopeProvider: 'all', } cacheStates.set(cache, state) } @@ -589,7 +699,10 @@ export function markCacheDirty(cache: SessionCache, provider: string, filePath?: const prior = state.bucketOf.get(`${provider}\0${filePath}`) if (prior !== undefined) markBucketDirty(state, provider, prior) const file = cache.providers[provider]?.files[filePath] - if (file) markBucketDirty(state, provider, cacheBucketMonth(file)) + // fileSpan, not cacheBucketMonth: a range-filtered record's turns are a + // slice, so recomputing the bucket from them would dirty the wrong month + // and resurrect a deleted file from the carried shard on the next save. + if (file) markBucketDirty(state, provider, fileSpan(file).bucket) // A path with neither a prior bucket nor a live entry (deleted before this // process ever saw it) still has to move `dirty`, or the save is skipped. state.dirty = true @@ -822,6 +935,12 @@ function validateCachedFile(f: unknown): f is CachedFile { && isOptionalLineage(o['lineage']) && Array.isArray(o['turns']) && (o['turns'] as unknown[]).every(validateTurn) + // `rangeFiltered` is memory-only (see RangeFilteredMeta): a record carrying + // it on disk is either corrupt or from a writer bug, so reject the file + // rather than serving a partial turn list as complete. Save paths exclude + // flagged records from every write set, so this branch is unreachable + // through the shipped code — it exists to fail closed, not to trigger. + && o['rangeFiltered'] === undefined } // A shard's payload: the provider's `files` map, restricted to one month. @@ -938,7 +1057,7 @@ async function adoptNewestPriorCache(): Promise { // process mints a new nonce and forces a reload, so cross-process freshness is // preserved; saveCache updates the memo write-through so the object handed out // stays the canonical one after a refresh. -let cacheMemo: { dir: string; nonce: string; scope: string; cache: SessionCache } | null = null +let cacheMemo: { dir: string; nonce: string; scope: string; provider: string; cache: SessionCache } | null = null export function clearLoadCacheMemo(): void { cacheMemo = null @@ -1010,13 +1129,31 @@ async function readEnvelope(dir: string): Promise { // A shard that is missing or malformed costs exactly the provider-months it // held, not the provider and never the whole cache: those files re-parse while // every other month keeps serving. -async function loadShard(path: string): Promise | null> { +/// Assemble a whole shard without ever materializing its text: files past +/// ~512MB exceed V8's max string length, so readFile+JSON.parse hard-fails +/// (RangeError) regardless of heap. Per-record streaming bounds transients by +/// the largest single record instead. Any invalid record drops the whole +/// shard, mirroring validateFiles entry for entry. +async function loadShardStreaming(path: string): Promise | null> { + const files: Record = {} try { - const parsed = JSON.parse(await readFile(path, 'utf-8')) - return validateFiles(parsed) ? parsed : null + await streamShardEntries(path, ({ key, value }) => { + if (!validateCachedFile(value)) throw new Error(`shard record invalid: ${path}`) + // Detach before storing: unlike streamShardArrayField (whose + // assembleTokens detaches at the token level), this pipeline is + // parser→streamObject direct, so values arrive as tokenizer slices. + // In place on the decoder-owned value — no copy, no serialization + // transient (see flattenJsonStrings). Keys arrive pre-flattened. + files[key] = flattenJsonStrings(value) + }) } catch { return null } + return files +} + +async function loadShard(path: string): Promise | null> { + return loadShardStreaming(path) } // Shards a resident process (codeburn serve) keeps parsed between requests, @@ -1070,54 +1207,490 @@ export async function loadShardMemoized(dir: string, name: string): Promise null) + // Vanished mid-load: serve the self-consistent bytes once, but never + // memoize them (a name-keyed memo would serve ghost data afterwards). + if (!st) return files + const bytes = st.size + shardMemo.set(key, { files, bytes, usedAt: now }) + shardMemoBytes += bytes + evictShardMemo(now) + return files +} + +type SliceItem = { + keep: boolean + turn: CachedTurn | null + /// Dedup keys of this turn's calls, for the overlap check and the marker. + keys: string[] + gitBranch?: string + prRefs?: string[] + month: string | null + newestMs: number +} +function turnNewestCallMs(turn: CachedTurn): number { + let max = 0 + for (const call of turn.calls) { + const ts = new Date(call.timestamp).getTime() + if (!Number.isNaN(ts) && ts > max) max = ts } - let files: Record + return max +} + +function droppedTurnKeys(turn: CachedTurn, into: string[]): void { + for (const call of turn.calls) into.push(call.deduplicationKey) +} + +function sliceItemFor(turn: CachedTurn, keep: boolean): SliceItem { + const keys: string[] = [] + droppedTurnKeys(turn, keys) + return { + keep, + turn: keep ? turn : null, + keys, + gitBranch: turn.gitBranch, + prRefs: turn.prRefs, + month: monthKey(turn.timestamp), + newestMs: turnNewestCallMs(turn), + } +} + +/// Incremental projection accumulator: the common-path replacement for +/// collecting one SliceItem per turn. For a 30k-turn file that was 30k +/// wrappers plus key arrays retained to file end; the accumulator keeps +/// only what the marker and output need — kept turns, dropped keys, kept +/// keys, counters, and scalar carry/suffix state. Decision logic lives in +/// finishSlice (shared with projectTurnSlice), so the second-pass fallback +/// in filterShardFile behaves identically. +type SliceAccumulator = { + kept: CachedTurn[] + droppedKeys: string[] + keptKeys: Set + total: number + keptCount: number + firstKept: number + lastKept: number + seenKept: boolean + carryBranch: string | undefined + carryPrRefs: string[] | undefined + prefixHadBranch: boolean + suffixHadBranch: boolean + anyBranch: boolean +} + +function createSliceAccumulator(): SliceAccumulator { + return { + kept: [], + droppedKeys: [], + keptKeys: new Set(), + total: 0, + keptCount: 0, + firstKept: -1, + lastKept: -1, + seenKept: false, + carryBranch: undefined, + carryPrRefs: undefined, + prefixHadBranch: false, + suffixHadBranch: false, + anyBranch: false, + } +} + +/// Fold one live turn with zero per-turn allocation beyond the key strings +/// themselves: keys stream straight into the shared arrays, scalars update +/// counters. Mirrors sliceItemFor plus the carry and suffix-branch scans +/// positionally (last-wins carry, suffix reset on kept). +function foldLiveTurn(acc: SliceAccumulator, turn: CachedTurn, keep: boolean): void { + if (keep) { + const index = acc.total++ + acc.kept.push(turn) + for (const call of turn.calls) { + acc.keptKeys.add(call.deduplicationKey) + } + acc.keptCount++ + if (!acc.seenKept) { + acc.seenKept = true + acc.firstKept = index + } + acc.lastKept = index + // The suffix scan restarts at the newest kept turn. + acc.suffixHadBranch = turn.gitBranch ? true : false + if (turn.gitBranch) acc.anyBranch = true + return + } + acc.total++ + for (const call of turn.calls) { + acc.droppedKeys.push(call.deduplicationKey) + } + if (!acc.seenKept) { + if (turn.gitBranch) { + acc.prefixHadBranch = true + acc.carryBranch = turn.gitBranch + } + if (turn.prRefs?.length) acc.carryPrRefs = turn.prRefs + } else if (turn.gitBranch) { + acc.suffixHadBranch = true + } + if (turn.gitBranch) acc.anyBranch = true +} + +/// Fold one pre-built SliceItem (second-pass path only): same scalar state +/// as foldLiveTurn, reading the item's stored arrays instead of the turn. +function foldItem(acc: SliceAccumulator, item: SliceItem): void { + if (item.keep) { + const index = acc.total++ + acc.kept.push(item.turn!) + for (const key of item.keys) acc.keptKeys.add(key) + acc.keptCount++ + if (!acc.seenKept) { + acc.seenKept = true + acc.firstKept = index + } + acc.lastKept = index + acc.suffixHadBranch = item.gitBranch ? true : false + if (item.gitBranch) acc.anyBranch = true + return + } + acc.total++ + for (const key of item.keys) acc.droppedKeys.push(key) + if (!acc.seenKept) { + if (item.gitBranch) { + acc.prefixHadBranch = true + acc.carryBranch = item.gitBranch + } + if (item.prRefs?.length) acc.carryPrRefs = item.prRefs + } else if (item.gitBranch) { + acc.suffixHadBranch = true + } + if (item.gitBranch) acc.anyBranch = true +} + +/// Shared projection decision + marker assembly. Contiguity and key-overlap +/// match projectTurnSlice exactly; the all-dropped file keeps the suffix +/// scan's whole-list coverage via anyBranch. +function finishSlice( + base: Record, + acc: SliceAccumulator, + span: { bucket: string; until: string }, + newestMs: number, + firstTurnProject: string | undefined, +): { whole: true } | { whole: false; file: CachedFile } { + if (acc.keptCount > 0 && acc.lastKept - acc.firstKept + 1 !== acc.keptCount) return { whole: true } + for (const key of acc.droppedKeys) if (acc.keptKeys.has(key)) return { whole: true } + const droppedHadBranch = acc.prefixHadBranch || (acc.keptCount === 0 ? acc.anyBranch : acc.suffixHadBranch) + // No flattening here either: every turn passing through the accumulator + // was assembled by assembleTokens (already detached), so carry refs, first + // project, and kept turns retain directly. + return { + whole: false, + file: { + ...base, + turns: acc.kept, + rangeFiltered: { + span, + newestCallMs: newestMs, + ...(firstTurnProject !== undefined ? { firstTurnProject } : {}), + droppedKeys: acc.droppedKeys, + // Carry needs a kept block to carry INTO: with zero kept turns the + // old prefix loop ran zero times (firstKept === -1), so the fields + // stay absent. droppedHadBranch still reports via anyBranch. + ...(acc.keptCount > 0 && acc.carryBranch !== undefined ? { carryBranch: acc.carryBranch } : {}), + ...(acc.keptCount > 0 && acc.carryPrRefs !== undefined ? { carryPrRefs: acc.carryPrRefs } : {}), + ...(droppedHadBranch ? { droppedHadBranch: true } : {}), + }, + } as CachedFile, + } +} + +/// Project one decoded shard record through a TurnFilter. Returns the record +/// unchanged (no marker) unless turns are actually dropped. Projection applies +/// only when it is exact, otherwise the caller must keep the whole record: +/// - PR-linked files stay whole: the PR-anchor logic reads spawn sets across +/// the full turn list (checked by the caller, which sees the metadata). +/// - kept turns must form ONE contiguous block in file order (a prefix, a +/// suffix, or all of them): the carry state below is walked positionally. +/// Append-only layouts yield a kept suffix; newest-first layouts a prefix. +/// - no dedup key may appear on both sides: the shared sets suppress in walk +/// order, and only single-sided keys preserve that order. +/// Kept turns are always whole objects — calls are never trimmed here, so +/// serve-time classification sees the identical input (see TurnFilter). +/// `span`/`newestMs`/`firstTurnProject` are aggregates over the FULL +/// pre-filter turn list (the marker carries them past the dropped turns). +function projectTurnSlice( + base: Record, + items: SliceItem[], + span: { bucket: string; until: string }, + newestMs: number, + firstTurnProject: string | undefined, +): { whole: true } | { whole: false; file: CachedFile } { + const keepFlags = items.map(item => item.keep) + const keptCount = keepFlags.filter(keep => keep).length + if (keptCount === items.length) { + return { whole: false, file: { ...base, turns: items.map(item => item.turn!) } as CachedFile } + } + const acc = createSliceAccumulator() + for (const item of items) foldItem(acc, item) + return finishSlice(base, acc, span, newestMs, firstTurnProject) +} + +function filterShardFile(value: unknown, keepTurn: TurnFilter): CachedFile | null { + if (!validateCachedFile(value)) return null + const file = value + if (file.turns.length === 0) return file + if (file.prLinks?.length) return file + const items = file.turns.map(turn => sliceItemFor(turn, keepTurn(turn))) + if (items.every(item => item.keep)) return file + const sliced = projectTurnSlice( + file, + items, + cacheFileSpan(file), + fileNewestCallMs(file), + file.turns[0]?.calls[0]?.project, + ) + return sliced.whole ? file : sliced.file +} + +/// Decode one shard with a TurnFilter, streaming turns so peak heap tracks +/// the retained slice plus one turn rather than the shard. Turns are +/// validated as decoded (kept or dropped); the unmarked reconstruction is +/// validated at file end, so an invalid record anywhere drops the whole +/// shard exactly as the whole-file decoder did. Any decode failure (including +/// a single invalid record) drops the WHOLE shard — mirroring validateFiles — +/// so a torn shard can never partially commit. +/// +/// Whole-file gates (PR links, gappy kept turns, cross-side key overlap) are +/// collected during the stream and decoded whole in one second pass over the +/// shard: rare cases pay a re-read while the common path stays bounded. +export async function loadShardFiltered( + dir: string, + name: string, + turnFilter: TurnFilter, +): Promise | null> { + const files: Record = {} + const needsFull = new Set() + type Build = { + meta: Record + acc: SliceAccumulator + spanMin: string | null + spanMax: string | null + newestMs: number + firstTurnProject?: string + } + const builds = new Map() + const shardPath = join(dir, name) + const fingerprintOf = async (): Promise => + stat(shardPath).then(s => `${s.dev}:${s.ino}:${s.size}:${s.mtimeMs}`, () => null) + const before = await fingerprintOf() try { - const parsed = JSON.parse(raw) - if (!validateFiles(parsed)) return null - files = parsed + await streamShardArrayField(shardPath, 'turns', { + onFileStart: key => { + builds.set(key, { meta: {}, acc: createSliceAccumulator(), spanMin: null, spanMax: null, newestMs: 0 }) + }, + onField: (key, field, value) => { + // `rangeFiltered` is memory-only: a record carrying it on disk is + // corrupt or a writer bug, and the whole-file decoder rejects it, so + // fail the same way instead of serving a partial list as complete. + if (field === 'rangeFiltered') throw new Error(`shard record invalid: ${name}`) + // The turns array streams element-wise; a field-shaped `turns` value + // is corrupt (or a duplicate key). The whole-file decoder failed + // validation here, so fail the same way instead of serving metadata + // without turns as complete. + if (field === 'turns') throw new Error(`shard record invalid: ${name}`) + builds.get(key)!.meta[field] = value + }, + onElement: (key, index, value) => { + const build = builds.get(key)! + if (!validateTurn(value)) throw new Error(`shard record invalid: ${name}`) + const turn = value + // Fold straight into the accumulator: no per-turn wrapper or key + // arrays (see SliceAccumulator). Scalars mirror sliceItemFor. + foldLiveTurn(build.acc, turn, turnFilter(turn)) + const month = monthKey(turn.timestamp) + if (month !== null) { + if (build.spanMin === null || month < build.spanMin) build.spanMin = month + if (build.spanMax === null || month > build.spanMax) build.spanMax = month + } + const newestMs = turnNewestCallMs(turn) + if (newestMs > build.newestMs) build.newestMs = newestMs + if (index === 0) build.firstTurnProject = turn.calls[0]?.project + }, + onFileEnd: (key, _count, arraySeen) => { + const build = builds.get(key)! + builds.delete(key) + // No turns field at all: the whole-file decoder failed validation + // here, so fail the same way. + if (!arraySeen) throw new Error(`shard record invalid: ${name}`) + if ((build.meta['prLinks'] as unknown[] | undefined)?.length) { + needsFull.add(key) + return + } + // Nothing dropped: serve the kept turns with no marker (the contract + // adds rangeFiltered only when turns were actually dropped). + if (build.acc.keptCount === build.acc.total) { + const file = { ...build.meta, turns: build.acc.kept } as CachedFile + if (!validateCachedFile(file)) throw new Error(`shard record invalid: ${name}`) + files[key] = file + return + } + const span = build.spanMin === null + ? { bucket: UNDATED_BUCKET, until: UNDATED_BUCKET } + : { bucket: build.spanMin, until: build.spanMax! } + const sliced = finishSlice({ ...build.meta }, build.acc, span, build.newestMs, build.firstTurnProject) + if (sliced.whole) { + needsFull.add(key) + return + } + // The marker is memory-only and fails validation by design, so + // validate the unmarked reconstruction: every turn was already + // validated at decode, which leaves exactly the metadata covered. + const { rangeFiltered: _marker, ...rest } = sliced.file + if (!validateCachedFile(rest)) throw new Error(`shard record invalid: ${name}`) + files[key] = sliced.file + }, + }) } catch { return null } - const bytes = Buffer.byteLength(raw) - shardMemo.set(key, { files, bytes, usedAt: now }) - shardMemoBytes += bytes - evictShardMemo(now) + if (needsFull.size > 0) { + // A concurrent atomic publish between the passes would apply pass-one + // decisions to pass-two bytes (or silently miss a needsFull key), so + // prove the file is untouched and every key completed, else drop the + // shard exactly as a torn file would. + const after = await stat(shardPath).then( + s => `${s.dev}:${s.ino}:${s.size}:${s.mtimeMs}`, + () => null, + ) + if (after === null || after !== before) return null + const completed = new Set() + const wholeBuilds = new Map; turns: CachedTurn[] }>() + try { + await streamShardArrayField(shardPath, 'turns', { + onFileStart: key => { + if (needsFull.has(key)) wholeBuilds.set(key, { meta: {}, turns: [] }) + }, + onField: (key, field, value) => { + const build = wholeBuilds.get(key) + if (build) build.meta[field] = value + }, + onElement: (key, _index, value) => { + const build = wholeBuilds.get(key) + if (!build) return + if (!validateTurn(value)) throw new Error(`shard record invalid: ${name}`) + build.turns.push(value) + }, + onFileEnd: (key, _count, arraySeen) => { + const build = wholeBuilds.get(key) + if (!build) return + wholeBuilds.delete(key) + if (!arraySeen) throw new Error(`shard record invalid: ${name}`) + const file = filterShardFile({ ...build.meta, turns: build.turns }, turnFilter) + if (!file) throw new Error(`shard record invalid: ${name}`) + files[key] = file + completed.add(key) + }, + }) + } catch { + return null + } + for (const key of needsFull) { + if (!completed.has(key)) return null + } + // A replacement during pass two would mix old decisions with new bytes. + if ((await fingerprintOf()) !== before) return null + } return files } +/// Strip range-filtered records from a write payload. Filtered records carry +/// only a slice of their turns; persisting them would truncate history. The +/// published shard bytes stay authoritative for stripped paths (see the merge +/// path in saveCache, which overlays memory records onto the published shard +/// instead of rewriting from memory alone). +function stripFilteredForWrite(files: Record): Record { + let stripped: Record | null = null + for (const file of Object.values(files)) { + if (!file.rangeFiltered) continue + if (!stripped) { + stripped = {} + for (const [p, f] of Object.entries(files)) { + if (!f.rangeFiltered) stripped[p] = f + } + break + } + } + return stripped ?? files +} + +/// Whether any record in a write payload carries the range-filtered marker. +function groupHasFiltered(files: Record): boolean { + return Object.values(files).some(f => f.rangeFiltered !== undefined) +} + +/// Whether any record anywhere in the cache carries the range-filtered +/// marker. Guards the save write-through memo (a filtered object must never +/// be served to a later full-scope load in this process). +function cacheHasFilteredRecords(cache: SessionCache): boolean { + return Object.values(cache.providers).some(section => + Object.values(section.files).some(file => file.rangeFiltered !== undefined)) +} + /** * Read the cache. With a `scope`, only the shards whose months can contribute a * turn to that range are read — everything else stays on disk and is carried - * across the next save untouched (see saveCache). Durable providers and any - * provider whose recorded fingerprint no longer matches are always read in - * full: the first because its cache is the only surviving record of pruned - * usage, the second because a fingerprint change discards the whole section and - * must see every entry it is discarding. +* across the next save untouched (see saveCache). Durable providers are always +* read in full: their cache is the only surviving record of pruned usage. A +* provider whose recorded fingerprint no longer matches is likewise never +* scoped: it is read in full so migration and lossless loads see every entry +* (orphaned paths cannot re-parse), and so the fingerprint reset inspects +* the complete prior section before the parse replaces it. Durable sections +* additionally carry their orphans forward across the reset itself (see +* getOrCreateProviderSection); non-durable entries are re-derived by the +* re-parse that follows. * * `CODEBURN_CACHE_SCOPE=all` is the escape hatch: it drops the scope here, at * the one place every caller routes through, so a suspect scoped read can be * compared against a full one without a rebuild. It is a READ policy and * deliberately not part of any env fingerprint (PROVIDER_ENV_VARS) — setting or * unsetting it must never invalidate a cache, only change how much of it is read. +* +* `opts.turnFilter` narrows turns at decode time for ranged queries: a kept +* turn is always whole (see TurnFilter), dropped turns contribute only their +* dedup keys plus carried scalars (see RangeFilteredMeta); PR-linked files, +* files with kept/dropped key overlap, and non-contiguous projections stay +* whole. Filtered loads bypass both memos. Durable sections are never +* filtered: orphan reconciliation reads the complete serve set. */ -export async function loadCache(scope?: CacheLoadScope): Promise { +export async function loadCache(scope?: CacheLoadScope, opts?: LoadCacheOptions): Promise { if (process.env['CODEBURN_CACHE_SCOPE'] === 'all') scope = undefined const dir = sessionCacheDir() const envelope = await readEnvelope(dir) if (!envelope) return afterMissingShardCache() const scopeKey = scope ? `${scope.fromMonth}..${scope.toMonth}` : 'all' - if (cacheMemo && cacheMemo.dir === dir && cacheMemo.nonce === envelope.nonce - && (cacheMemo.scope === 'all' || cacheMemo.scope === scopeKey)) return cacheMemo.cache + // A filtered load must never reuse a memoized full cache (it would serve + // unfiltered turns), and a full load must never reuse a filtered one — the + // write gate below guarantees only full results are stored, so skipping the + // read whenever a filter is active is both safe and sufficient. Provider + // identity joins the key by exact match (no 'all'-covers-scoped): a scoped + // memo must never serve an all-provider load (it holds shells), and an + // all-provider memo must never serve a scoped load (it defeats the bound). + const memoProvider = opts?.providerFilter ?? 'all' + if (!opts?.turnFilter && cacheMemo && cacheMemo.dir === dir && cacheMemo.nonce === envelope.nonce + && (cacheMemo.scope === 'all' || cacheMemo.scope === scopeKey) + && cacheMemo.provider === memoProvider) return cacheMemo.cache const cache: SessionCache = { version: CACHE_VERSION, providers: {}, complete: envelope.complete === true } const state = stateOf(cache) - const reads: Promise[] = [] + // Every selected shard, in envelope order. Reads run SERIALLY through this + // one queue: starting each provider's loads eagerly (and all providers at + // once) overlaps multi-hundred-MB tokenizer transients and OOMs small heaps + // even when every individual shard would fit. Thunks start no I/O until + // awaited below, so peak tracks one shard's transient plus the retained + // slices. Envelope order also keeps the merge deterministic: a path in two + // shards resolves to the freshest fingerprint, and the next save prunes the + // loser instead of letting it linger. + const pending: { provider: string; section: ProviderSection; bucket: string; run: () => Promise | null> }[] = [] for (const [provider, meta] of Object.entries(envelope.providers)) { const section: ProviderSection = { envFingerprint: meta.envFingerprint, @@ -1131,50 +1704,67 @@ export async function loadCache(scope?: CacheLoadScope): Promise { // belong to, and what stops the reconcile from re-parsing under a // fingerprint the envelope already agrees with. cache.providers[provider] = section - // Durable providers are ALWAYS loaded in full, by name as well as by the + // Provider-scoped query: other sections come back as empty shells (no + // shard reads) and ride the next save untouched via the unloaded-months + // carry. `loaded` is an empty set (nothing loaded) with full prior refs, + // exactly like an out-of-scope month set. + if (opts?.providerFilter !== undefined && opts.providerFilter !== 'all' && opts.providerFilter !== provider) { + state.loaded.set(provider, new Set()) + state.shards.set(provider, meta.shards) + state.fingerprints.set(provider, meta.envFingerprint) + continue + } // envelope flag: copilot's serve-time reconciliation pairs store rows and // retires residuals over the complete cached serve set, so a scoped load // of a copilot section persisted before the durable stamp landed would // make pairing range-dependent. The name check closes that window. - const full = !scope || meta.durable === true || DURABLE_PROVIDER_NAMES.has(provider) || meta.envFingerprint !== computeEnvFingerprint(provider) + const fingerprintMismatch = meta.envFingerprint !== computeEnvFingerprint(provider) + const durableSection = meta.durable === true || DURABLE_PROVIDER_NAMES.has(provider) + const full = !scope || durableSection || fingerprintMismatch const loaded: Set | null = full ? null : new Set() - // Shards are read concurrently but merged in envelope order, so the result - // never depends on which read finished first. A path that somehow ended up - // in two shards resolves to the FRESHEST fingerprint and dirties both - // buckets, so the next save prunes the loser instead of letting it linger. - const pending: { bucket: string; files: Promise | null> }[] = [] + const useFilter = opts?.turnFilter && !full for (const [bucket, ref] of Object.entries(meta.shards)) { if (loaded && !shardInScope(bucket, ref.until, scope!)) continue - loaded?.add(bucket) - pending.push({ bucket, files: loadShardMemoized(dir, ref.name) }) + // A ranged query narrows turns at decode time (see TurnFilter): the full + // month shard is streamed but only in-range turns are retained, bounding + // peak heap by the query instead of the corpus. Durable sections and + // fingerprint-mismatched sections always decode whole: orphan + // reconciliation and the fingerprint reset read the complete serve set. + const shardName = ref.name + pending.push({ + provider, + section, + bucket, + run: useFilter + ? () => loadShardFiltered(dir, shardName, opts!.turnFilter!) + : () => loadShardMemoized(dir, shardName), + }) } - reads.push((async () => { - for (const { bucket, files: read } of pending) { - const files = await read - // Unreadable: the bucket counts as loaded-and-empty and is marked - // dirty, so the re-parsed files replace it instead of the stale shard - // being carried forward forever. - if (!files) { markBucketDirty(state, provider, bucket); continue } - for (const [path, file] of Object.entries(files)) { - const key = `${provider}\0${path}` - const seenIn = state.bucketOf.get(key) - if (seenIn !== undefined) { - markBucketDirty(state, provider, seenIn) - markBucketDirty(state, provider, bucket) - if (section.files[path]!.fingerprint.mtimeMs >= file.fingerprint.mtimeMs) continue - } - state.bucketOf.set(key, bucket) - section.files[path] = file - } - } - })()) state.loaded.set(provider, loaded) state.shards.set(provider, meta.shards) state.fingerprints.set(provider, meta.envFingerprint) } - await Promise.all(reads) + for (const { provider, section, bucket, run } of pending) { + const files = await run() + // Unreadable: the bucket counts as loaded-and-empty and is marked + // dirty, so the re-parsed files replace it instead of the stale shard + // being carried forward forever. + if (!files) { markBucketDirty(state, provider, bucket); continue } + for (const [path, file] of Object.entries(files)) { + const key = `${provider}\0${path}` + const seenIn = state.bucketOf.get(key) + if (seenIn !== undefined) { + markBucketDirty(state, provider, seenIn) + markBucketDirty(state, provider, bucket) + if (section.files[path]!.fingerprint.mtimeMs >= file.fingerprint.mtimeMs) continue + } + state.bucketOf.set(key, bucket) + section.files[path] = file + } + } state.scope = scopeKey - cacheMemo = { dir, nonce: envelope.nonce, scope: scopeKey, cache } + state.scopeProvider = opts?.providerFilter ?? 'all' + if (!opts?.turnFilter) cacheMemo = { dir, nonce: envelope.nonce, scope: scopeKey, provider: state.scopeProvider, cache } return cache } @@ -1299,7 +1889,7 @@ function bucketFiles(section: ProviderSection): { groups: Map>() const until = new Map() for (const [path, file] of Object.entries(section.files)) { - const span = cacheFileSpan(file) + const span = fileSpan(file) let group = groups.get(span.bucket) if (!group) { group = {}; groups.set(span.bucket, group) } group[path] = file @@ -1312,7 +1902,7 @@ function bucketFiles(section: ProviderSection): { groups: Map): string { let until = UNDATED_BUCKET for (const file of Object.values(files)) { - const month = cacheFileSpan(file).until + const month = fileSpan(file).until if (month > until) until = month } return until @@ -1343,6 +1933,15 @@ function yieldToEventLoop(): Promise { return new Promise(resolve => setImmediate(resolve)) } +/// A bucket holding range-filtered records cannot be published without its +/// authoritative shard bytes (they live only on disk). Raised instead of +/// truncating; saveCache converts it to a failed save so callers degrade to +/// a read-only serve and retry. +class MergeAuthorityError extends Error { + constructor(bucket: string) { + super(`cannot publish bucket without its authoritative shard: ${bucket}`) + } +} export async function saveCache(cache: SessionCache, verifyStillOwner?: () => Promise): Promise { const dir = sessionCacheDir() if (!existsSync(dir)) await mkdir(dir, { recursive: true, mode: 0o700 }) @@ -1361,9 +1960,24 @@ export async function saveCache(cache: SessionCache, verifyStillOwner?: () => Pr // Overlay this run's entries for `bucket` onto the published shard `from`, // minus any path that has since moved to another month. const mergeShard = async (provider: string, plan: ProviderPlan, bucket: string, from: string | undefined): Promise => { - const files = plan.groups.get(bucket)! + // Buckets holding range-filtered records merge by streaming (below): + // materializing the published shard would reintroduce the OOM this + // feature removes. Unflagged buckets keep the in-memory merge verbatim. + if (groupHasFiltered(plan.groups.get(bucket)!)) { + return streamMergeBucket(provider, plan, bucket, from) + } + // Range-filtered records are never written: the published bytes stay + // authoritative for their paths, so the merge overlays only complete + // records and carries everything else verbatim (see RangeFilteredMeta). + const files = stripFilteredForWrite(plan.groups.get(bucket)!) + const hadFiltered = groupHasFiltered(plan.groups.get(bucket)!) const onDisk = from ? await loadShard(join(dir, from)) : null - if (!onDisk) return writeShard(provider, bucket, files) + if (!onDisk) { + // Without the published bytes there is nothing to carry the filtered + // paths from: writing the stripped set would truncate history. + if (hadFiltered) throw new MergeAuthorityError(bucket) + return writeShard(provider, bucket, files) + } // A file whose month this run never loaded has no visible cache entry, so it // looks uncached and is re-parsed into the same bucket — re-deriving the // entry the shard already holds. Republishing then churns the shard's nonce @@ -1377,6 +1991,112 @@ export async function saveCache(cache: SessionCache, verifyStillOwner?: () => Pr return writeShard(provider, bucket, { ...onDisk, ...files }) } + // Streaming merge for buckets holding range-filtered records: the published + // shard is far too large to materialize, so entries stream through one at a + // time. Dirty complete records substitute, moved paths drop, everything else + // is carried verbatim. Memory stays bounded by the largest single record + // plus the dirty set, never the shard. A missing/unreadable/torn published + // shard aborts the save (MergeAuthorityError) instead of publishing a + // truncation; write failures propagate like any other save I/O error. + const streamMergeBucket = async ( + provider: string, + plan: ProviderPlan, + bucket: string, + from: string | undefined, + ): Promise => { + if (!from || !existsSync(join(dir, from))) throw new MergeAuthorityError(bucket) + const group = plan.groups.get(bucket)! + const overwrites = new Map() + for (const [path, file] of Object.entries(group)) { + if (!file.rangeFiltered) overwrites.set(path, file) + } + const name = shardFileName(provider, bucket) + const finalPath = join(dir, name) + const tempPath = `${finalPath}.${randomBytes(8).toString('hex')}.tmp` + const handle = await open(tempPath, 'w', 0o600) + // Write a whole chunk: FileHandle.write may write partially (bytesWritten), + // especially for large serialized entries, and a short write without a loop + // produces a corrupt shard. Loop until the buffer is fully flushed. + const writeAll = async (text: string): Promise => { + const buffer = Buffer.from(text, 'utf-8') + let offset = 0 + while (offset < buffer.length) { + const { bytesWritten } = await handle.write(buffer, offset, buffer.length - offset, null) + offset += bytesWritten + } + } + let changed = false + let first = true + let until = UNDATED_BUCKET + const fail = async (err: unknown): Promise => { + try { await handle.close() } catch {} + await retryCacheFileMutation(() => unlink(tempPath)) + throw err + } + const emitPair = async (key: string, value: unknown): Promise => { + await writeAll(`${first ? '' : ','}${JSON.stringify(key)}:${JSON.stringify(value)}`) + first = false + } + const trackUntil = (month: string): void => { + if (month > until) until = month + } + try { + await writeAll('{') + await streamShardEntries(join(dir, from), async ({ key, value }) => { + if (plan.moved.has(key)) { + changed = true + return + } + const overwrite = overwrites.get(key) + if (overwrite !== undefined) { + overwrites.delete(key) + if (JSON.stringify(value) !== JSON.stringify(overwrite)) changed = true + await emitPair(key, overwrite) + trackUntil(fileSpan(overwrite).until) + return + } + if (!validateCachedFile(value)) throw new MergeAuthorityError(bucket) + await emitPair(key, value) + trackUntil(fileSpan(value).until) + }) + for (const [key, file] of overwrites) { + changed = true + await emitPair(key, file) + trackUntil(fileSpan(file).until) + } + await writeAll('}') + await handle.sync() + await handle.close() + } catch (err) { + const code = (err as { code?: string })?.code + if (err instanceof MergeAuthorityError || code === 'ENOENT') { + await fail(new MergeAuthorityError(bucket)) + } + await fail(err) + } + if (!changed) { + await retryCacheFileMutation(() => unlink(tempPath)) + return { name: from, until } + } + try { + for (let attempt = 0; attempt < 3; attempt++) { + try { + await rename(tempPath, finalPath) + break + } catch (err) { + const code = (err as NodeJS.ErrnoException).code + if ((code !== 'EPERM' && code !== 'EBUSY') || attempt === 2) throw err + await new Promise(resolve => { setTimeout(resolve, 10 * (attempt + 1)) }) + } + } + } catch (err) { + await retryCacheFileMutation(() => unlink(tempPath)) + throw err + } + written.add(name) + return { name, until } + } + try { // ── Phase one: everything that can be written from memory alone ────── for (const [provider, section] of Object.entries(cache.providers)) { @@ -1393,6 +2113,18 @@ export async function saveCache(cache: SessionCache, verifyStillOwner?: () => Pr const plan: ProviderPlan = { section, groups, loaded, priorRefs, reset, moved: new Set(), deferred: [], mergedFrom: new Map(), refs: {} } plans.set(provider, plan) + // Reset (fingerprint change): old refs are retirement-only authority, + // never merge input and never reuse candidates. Write only freshly + // parsed groups, drop every old ref, and skip moved inference (stale + // bucketOf mappings would misfire on the replaced section). + if (plan.reset) { + for (const [bucket, files] of groups) { + plan.refs[bucket] = await writeShard(provider, bucket, stripFilteredForWrite(files)) + await yieldToEventLoop() + } + continue + } + // An entry whose bucket this run never loaded may ALSO still exist, under // an older month, in a shard we are about to carry across verbatim — a // re-parse that shifted the file's oldest turn, or (the common #441 path) @@ -1403,7 +2135,7 @@ export async function saveCache(cache: SessionCache, verifyStillOwner?: () => Pr if (loaded) { for (const [path, file] of Object.entries(section.files)) { if (state.bucketOf.has(`${provider}\0${path}`)) continue - const bucket = cacheFileSpan(file).bucket + const bucket = fileSpan(file).bucket if (!loaded.has(bucket) || bucket === UNDATED_BUCKET) plan.moved.add(path) } } @@ -1422,8 +2154,11 @@ export async function saveCache(cache: SessionCache, verifyStillOwner?: () => Pr // into the bucket, so the published shard's other entries have to be // merged back in or the save would drop them. Deferred to phase two so // the read happens against the CURRENT shard, not a stale name. - if (loaded && !loaded.has(bucket) && prior) { plan.deferred.push(bucket); continue } - plan.refs[bucket] = await writeShard(provider, bucket, files) + // Dirty plus range-filtered: memory holds only a slice of this bucket, + // so like a dirty-but-unloaded bucket it must merge against the + // CURRENT published shard rather than rewriting from memory alone. + if ((loaded && !loaded.has(bucket) && prior) || groupHasFiltered(files)) { plan.deferred.push(bucket); continue } + plan.refs[bucket] = await writeShard(provider, bucket, stripFilteredForWrite(files)) // Surrender the event loop between shard writes so an interactive // TTY's stdin handler (Ink's useInput) can run while a long save // publishes a 21k-file cache. The yield is BETWEEN shards - never @@ -1512,7 +2247,13 @@ export async function saveCache(cache: SessionCache, verifyStillOwner?: () => Pr const shards: Record = {} for (const [bucket, ref] of Object.entries(plan.refs)) { if (written.has(ref.name) || existsSync(join(dir, ref.name))) { shards[bucket] = ref; continue } - const files = plan.groups.get(bucket) + const group = plan.groups.get(bucket) + // A filtered record can never stand in for its published bytes: if the + // authoritative shard vanished mid-save, abort rather than publish a + // truncation (callers degrade to a read-only serve and retry). Only + // complete groups may be rewritten here. + if (group && groupHasFiltered(group)) throw new MergeAuthorityError(bucket) + const files = group && stripFilteredForWrite(group) // A carried month whose file vanished and whose content was never in // memory cannot be rewritten; dropping the reference is the only honest // option, and the sweep retires the name. @@ -1557,17 +2298,23 @@ export async function saveCache(cache: SessionCache, verifyStillOwner?: () => Pr state.shards.set(provider, meta.shards) state.fingerprints.set(provider, meta.envFingerprint) for (const [path, file] of Object.entries(cache.providers[provider]!.files)) { - state.bucketOf.set(`${provider}\0${path}`, cacheFileSpan(file).bucket) + state.bucketOf.set(`${provider}\0${path}`, fileSpan(file).bucket) } } // Write-through: the object just published IS the freshest state, so the // next loadCache in this process reuses it instead of re-parsing. Its scope // is whatever was loaded, not `all` — a save never widens what is in memory. - cacheMemo = { dir, nonce: envelope.nonce, scope: state.scope, cache } + // A filtered cache object must never enter the memo: a later full-scope + // load in this process would serve its missing turns as complete. + if (!cacheHasFilteredRecords(cache)) cacheMemo = { dir, nonce: envelope.nonce, scope: state.scope, provider: state.scopeProvider, cache } for (const name of retired) await retryCacheFileMutation(() => unlink(join(dir, name))) return true } catch (err) { for (const name of written) await retryCacheFileMutation(() => unlink(join(dir, name))) + // Merge-authority abort: a flagged bucket lost its published shard + // mid-save. Fail the save (callers degrade to a read-only serve and + // retry) rather than publish a truncated shard. + if (err instanceof MergeAuthorityError) return false throw err } } @@ -1715,6 +2462,12 @@ export function reconcileFile( return { action: 'unchanged' } } + // A range-filtered record holds only a slice of its turns: an append would + // merge the fresh tail onto that slice, keep the marker, and lose the parse + // at the next save (which strips flagged records). Force a full re-parse + // from byte 0 so the replacement is a complete unmarked record. Unchanged + // files (fingerprint match above) keep serving from the slice. + if (cached.rangeFiltered) return { action: 'modified' } if ( cached.lastCompleteLineOffset !== undefined && // Defensive: never resume past the file's current end. A truncate-then-regrow diff --git a/src/shard-stream.ts b/src/shard-stream.ts new file mode 100644 index 000000000..6dce0c306 --- /dev/null +++ b/src/shard-stream.ts @@ -0,0 +1,342 @@ +// Bounded-memory JSON object decoding for multi-hundred-MB cache shards. +import type { FileHandle } from 'node:fs/promises' + +/// Write a whole text chunk: FileHandle.write may write partially +/// (bytesWritten), especially for large serialized entries, and a short write +/// without a loop produces a corrupt shard. Loop until fully flushed. +export async function writeChunk(handle: FileHandle, text: string): Promise { + const buffer = Buffer.from(text, 'utf-8') + let offset = 0 + while (offset < buffer.length) { + const { bytesWritten } = await handle.write(buffer, offset, buffer.length - offset, null) + offset += bytesWritten + } +} + +// +// A shard payload is one top-level JSON object mapping paths to records. +// `JSON.parse` retains the whole text plus the whole object graph at once +// (~3x file bytes of live heap); on the 498MB OMP month shard that alone +// OOMs a default-heap run. These helpers stream-decode instead, emitting one +// top-level entry at a time so callers retain only what the query needs. +// +// Error semantics mirror whole-file parsing exactly: ANY decode failure +// (truncated file, invalid token, read error, non-object root) throws, and +// callers must discard the entries decoded so far — a torn shard is dropped +// whole, never partially committed. Chunk-boundary correctness (UTF-8 splits, +// `\uXXXX` escapes spanning reads) comes from the stream-json tokenizer, not +// hand-rolled state; do not reimplement it here. +import { createReadStream } from 'node:fs' +import { open } from 'node:fs/promises' +import { pipeline } from 'node:stream/promises' +import { parser } from 'stream-json' +import { Assembler } from 'stream-json/assembler.js' +import { streamObject, type StreamObjectItem } from 'stream-json/streamers/stream-object.js' + +import { flatString } from './content-utils.js' + +// Test barrier (see the load-spanning-flush test): invoked once the entries +// stream opens its file descriptor, so tests can stage a concurrent publish +// deterministically. Never set outside tests. +let afterStreamOpenForTests: (() => void) | null = null +export function __setAfterStreamOpenForTests(hook: (() => void) | null): void { + afterStreamOpenForTests = hook +} + +export type ShardEntry = { key: string; value: unknown } + +/** First non-whitespace byte of a file, or null if empty. A BOM is not +* skipped: `JSON.parse` rejects it, so a BOM-prefixed shard is dropped just +* like the whole-file decoder dropped it (the tokenizer would agree). */ +async function peekFirstByte(path: string): Promise { + const handle = await open(path, 'r') + try { + const buf = Buffer.alloc(64) + const { bytesRead } = await handle.read(buf, 0, 64, 0) + for (let i = 0; i < bytesRead; i++) { + const byte = buf[i]! + if (byte === 0x20 || byte === 0x09 || byte === 0x0a || byte === 0x0d) continue + return byte + } + return null + } finally { + await handle.close() + } +} + +/** +* Stream the top-level entries of the JSON object at `path`, invoking +* `onEntry` once per entry in document order (awaited, so writers keep +* backpressure instead of buffering the shard). Resolves only after a clean +* end-of-stream; anything else (truncated/corrupt/unreadable input, a +* non-object root) rejects, and the caller must discard entries seen so far. +*/ +export async function streamShardEntries( + path: string, + onEntry: (entry: ShardEntry) => void | Promise, +): Promise { + let first: number | null + try { + first = await peekFirstByte(path) + } catch { + throw new Error(`shard unreadable: ${path}`) + } + if (first === null) throw new Error(`shard unreadable: ${path}`) + if (first !== 0x7b) throw new Error(`shard root is not an object: ${path}`) + await pipeline( + createReadStream(path), + parser.asStream(), + streamObject.asStream(), + async function* (entries: AsyncIterable) { + for await (const entry of entries) { + if (typeof entry?.key !== 'string') throw new Error(`shard entry without key: ${path}`) + await onEntry({ key: flatString(entry.key), value: entry.value }) + } + }, + ) +} + +// +// Per-element streaming inside one named array field of every top-level +// entry. `streamShardEntries` above assembles each whole record before the +// caller sees it; when the record is a cached session file with tens of +// thousands of turns, that one assembly is the OOM (a single month shard can +// hold a record whose turns expand past 512MB). This walker assembles at most +// ONE array element (plus one metadata field) at a time: element callbacks +// decide keep/drop per element while dropped elements are still hot, so peak +// Mechanics: a hand-rolled token walk over the packing parser's output (keys +// as `keyValue`, strings/numbers whole — the parser defaults, which the +// Assembler also requires). Genuinely unexpected shapes (chunked strings, +// separate key tokens, deeper nesting) throw fail-closed: the caller discards +// the shard exactly as for a torn file. A non-array value under the array +// field name is NOT one of those: it is emitted as a plain field so the +// consumer decides (session shards reject it at validation; the codex +// retained-key scan keeps the entry). Capture boundaries are exact stack +// paths (`[entryKey, arrayField, index]` for elements, `[entryKey, field]` +// for metadata), so dotted entry keys can never misfire them. +export type ShardArrayFieldCallbacks = { + onFileStart?: (key: string) => void | Promise + /** Every non-array top-level field of the entry, in document order. */ + onField?: (key: string, field: string, value: unknown) => void | Promise + /** One array element, in document order with its index. */ + onElement?: (key: string, index: number, value: unknown) => void | Promise + /** arraySeen is false when the entry has no arrayField at all. */ + onFileEnd?: (key: string, elementCount: number, arraySeen: boolean) => void | Promise +} + +type ParserToken = Parameters[0] + +const SCALAR_TOKENS = new Set(['stringValue', 'numberValue', 'nullValue', 'trueValue', 'falseValue']) +const START_TOKENS = new Set(['startObject', 'startArray']) + +function assembleTokens(tokens: ParserToken[]): unknown { + // Detach strings from the tokenizer's buffers: every retained stringValue + // or key would otherwise pin its whole input chunk for the life of the + // cache (V8 SlicedString), ballooning streaming decode past whole-file + // JSON.parse. Buffer round-trip forces fresh flat strings (see flatString). + const asm = new Assembler() + for (const token of tokens) { + if ((token.name === 'stringValue' || token.name === 'keyValue') && 'value' in token && typeof token.value === 'string') { + asm.consume({ ...token, value: flatString(token.value) }) + } else { + asm.consume(token) + } + } + return asm.current +} + +export async function streamShardArrayField( + path: string, + arrayField: string, + cb: ShardArrayFieldCallbacks, + opts?: { rootField?: string }, +): Promise { + let first: number | null + try { + first = await peekFirstByte(path) + } catch { + throw new Error(`shard unreadable: ${path}`) + } + if (first === null) throw new Error(`shard unreadable: ${path}`) + if (first !== 0x7b) throw new Error(`shard root is not an object: ${path}`) + const source = createReadStream(path) + const openHook = afterStreamOpenForTests + if (openHook) source.once('open', openHook) + await pipeline( + source, + // Packed tokens only (keys as `keyValue`, whole strings/numbers): the + // defaults also emit streamed duplicates (startKey/stringChunk/...) that + // this walk does not track. + parser.asStream({ streamValues: false }), + async function* (tokens: AsyncIterable) { + // Container frames; the root object itself is never pushed, so a file + const stack: { key: string | number | null; isArray: boolean; nextIndex: number }[] = [] + const rootField = opts?.rootField + let seenRoot = false + // Envelope skipping (rootField set): depth-0 siblings of the entries + // object are ignored (scalars) or depth-counted without pushing + // (containers), so their bytes never materialize. + let skipDepth: number | null = null + let enteredRoot = false + let pendingKey: string | null = null + let fileKey: string | null = null + let elementCount = 0 + let arraySeen = false + // Set once the array field's value arrives in any shape (array or + // field-emitted): JSON objects never repeat keys, so a second one is + // corrupt input and fails closed (JSON.parse would silently last-win). + let arrayFieldSeen = false + // Active token capture (one element or one metadata field at a time). + let capture: { tokens: ParserToken[]; startDepth: number; kind: 'element' | 'field'; index: number; field: string } | null = null + const fail = (msg: string): never => { + throw new Error(`${msg}: ${path}`) + } + const stackPath = (): (string | number | null)[] => stack.map(frame => frame.key) + for await (const token of tokens) { + const { name } = token + const value = 'value' in token ? (token.value as unknown) : undefined + if (name === 'keyValue') { + if (skipDepth !== null) continue + if (typeof value !== 'string') fail('shard key is not a string') + if (capture) capture.tokens.push(token) + // Detach: this key flows into stack frames, file keys, field names + // and map keys retained long-term (same SlicedString hazard as values). + pendingKey = flatString(value as string) + continue + } + if (START_TOKENS.has(name)) { + if (!seenRoot) { + if (stack.length !== 0 || pendingKey !== null || name !== 'startObject') fail('shard root is not an object') + seenRoot = true + continue + } + if (skipDepth !== null) { + skipDepth++ + pendingKey = null + continue + } + if (stack.length === 0 && rootField !== undefined && !enteredRoot) { + // are depth-counted without pushing; the entries object itself + // is entered without pushing so inner entries sit at depth 0. + const key = pendingKey + pendingKey = null + if (key === rootField) { + if (name !== 'startObject') fail(`shard field ${rootField} is not an object`) + enteredRoot = true + } else { + skipDepth = 1 + } + continue + } + const parent = stack.length === 0 ? null : stack[stack.length - 1]! + const key = pendingKey ?? (parent?.isArray ? parent.nextIndex++ : null) + pendingKey = null + if (capture) capture.tokens.push(token) + stack.push({ key, isArray: name === 'startArray', nextIndex: 0 }) + if (!capture) { + const at = stackPath() + if (at.length === 1) { + if (typeof key !== 'string' || name !== 'startObject') fail('shard entry is not an object') + fileKey = key as string + elementCount = 0 + arraySeen = false + arrayFieldSeen = false + await cb.onFileStart?.(fileKey) + } else if (at.length === 2) { + if (at[1] === arrayField) { + if (arrayFieldSeen) fail(`duplicate shard field ${arrayField}`) + arrayFieldSeen = true + } + if (at[1] === arrayField && name === 'startArray') { + arraySeen = true + } else { + // Nested object/array metadata field — or a non-array value + // under the array field name, which the consumer judges + // (session shards reject it; the codex scan keeps the entry). + capture = { tokens: [token], startDepth: stack.length, kind: 'field', index: -1, field: at[1] as string } + } + } else if (at.length === 3 && at[1] === arrayField) { + capture = { tokens: [token], startDepth: stack.length, kind: 'element', index: key as number, field: '' } + } else { + fail('shard entry has unexpected nesting') + } + } + continue + } + if (SCALAR_TOKENS.has(name)) { + if (skipDepth !== null) continue + const parent = stack.length === 0 ? null : stack[stack.length - 1]! + const key = pendingKey ?? (parent?.isArray ? parent.nextIndex++ : null) + pendingKey = null + if (capture) { + capture.tokens.push(token) + continue + } + const at = [...stackPath(), key] + if (at.length === 1) { + // Envelope sibling scalar (e.g. version) when scoped outside the + // entries object; a file value otherwise, which is corrupt. + if (rootField === undefined || enteredRoot) fail('shard entry is not an object') + continue + } + // Scalars assemble through the same path as containers: + // `numberValue` carries the lexeme as a string, so forwarding + // `value` would turn every numeric field into a string. + else if (at.length === 2) { + if (at[1] === arrayField) { + if (arrayFieldSeen) fail(`duplicate shard field ${arrayField}`) + arrayFieldSeen = true + } + await cb.onField?.(fileKey!, at[1] as string, assembleTokens([token])) + } else if (at.length === 3 && at[1] === arrayField) { + await cb.onElement?.(fileKey!, key as number, assembleTokens([token])) + elementCount++ + } else { + fail('shard entry has unexpected nesting') + } + continue + } + if (name === 'endObject' || name === 'endArray') { + if (skipDepth !== null) { + skipDepth-- + if (skipDepth === 0) { + skipDepth = null + pendingKey = null + } + continue + } + if (stack.length === 0) { + // Entries-object close when scoped (nothing was pushed for it), + // else the root close (nothing was pushed for it either). + if (name !== 'endObject') fail('shard root is not an object') + enteredRoot = false + continue + } + if (capture) { + capture.tokens.push(token) + if (stack.length === capture.startDepth) { + const done = capture + capture = null + const assembled = assembleTokens(done.tokens) + if (done.kind === 'element') { + await cb.onElement?.(fileKey!, done.index, assembled) + elementCount++ + } else { + await cb.onField?.(fileKey!, done.field, assembled) + } + } + } + stack.pop() + if (stack.length === 0 && fileKey !== null) { + const doneKey = fileKey + fileKey = null + await cb.onFileEnd?.(doneKey, elementCount, arraySeen) + } + continue + } + fail(`shard has an unsupported token ${name}`) + } + if (capture || stack.length !== 0 || fileKey !== null) fail('shard ended mid-entry') + }, + ) +} diff --git a/tests/cache-directory-switch.test.ts b/tests/cache-directory-switch.test.ts index 40a462394..14bc9e421 100644 --- a/tests/cache-directory-switch.test.ts +++ b/tests/cache-directory-switch.test.ts @@ -260,6 +260,10 @@ describe('call-time CODEBURN_CACHE_DIR isolation', () => { process.env['CODEBURN_CACHE_DIR'] = cacheDir await writeCachedCodexResults(codexSource, 'project', [call('codex', 'before')], (await fingerprintFile(codexSource))!) await flushCodexCache() + // Populate the resident snapshot while disk still says 'before': without + // the clear below, lookups keep serving this warm copy (flush releases + // unflushed writes but never the snapshots they were read into). + expect((await readCachedCodexResults(codexSource))?.calls.map(entry => entry.model)).toEqual(['before']) expect(await readAntigravityModel(antigravitySource)).toBe('before') // Another process republishes both cache files. Without the clear, the diff --git a/tests/content-utils.test.ts b/tests/content-utils.test.ts index 34a00ee51..e077fece6 100644 --- a/tests/content-utils.test.ts +++ b/tests/content-utils.test.ts @@ -41,3 +41,39 @@ describe('normalizeContentBlocks', () => { } }) }) + +describe('flattenJsonStrings', () => { + it('retains the decoded graph itself with equal content (no copy)', async () => { + const { flattenJsonStrings } = await import('../src/content-utils.js') + const value = { + path: '/live/sessions/x.jsonl', + calls: [{ deduplicationKey: 'omp:1', nested: { list: ['a', 1, true, null] } }], + empty: {}, + } + const pristine = structuredClone(value) + const out = flattenJsonStrings(value) + expect(out).toBe(value) + expect(out.calls).toBe(value.calls) + expect(out).toEqual(pristine) + }) + + it('flattens keys as well as string leaves, preserving key order', async () => { + const { flattenJsonStrings } = await import('../src/content-utils.js') + const slicedKey = `x${'y'.repeat(64)}`.slice(1) + const value = { a: 1, [slicedKey]: { v: 's' }, z: 2 } + const out = flattenJsonStrings(value) + expect(out).toBe(value) + expect(Object.keys(out)).toEqual(['a', slicedKey.slice(0), 'z']) + expect(out[slicedKey]).toEqual({ v: 's' }) + }) + + it('preserves own __proto__ keys from valid JSON without changing the prototype', async () => { + const { flattenJsonStrings } = await import('../src/content-utils.js') + const value = JSON.parse('{"__proto__":{"x":"y"},"a":"b"}') as Record + const out = flattenJsonStrings(value) + expect(out).toBe(value) + expect(Object.prototype.hasOwnProperty.call(out, '__proto__')).toBe(true) + expect(Object.getPrototypeOf(out)).toBe(Object.prototype) + expect((out as Record)['__proto__']).toEqual({ x: 'y' }) + }) +}) diff --git a/tests/providers/antigravity.test.ts b/tests/providers/antigravity.test.ts index a7e9ddcde..98294cafb 100644 --- a/tests/providers/antigravity.test.ts +++ b/tests/providers/antigravity.test.ts @@ -393,7 +393,7 @@ describe('antigravity provider helpers', () => { path: getAntigravityStatusLineEventsPath(), project: 'antigravity-cli', provider: 'antigravity', - }, new Set(['antigravity:rpc-covered-conversation:0'])) + }, new Set(['antigravity:rpc-covered-conversation:0'])) const calls = [] for await (const call of parser.parse()) calls.push(call) diff --git a/tests/session-cache-range-filter.test.ts b/tests/session-cache-range-filter.test.ts new file mode 100644 index 000000000..e4ea74351 --- /dev/null +++ b/tests/session-cache-range-filter.test.ts @@ -0,0 +1,675 @@ +// Range-filtered shard loads (OOM fix): a ranged query must be able to decode +// month shards WITHOUT retaining every turn. Out-of-range turns are dropped at +// decode time; their dedup keys are kept for pre-seeding, and every scalar the +// save/evict/orphan paths read is carried on the record, so a filtered load +// can never serialize back as a truncated authoritative shard. +// +// The query range here is deliberately ONE DAY inside a month shard (July 15 +// within July): same-month July 1/14/16/31 turns must be discarded while July +// 15 and midnight-straddling turns remain. A month-wide filter would pass +// while `overview -p today` still retained the whole September shard. +// +// Parity proof fixtures (projection applies only when exact; otherwise the +// whole record is kept unflagged): +// - same key in a kept AND a dropped turn of one file -> whole file (a flat +// key bag cannot preserve intra-file walk order); +// - PR-linked files stay whole (the anchor logic reads spawn sets across the +// full turn list); +// - kept turns must form ONE contiguous block in file order (carry state is +// positional; covers append-only suffixes and newest-first prefixes); +// - cross-file suppression relies on file-open-ordered seeding (pinned by the +// ordering unit test); the serve loops must seed when they open each file. +// +// These tests pin the feature BEFORE the implementation is complete: the +// `opts` argument to loadCache is ignored until then, so the filtering +// assertions fail (red) while the guard assertions already pass. +// Not-yet-existing exports are reached by static namespace import plus an +// existence assertion, so a missing export fails its own test instead of +// the suite. +import { mkdir, readFile, readdir, rm, stat, writeFile } from 'fs/promises' +import { existsSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { + CACHE_VERSION, + clearLoadCacheMemo, + clearShardMemo, + computeEnvFingerprint, + loadCache, + loadShardFiltered, + markCacheDirty, + monthScopeForRange, + reconcileFile, + saveCache, + seedDroppedKeys, + sessionCacheDir, + type CachedFile, + type SessionCache, +} from '../src/session-cache.js' +import * as parserModule from '../src/parser.js' +import { + clearCodexMemCaches, + codexCacheFileName, + fingerprintFile, + readCachedCodexResults, +} from '../src/codex-cache.js' +import type { ParsedProviderCall } from '../src/providers/types.js' + +let TMP_DIR: string + +beforeEach(async () => { + TMP_DIR = join(tmpdir(), `codeburn-rangefilter-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`) + process.env['CODEBURN_CACHE_DIR'] = TMP_DIR + await mkdir(TMP_DIR, { recursive: true }) + clearLoadCacheMemo() + clearShardMemo() + clearCodexMemCaches() +}) + +afterEach(async () => { + clearCodexMemCaches() + if (existsSync(TMP_DIR)) await rm(TMP_DIR, { recursive: true }) +}) + +type Turn = CachedFile['turns'][number] + +function callAt(timestamp: string, key: string): Turn['calls'][number] { + return { + provider: 'omp', + model: 'm', + usage: { + inputTokens: 10, + outputTokens: 5, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + cacheCreationOneHourTokens: 0, + }, + speed: 'standard', + timestamp, + tools: [], + bashCommands: [], + skills: [], + subagentTypes: [], + deduplicationKey: key, + } +} + +function turnAt(timestamp: string, key: string, extraCalls: Array<{ timestamp: string; key: string }> = []): Turn { + return { + timestamp, + sessionId: 'sess-1', + userMessage: 'do the thing', + calls: [callAt(timestamp, key), ...extraCalls.map(e => callAt(e.timestamp, e.key))], + } +} + +function cachedFile(overrides: Partial = {}): CachedFile { + return { + fingerprint: { dev: 1, ino: 2, mtimeMs: 3, sizeBytes: 4 }, + lastCompleteLineOffset: 128, + mcpInventory: [], + turns: [turnAt('2026-07-15T10:00:00Z', 'day15-1')], + ...overrides, + } +} + +// The query under test: ONE day. The scope stays month-wide (that is the +// production shape: monthScopeForRange over a today range selects the current +// month shard), so same-month turns outside the day must still be discarded. +const DAY = { start: new Date('2026-07-15T00:00:00Z'), end: new Date('2026-07-15T23:59:59.999Z') } + +// Local copy of the keep-predicate semantics (any call timestamp in range). +// The implementation must behave identically to turnSlicedToRange's null rule; +// a parity test below pins the real export against this oracle. +function keepDay(turn: Turn): boolean { + return turn.calls.some(c => { + const ts = new Date(c.timestamp).getTime() + return !Number.isNaN(ts) && ts >= DAY.start.getTime() && ts <= DAY.end.getTime() + }) +} + +async function seedJulyCorpus(): Promise> { + const files: Record = { + '/live/june.jsonl': cachedFile({ turns: [turnAt('2026-06-10T10:00:00Z', 'june-1')] }), + '/live/span.jsonl': cachedFile({ + // Dropped June prefix carries branch + PR state into the kept straddle; + // the straddling turn is kept WHOLE (all calls), never call-trimmed. + turns: [ + { ...turnAt('2026-06-20T10:00:00Z', 'juneb-1'), gitBranch: 'main', prRefs: ['https://github.com/o/r/pull/1'] }, + { + ...turnAt('2026-07-14T23:59:00Z', 'span-14', [{ timestamp: '2026-07-15T00:01:00Z', key: 'span-15' }]), + userMessage: 'long night 😀日本語', + }, + turnAt('2026-07-14T10:00:00Z', 'july14-only'), + ], + }), + '/live/july.jsonl': cachedFile({ + // Same-month decoys around the query day: only day15-1 survives. + turns: [ + turnAt('2026-07-01T10:00:00Z', 'july1-1'), + turnAt('2026-07-15T10:00:00Z', 'day15-1'), + turnAt('2026-07-16T10:00:00Z', 'july16-1'), + turnAt('2026-07-31T10:00:00Z', 'july31-1'), + ], + }), + // PR-linked files stay whole: the anchor logic reads spawn sets across + // the full turn list, which a slice cannot reproduce. + '/live/pr.jsonl': cachedFile({ + prLinks: ['https://github.com/o/r/pull/9'], + turns: [turnAt('2026-06-11T10:00:00Z', 'pr-june'), turnAt('2026-07-16T10:00:00Z', 'pr-july')], + }), + // Same key on both sides of the boundary: the whole file stays whole, so + // no retained turn is ever suppressed differently than the full walk. + '/live/dup.jsonl': cachedFile({ + turns: [turnAt('2026-07-15T10:00:00Z', 'shared-x'), turnAt('2026-07-20T10:00:00Z', 'shared-x')], + }), + // Interleaved kept/dropped/kept: carry state would be positional + // guesswork, so the whole file stays whole. + '/live/noncontig.jsonl': cachedFile({ + turns: [ + turnAt('2026-07-15T08:00:00Z', 'nc-1'), + turnAt('2026-07-16T08:00:00Z', 'nc-drop'), + turnAt('2026-07-15T18:00:00Z', 'nc-2'), + ], + }), + } + const cache: SessionCache = { + version: CACHE_VERSION, + complete: true, + providers: { omp: { envFingerprint: computeEnvFingerprint('omp'), files } }, + } + markCacheDirty(cache, 'omp') + await saveCache(cache) + clearLoadCacheMemo() + clearShardMemo() + return files +} + +function julyMonthScope() { + return monthScopeForRange(new Date('2026-07-15T00:00:00Z'), new Date('2026-07-15T23:59:59.999Z')) +} + +describe('range-filtered shard load', () => { + it('keeps whole intersecting turns and drops the rest with keys collected', async () => { + await seedJulyCorpus() + const loaded = await loadCache(julyMonthScope(), { turnFilter: keepDay }) + const files = loaded.providers['omp']!.files + + // June-only file: no turns retained, keys + scalars carried. + expect(files['/live/june.jsonl']!.turns).toEqual([]) + expect(files['/live/june.jsonl']!.rangeFiltered?.droppedKeys).toEqual(['june-1']) + expect(files['/live/june.jsonl']!.fingerprint).toEqual({ dev: 1, ino: 2, mtimeMs: 3, sizeBytes: 4 }) + + // Span file: prefix carry captured, straddling turn kept WHOLE (both + // calls), July-14-only dropped. + const span = files['/live/span.jsonl']! + expect(span.turns.map(t => t.calls.map(c => c.deduplicationKey))).toEqual([['span-14', 'span-15']]) + expect(span.rangeFiltered?.droppedKeys).toEqual(['juneb-1', 'july14-only']) + expect(span.rangeFiltered?.carryBranch).toBe('main') + expect(span.rangeFiltered?.carryPrRefs).toEqual(['https://github.com/o/r/pull/1']) + expect(span.rangeFiltered?.droppedHadBranch).toBe(true) + + // July file: same-month decoys dropped in walk order, query day kept. + const july = files['/live/july.jsonl']! + expect(july.turns.map(t => t.calls.map(c => c.deduplicationKey))).toEqual([['day15-1']]) + expect(july.rangeFiltered?.droppedKeys).toEqual(['july1-1', 'july16-1', 'july31-1']) + }) + + it('keeps PR-linked, key-overlap and interleaved files whole', async () => { + await seedJulyCorpus() + const loaded = await loadCache(julyMonthScope(), { turnFilter: keepDay }) + const files = loaded.providers['omp']!.files + for (const path of ['/live/pr.jsonl', '/live/dup.jsonl', '/live/noncontig.jsonl']) { + expect(files[path]!.rangeFiltered).toBeUndefined() + } + expect(files['/live/pr.jsonl']!.turns).toHaveLength(2) + expect(files['/live/dup.jsonl']!.turns.map(t => t.calls.map(c => c.deduplicationKey))).toEqual([['shared-x'], ['shared-x']]) + expect(files['/live/noncontig.jsonl']!.turns).toHaveLength(3) + }) + + it('matches turnSlicedToRange null-rule exactly, including straddles', async () => { + const parserExports = parserModule as unknown as { + turnIntersectsRange?: (turn: Turn, range: { start: Date; end: Date }) => boolean + } + expect(typeof parserExports.turnIntersectsRange).toBe('function') + const intersects = parserExports.turnIntersectsRange! + const cases: Array<[Turn, boolean]> = [ + [turnAt('2026-07-15T10:00:00Z', 'a'), true], + [turnAt('2026-07-16T10:00:00Z', 'b'), false], + [turnAt('2026-07-01T10:00:00Z', 'c'), false], + // Straddling turn: one call each side — kept (serve trims calls itself). + [turnAt('2026-07-14T23:59:00Z', 'd', [{ timestamp: '2026-07-15T00:01:00Z', key: 'e' }]), true], + // Unparseable timestamps never intersect. + [{ ...turnAt('2026-07-15T10:00:00Z', 'f'), calls: [{ ...callAt('not-a-date', 'f') }] }, false], + // Empty call list intersects nothing. + [{ ...turnAt('2026-07-15T10:00:00Z', 'g'), calls: [] }, false], + ] + for (const [turn, expected] of cases) { + expect(intersects(turn, DAY)).toBe(expected) + expect(keepDay(turn)).toBe(expected) + } + }) + + it('seeds dropped keys in file-open walk order (cross-file suppression contract)', async () => { + const fileA = cachedFile({ + turns: [], + rangeFiltered: { + span: { bucket: '2026-07', until: '2026-07' }, + newestCallMs: 1, + droppedKeys: ['shared-x'], + }, + }) + const fileB = cachedFile({ turns: [turnAt('2026-07-15T10:00:00Z', 'shared-x')] }) + // Walk order A then B (the serve loops must seed when they open each + // file): B's kept turn suppresses exactly as the full walk would, where + // A's out-of-range turn would have added the key at A's position. + const seen = new Set() + seedDroppedKeys(seen, fileA) + const suppressed = fileB.turns[0]!.calls.some(c => seen.has(c.deduplicationKey)) + expect(suppressed).toBe(true) + // Reversed: B first counts (nothing seeded yet) — order is load-bearing. + const seen2 = new Set() + const counted = !fileB.turns[0]!.calls.some(c => seen2.has(c.deduplicationKey)) + expect(counted).toBe(true) + }) + + it('leaves full loads untouched (no flags, complete turns)', async () => { + await seedJulyCorpus() + const loaded = await loadCache(julyMonthScope()) + for (const file of Object.values(loaded.providers['omp']!.files)) { + expect(file.rangeFiltered).toBeUndefined() + } + expect(loaded.providers['omp']!.files['/live/span.jsonl']!.turns).toHaveLength(3) + }) + + it('does not let a filtered load poison a later full load (memo hygiene)', async () => { + await seedJulyCorpus() + await loadCache(julyMonthScope(), { turnFilter: keepDay }) + const full = await loadCache(julyMonthScope()) + expect(full.providers['omp']!.files['/live/june.jsonl']!.turns).toHaveLength(1) + expect(full.providers['omp']!.files['/live/june.jsonl']!.rangeFiltered).toBeUndefined() + }) + + it('isolates concurrent filtered loads by range (no shared mutable projection)', async () => { + await seedJulyCorpus() + const june = { start: new Date('2026-06-01T00:00:00Z'), end: new Date('2026-06-30T23:59:59.999Z') } + const keepJune = (turn: Turn): boolean => turn.calls.some(c => { + const ts = new Date(c.timestamp).getTime() + return !Number.isNaN(ts) && ts >= june.start.getTime() && ts <= june.end.getTime() + }) + const [dayLoad, juneLoad] = await Promise.all([ + loadCache(julyMonthScope(), { turnFilter: keepDay }), + loadCache(monthScopeForRange(new Date('2026-06-01T00:00:00Z'), new Date('2026-06-30T23:59:59.999Z')), { turnFilter: keepJune }), + ]) + expect(dayLoad.providers['omp']!.files['/live/july.jsonl']!.turns.map(t => t.calls.map(c => c.deduplicationKey))).toEqual([['day15-1']]) + expect(juneLoad.providers['omp']!.files['/live/june.jsonl']!.turns).toHaveLength(1) + expect(juneLoad.providers['omp']!.files['/live/july.jsonl']).toBeUndefined() + }) + + it('loads mismatch sections whole under a ranged query instead of projecting them', async () => { + // A parse-version bump changes the fingerprint; the section is discarded + // at parse time, but the load still reads it in full (never scoped, never + // projected) so the fingerprint reset and orphan carry-forward see every + // entry. Projecting a section the parse is about to discard would save + // nothing and would hide orphaned paths from the reset. + const cache: SessionCache = { + version: CACHE_VERSION, + complete: true, + providers: { + omp: { + envFingerprint: 'stale-fingerprint', + files: { '/live/july.jsonl': cachedFile({ turns: [turnAt('2026-07-15T10:00:00Z', 'x')] }) }, + }, + }, + } + markCacheDirty(cache, 'omp') + await saveCache(cache) + clearLoadCacheMemo() + clearShardMemo() + const loaded = await loadCache(julyMonthScope(), { turnFilter: keepDay }) + expect(loaded.providers['omp']).toBeDefined() + const file = loaded.providers['omp']!.files['/live/july.jsonl']! + expect(file.turns.map(t => t.calls.map(c => c.deduplicationKey))).toEqual([['x']]) + expect(file.rangeFiltered).toBeUndefined() + }) + + it('retires old-only paths on mismatch save instead of merging them forward', async () => { + // Seed with a stale fingerprint, then simulate what the parse does on a + // reset (see getOrCreateProviderSection): replace the section with fresh + // records under the current fingerprint and save. The old shard must be + // retired and its paths gone — never merged into the reset, never reused. + const cache: SessionCache = { + version: CACHE_VERSION, + complete: true, + providers: { + omp: { + envFingerprint: 'stale-fingerprint', + files: { '/live/old.jsonl': cachedFile({ turns: [turnAt('2026-07-15T10:00:00Z', 'old-1')] }) }, + }, + }, + } + markCacheDirty(cache, 'omp') + await saveCache(cache) + clearLoadCacheMemo() + clearShardMemo() + const before = (await readdir(sessionCacheDir())).filter(n => n.startsWith('omp.')).sort() + expect(before.length).toBeGreaterThan(0) + + const loaded = await loadCache(julyMonthScope(), { turnFilter: keepDay }) + // Mismatched sections load whole (see above); the parse replacement below + // is what discards them, and the save retires the old shards. + expect(Object.keys(loaded.providers['omp']!.files)).toEqual(['/live/old.jsonl']) + // Parse replacement: fresh section under the current fingerprint. + loaded.providers['omp'] = { + envFingerprint: computeEnvFingerprint('omp'), + files: { '/live/new.jsonl': cachedFile({ turns: [turnAt('2026-07-15T10:00:00Z', 'new-1')] }) }, + } + markCacheDirty(loaded, 'omp') + expect(await saveCache(loaded)).toBe(true) + + clearLoadCacheMemo() + clearShardMemo() + const reloaded = await loadCache() + expect(Object.keys(reloaded.providers['omp']!.files).sort()).toEqual(['/live/new.jsonl']) + const after = (await readdir(sessionCacheDir())).filter(n => n.startsWith('omp.')).sort() + for (const name of before) expect(after).not.toContain(name) + }) +}) + +describe('reconcile forces full re-parse on flagged change (blocker: no append onto a slice)', () => { + it('flagged + changed fingerprint reports modified, never appended', () => { + const flagged = cachedFile({ + turns: [], + rangeFiltered: { + span: { bucket: '2026-07', until: '2026-07' }, + newestCallMs: 1, + droppedKeys: ['old-1'], + }, + }) + expect(reconcileFile({ dev: 1, ino: 2, mtimeMs: 3, sizeBytes: 4 }, flagged).action).toBe('unchanged') + expect(reconcileFile({ dev: 1, ino: 2, mtimeMs: 4, sizeBytes: 4096 }, flagged).action).toBe('modified') + const full = cachedFile() + expect(reconcileFile({ dev: 1, ino: 2, mtimeMs: 4, sizeBytes: 4096 }, full).action).not.toBe('unchanged') + }) +}) + +describe('filtered save round-trip (blocker: no silent truncation)', () => { + it('mutate/save one in-range file, full-reload: every out-of-range turn preserved', async () => { + const original = await seedJulyCorpus() + const filtered = await loadCache(julyMonthScope(), { turnFilter: keepDay }) + // The filter visibly engaged: fully out-of-range files carry the marker. + expect(filtered.providers['omp']!.files['/live/june.jsonl']!.rangeFiltered).toBeDefined() + + // Simulate this run re-parsing one in-range file: fresh FULL record. + const omp = filtered.providers['omp']! + omp.files['/live/july.jsonl'] = cachedFile({ + fingerprint: { dev: 1, ino: 2, mtimeMs: 999, sizeBytes: 4096 }, + turns: [turnAt('2026-07-15T12:00:00Z', 'day15-fresh')], + }) + markCacheDirty(filtered, 'omp', '/live/july.jsonl') + await saveCache(filtered) + + clearLoadCacheMemo() + clearShardMemo() + const reloaded = await loadCache() + const files = reloaded.providers['omp']!.files + + // Untouched files: byte/semantically identical, no filter residue — + // including same-month decoys, the straddling turn's June call, and the + // whole-kept PR/dup/noncontig files. + expect(files['/live/june.jsonl']).toEqual(original['/live/june.jsonl']) + expect(files['/live/span.jsonl']).toEqual(original['/live/span.jsonl']) + expect(files['/live/pr.jsonl']).toEqual(original['/live/pr.jsonl']) + expect(files['/live/dup.jsonl']).toEqual(original['/live/dup.jsonl']) + expect(files['/live/noncontig.jsonl']).toEqual(original['/live/noncontig.jsonl']) + expect(files['/live/june.jsonl']!.rangeFiltered).toBeUndefined() + // Mutated file carries the fresh content. + expect(files['/live/july.jsonl']!.turns.map(t => t.calls.map(c => c.deduplicationKey))).toEqual([['day15-fresh']]) + // Totals across the corpus: 1 June + 3 span + 1 fresh + 2 PR + 2 dup + 3 noncontig. + const turnCount = Object.values(files).reduce((n, f) => n + f.turns.length, 0) + expect(turnCount).toBe(1 + 3 + 1 + 2 + 2 + 3) + }) + + it('streaming merge preserves flagged bytes while substituting dirty records', async () => { + // Dirty July bucket WITH a flagged file still in it: the save must merge + // (streaming, bounded) rather than rewrite from memory, and a full reload + // of the newly published shard must validate every record. + await seedJulyCorpus() + const filtered = await loadCache(julyMonthScope(), { turnFilter: keepDay }) + expect(filtered.providers['omp']!.files['/live/july.jsonl']!.rangeFiltered).toBeDefined() + + // Add a large fresh file (multi-megabyte content exercises partial-write + // safety in the merge writer) and dirty only its path. + const bigMessage = `merge payload 😀日本語 x `.repeat(200000) + const omp = filtered.providers['omp']! + omp.files['/live/july2.jsonl'] = cachedFile({ + fingerprint: { dev: 9, ino: 9, mtimeMs: 999, sizeBytes: 4096 }, + turns: [{ ...turnAt('2026-07-15T12:00:00Z', 'july2-fresh'), userMessage: bigMessage }], + }) + markCacheDirty(filtered, 'omp', '/live/july2.jsonl') + await saveCache(filtered) + + clearLoadCacheMemo() + clearShardMemo() + const reloaded = await loadCache() + const files = reloaded.providers['omp']!.files + // Flagged file's published full turns survived the merge byte-identical. + expect(files['/live/july.jsonl']!.turns.map(t => t.calls.map(c => c.deduplicationKey))).toEqual([ + ['july1-1'], ['day15-1'], ['july16-1'], ['july31-1'], + ]) + expect(files['/live/july.jsonl']!.rangeFiltered).toBeUndefined() + // Fresh multi-megabyte record round-tripped exactly (reload validation). + expect(files['/live/july2.jsonl']!.turns[0]!.userMessage).toBe(bigMessage) + expect(files['/live/span.jsonl']!.turns).toHaveLength(3) + }) + + it('missing authority aborts the save instead of publishing a truncation', async () => { + await seedJulyCorpus() + const filtered = await loadCache(julyMonthScope(), { turnFilter: keepDay }) + // Dirty the July bucket WITHOUT replacing its flagged file: add a fresh + // file so the flagged july.jsonl must merge against published bytes. + const omp = filtered.providers['omp']! + omp.files['/live/july2.jsonl'] = cachedFile({ + fingerprint: { dev: 9, ino: 9, mtimeMs: 999, sizeBytes: 4096 }, + turns: [turnAt('2026-07-15T12:00:00Z', 'july2-fresh')], + }) + markCacheDirty(filtered, 'omp', '/live/july2.jsonl') + // Delete every published July shard: the merge has no authority left. + const dir = sessionCacheDir() + const names = (await readdir(dir)).filter(n => n.startsWith('omp.2026-07')) + expect(names.length).toBeGreaterThan(0) + for (const name of names) await rm(join(dir, name)) + const envelopeBefore = await readFile(join(dir, 'envelope.json'), 'utf-8') + const published = await saveCache(filtered) + expect(published).toBe(false) + // No truncated envelope went out: the envelope still names the (now + // unlinked) prior shards, and no replacement shard was published. + expect(await readFile(join(dir, 'envelope.json'), 'utf-8')).toBe(envelopeBefore) + expect((await readdir(dir)).filter(n => n.startsWith('omp.2026-07') && !names.includes(n))).toEqual([]) + }) + + it('torn shards fail closed: truncation drops the shard, keeps the month', async () => { + await seedJulyCorpus() + const dir = sessionCacheDir() + const names = (await readdir(dir)).filter(n => n.startsWith('omp.2026-07')) + expect(names.length).toBeGreaterThan(0) + const shardPath = join(dir, names[0]!) + const raw = await readFile(shardPath, 'utf-8') + // Truncate mid-object (valid prefix, cut inside a turn). + await writeFile(shardPath, raw.slice(0, Math.floor(raw.length / 2))) + + clearLoadCacheMemo() + clearShardMemo() + const loaded = await loadCache(julyMonthScope(), { turnFilter: keepDay }) + // Mirrors the existing corrupt-shard path: the torn shard is dropped + // whole (zero partial entries committed) while the intact June shard + // still serves (june + span + pr live in June buckets). + expect(Object.keys(loaded.providers['omp']?.files ?? {}).sort()).toEqual([ + '/live/june.jsonl', + '/live/pr.jsonl', + '/live/span.jsonl', + ]) + }) + + it('large multi-byte content decodes identically to JSON.parse', async () => { + const big = `emoji 😀 CJK 日本語 surrogate pair 𝌆 tail `.repeat(4000) + const cache: SessionCache = { + version: CACHE_VERSION, + complete: true, + providers: { + omp: { + envFingerprint: computeEnvFingerprint('omp'), + files: { '/live/big.jsonl': cachedFile({ turns: [{ ...turnAt('2026-07-15T10:00:00Z', 'big-1'), userMessage: big }] }) }, + }, + }, + } + markCacheDirty(cache, 'omp') + await saveCache(cache) + clearLoadCacheMemo() + clearShardMemo() + const loaded = await loadCache(julyMonthScope(), { turnFilter: keepDay }) + expect(loaded.providers['omp']!.files['/live/big.jsonl']!.turns[0]!.userMessage).toBe(big) + }) +}) + +describe('codex-results guard (blocker: resume state survives filtered session runs)', () => { + function codexCall(key: string): ParsedProviderCall { + return { + provider: 'codex', + model: 'm', + inputTokens: 1, + outputTokens: 1, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costUSD: 0, + tools: [], + bashCommands: [], + timestamp: '2026-07-15T00:00:00.000Z', + speed: 'standard', + deduplicationKey: `codex:${key}`, + userMessage: '', + sessionId: key, + } + } + + it('leaves the codex result cache byte-identical with exact hits intact', async () => { + // Real source file so the fingerprint matches and the lookup is genuine + // (an invented fingerprint misses and proves nothing). + const sourcePath = join(TMP_DIR, 'rollout.jsonl') + await writeFile(sourcePath, '{}\n') + const fp = await fingerprintFile(sourcePath) + expect(fp).not.toBeNull() + const cachePath = join(TMP_DIR, codexCacheFileName()) + const entry = { + dev: fp!.dev, + ino: fp!.ino, + mtimeMs: fp!.mtimeMs, + sizeBytes: fp!.sizeBytes, + project: 'fixture', + calls: [codexCall('cx-1')], + resumeOffset: 3, + resumeState: { sessionId: 's' }, + resumeCallCount: 1, + } + await writeFile(cachePath, JSON.stringify({ version: 15, files: { [sourcePath]: entry } })) + const before = await readFile(cachePath, 'utf-8') + + // Exact hit works before the session-cache cycle (genuine, fp-matched). + const hitBefore = await readCachedCodexResults(sourcePath) + expect(hitBefore?.kind).toBe('exact') + const callsBefore = hitBefore && 'calls' in hitBefore ? hitBefore.calls.map(c => c.deduplicationKey) : [] + expect(callsBefore).toEqual(['codex:cx-1']) + clearCodexMemCaches() + + // A filtered session-cache load+save cycle must not touch codex-results. + await seedJulyCorpus() + await loadCache(julyMonthScope(), { turnFilter: keepDay }) + + expect(await readFile(cachePath, 'utf-8')).toBe(before) + const st = await stat(sourcePath) + expect(st.size).toBeGreaterThan(0) + const hitAfter = await readCachedCodexResults(sourcePath) + expect(hitAfter?.kind).toBe('exact') + const callsAfter = hitAfter && 'calls' in hitAfter ? hitAfter.calls.map(c => c.deduplicationKey) : [] + expect(callsAfter).toEqual(['codex:cx-1']) + // Resume fields round-trip byte-identically (not just the served calls). + expect(JSON.parse(await readFile(cachePath, 'utf-8')).files[sourcePath]).toEqual(entry) + }) +}) + +describe('range-filtered load at record scale', () => { + it('decodes a 30k-turn record per-turn with an exact marker', async () => { + // The overview-today OOM: one record holds far more turns than the query + // keeps. The load must stream it (never assemble the whole turns array) + // and still project exactly: kept suffix, walk-ordered dropped keys, + // full-list span/newest carried on the marker. + const dir = sessionCacheDir() + await mkdir(dir, { recursive: true }) + const turns: Turn[] = [] + for (let i = 0; i < 15000; i++) turns.push(turnAt('2026-06-10T10:00:00Z', `june-${i}`)) + for (let i = 0; i < 14999; i++) turns.push(turnAt('2026-07-01T10:00:00Z', `july1-${i}`)) + turns.push(turnAt('2026-07-15T12:00:00Z', 'kept-1')) + const name = 'omp.2026-06.scale.json' + await writeFile(join(dir, name), JSON.stringify({ '/live/big.jsonl': cachedFile({ turns }) })) + const files = (await loadShardFiltered(dir, name, keepDay))! + expect(Object.keys(files)).toEqual(['/live/big.jsonl']) + const file = files['/live/big.jsonl']! + expect(file.turns.length).toBe(1) + expect(file.turns[0]!.calls[0]!.deduplicationKey).toBe('kept-1') + const marker = file.rangeFiltered! + expect(marker.droppedKeys.length).toBe(29999) + expect(marker.droppedKeys[0]).toBe('june-0') + expect(marker.droppedKeys[15000]).toBe('july1-0') + expect(marker.droppedKeys[29998]).toBe('july1-14998') + expect(marker.span).toEqual({ bucket: '2026-06', until: '2026-07' }) + expect(marker.newestCallMs).toBe(new Date('2026-07-15T12:00:00Z').getTime()) + expect(marker).not.toHaveProperty('carryBranch') + expect(marker).not.toHaveProperty('firstTurnProject') + }) + it('omits carry fields when every turn is dropped', async () => { + // Old prefix loop ran zero times when firstKept stayed -1: carryBranch + // and carryPrRefs must be absent (there is no kept block to carry + // into), while droppedHadBranch still reports the dropped branches. + const dir = sessionCacheDir() + await mkdir(dir, { recursive: true }) + const turns: Turn[] = [ + { ...turnAt('2026-06-10T10:00:00Z', 'june-0'), gitBranch: 'main', prRefs: ['x#1'] }, + { ...turnAt('2026-06-11T10:00:00Z', 'june-1'), gitBranch: 'main' }, + ] + const name = 'omp.2026-06.alldropped.json' + await writeFile(join(dir, name), JSON.stringify({ '/live/old.jsonl': cachedFile({ turns }) })) + const files = (await loadShardFiltered(dir, name, keepDay))! + const file = files['/live/old.jsonl']! + expect(file.turns).toEqual([]) + const marker = file.rangeFiltered! + expect(marker.droppedKeys).toEqual(['june-0', 'june-1']) + expect(marker).not.toHaveProperty('carryBranch') + expect(marker).not.toHaveProperty('carryPrRefs') + expect(marker.droppedHadBranch).toBe(true) + }) + it('leaves all-kept and empty files unmarked', async () => { + // No marker unless turns were actually dropped: all-kept (and empty) + // records serve their turns with rangeFiltered undefined. + const dir = sessionCacheDir() + await mkdir(dir, { recursive: true }) + const name = 'omp.2026-07.allkept.json' + await writeFile(join(dir, name), JSON.stringify({ + '/live/kept.jsonl': cachedFile({ turns: [turnAt('2026-07-15T10:00:00Z', 'k-1'), turnAt('2026-07-15T11:00:00Z', 'k-2')] }), + '/live/empty.jsonl': cachedFile({ turns: [] }), + })) + const files = (await loadShardFiltered(dir, name, keepDay))! + expect(files['/live/kept.jsonl']!.turns.length).toBe(2) + expect(files['/live/kept.jsonl']!.rangeFiltered).toBeUndefined() + expect(files['/live/empty.jsonl']!.turns).toEqual([]) + expect(files['/live/empty.jsonl']!.rangeFiltered).toBeUndefined() + }) +}) + diff --git a/tests/session-cache-shards.test.ts b/tests/session-cache-shards.test.ts index 4855e3d9c..9ee3c217e 100644 --- a/tests/session-cache-shards.test.ts +++ b/tests/session-cache-shards.test.ts @@ -582,6 +582,88 @@ describe('scoped load', () => { }) }) +describe('provider-scoped load', () => { + async function seedTwoProviders(): Promise { + const cache: SessionCache = { + version: CACHE_VERSION, + complete: true, + providers: { + claude: { + envFingerprint: computeEnvFingerprint('claude'), + files: { '/live/c.jsonl': fileSpanning('2026-06-10T10:00:00Z') }, + }, + codex: { + envFingerprint: computeEnvFingerprint('codex'), + files: { '/live/x.jsonl': fileSpanning('2026-06-10T10:00:00Z') }, + }, + }, + } + markCacheDirty(cache, 'claude') + markCacheDirty(cache, 'codex') + await saveCache(cache) + } + + const juneScope = monthScopeForRange(new Date('2026-06-01T00:00:00Z'), new Date('2026-06-30T23:59:59Z')) + + it('loads only the selected provider, others come back as empty shells', async () => { + await seedTwoProviders() + clearLoadCacheMemo() + const scoped = await loadCache(juneScope, { providerFilter: 'codex' }) + expect(Object.keys(scoped.providers['codex']!.files)).toEqual(['/live/x.jsonl']) + expect(scoped.providers['claude']).toBeDefined() + expect(Object.keys(scoped.providers['claude']!.files)).toEqual([]) + }) + + it('save from a narrowed load leaves other providers byte-identical', async () => { + await seedTwoProviders() + const before = await shardBytes() + const kept = (await envelope()).providers['claude']!.shards['2026-06']!.name + + clearLoadCacheMemo() + const scoped = await loadCache(juneScope, { providerFilter: 'codex' }) + scoped.providers['codex']!.files['/live/x2.jsonl'] = fileSpanning('2026-06-20T10:00:00Z') + markCacheDirty(scoped, 'codex', '/live/x2.jsonl') + await saveCache(scoped) + + const after = await shardBytes() + expect(after.get(kept), 'narrowed-out provider rewritten').toBe(before.get(kept)) + + clearLoadCacheMemo() + const full = await loadCache() + expect(Object.keys(full.providers['claude']!.files)).toEqual(['/live/c.jsonl']) + expect(Object.keys(full.providers['codex']!.files).sort()).toEqual(['/live/x.jsonl', '/live/x2.jsonl']) + }) + + it('isolates memos across sequential provider loads', async () => { + await seedTwoProviders() + clearLoadCacheMemo() + const a = await loadCache(juneScope, { providerFilter: 'claude' }) + expect(Object.keys(a.providers['claude']!.files)).toEqual(['/live/c.jsonl']) + expect(Object.keys(a.providers['codex']!.files)).toEqual([]) + // Same request reuses the memo (identical object). + expect(await loadCache(juneScope, { providerFilter: 'claude' })).toBe(a) + // Another provider never observes the first load's shells... + const b = await loadCache(juneScope, { providerFilter: 'codex' }) + expect(b).not.toBe(a) + expect(Object.keys(b.providers['codex']!.files)).toEqual(['/live/x.jsonl']) + expect(Object.keys(b.providers['claude']!.files)).toEqual([]) + // ...and the all-provider load still sees everything. + const all = await loadCache(juneScope) + expect(all).not.toBe(a) + expect(all).not.toBe(b) + expect(Object.keys(all.providers['claude']!.files)).toEqual(['/live/c.jsonl']) + expect(Object.keys(all.providers['codex']!.files)).toEqual(['/live/x.jsonl']) + }) + it('narrows undated lifetime loads by provider too', async () => { + await seedTwoProviders() + clearLoadCacheMemo() + const scoped = await loadCache(undefined, { providerFilter: 'codex' }) + expect(Object.keys(scoped.providers['codex']!.files)).toEqual(['/live/x.jsonl']) + expect(scoped.providers['claude']).toBeDefined() + expect(Object.keys(scoped.providers['claude']!.files)).toEqual([]) + }) +}) + describe('v8 -> v9 migration', () => { it('re-buckets the v8 provider shards losslessly and retires the v8 directory', async () => { const v8Dir = join(TMP_DIR, 'session-cache.v8') diff --git a/tests/shard-stream-turns.test.ts b/tests/shard-stream-turns.test.ts new file mode 100644 index 000000000..af4534414 --- /dev/null +++ b/tests/shard-stream-turns.test.ts @@ -0,0 +1,152 @@ +// Per-element shard streaming: records with huge turn lists must decode one +// turn at a time (the overview-today OOM), never assembled whole. +import { mkdtemp, rm, writeFile } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { streamShardArrayField } from '../src/shard-stream.js' + +let dir: string + +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'codeburn-shard-walk-')) +}) + +afterEach(async () => { + await rm(dir, { recursive: true, force: true }) +}) + +async function writeShard(name: string, content: string): Promise { + const path = join(dir, name) + await writeFile(path, content) + return path +} + +describe('streamShardArrayField', () => { + it('yields fields and turns per file in order, dotted keys intact', async () => { + const path = await writeShard('s.json', JSON.stringify({ + 'a.calls.v1.jsonl': { + fingerprint: { dev: 1, ino: 2, mtimeMs: 3, sizeBytes: 4 }, + mcpInventory: [], + turns: [ + { timestamp: 't1', sessionId: 's', userMessage: 'u', calls: [] }, + { timestamp: 't2', sessionId: 's', userMessage: 'u', calls: [{ x: 1 }] }, + ], + }, + 'b.jsonl': { fingerprint: { dev: 5, ino: 6, mtimeMs: 7, sizeBytes: 8 }, mcpInventory: ['m'], turns: [] }, + })) + const events: string[] = [] + const fields: [string, string][] = [] + const elements: [string, number][] = [] + let endInfo: [string, number, boolean][] = [] + await streamShardArrayField(path, 'turns', { + onFileStart: key => { events.push(`start:${key}`) }, + onField: (key, field, value) => { + fields.push([key, field]) + if (field === 'fingerprint') { + expect(value).toEqual(key === 'b.jsonl' + ? { dev: 5, ino: 6, mtimeMs: 7, sizeBytes: 8 } + : { dev: 1, ino: 2, mtimeMs: 3, sizeBytes: 4 }) + } + }, + onElement: (key, index, value) => { + elements.push([key, index]) + events.push(`turn:${key}:${index}:${(value as { timestamp: string }).timestamp}`) + }, + onFileEnd: (key, count, seen) => { endInfo.push([key, count, seen]) }, + }) + expect(events).toEqual([ + 'start:a.calls.v1.jsonl', + 'turn:a.calls.v1.jsonl:0:t1', + 'turn:a.calls.v1.jsonl:1:t2', + 'start:b.jsonl', + ]) + expect(elements).toEqual([['a.calls.v1.jsonl', 0], ['a.calls.v1.jsonl', 1]]) + expect(endInfo).toEqual([['a.calls.v1.jsonl', 2, true], ['b.jsonl', 0, true]]) + }) + + it('emits a non-array field value for the consumer to judge', async () => { + const notArray = await writeShard('n.json', JSON.stringify({ 'f.jsonl': { turns: { oops: 1 }, n: 5 } })) + const fields: [string, string, unknown][] = [] + const ends: [string, number, boolean][] = [] + await streamShardArrayField(notArray, 'turns', { + onField: (k, f, v) => { fields.push([k, f, v]) }, + onFileEnd: (k, c, s) => { ends.push([k, c, s]) }, + }) + expect(fields).toEqual([['f.jsonl', 'turns', { oops: 1 }], ['f.jsonl', 'n', 5]]) + expect(ends).toEqual([['f.jsonl', 0, false]]) + }) + + it('reports a missing array field', async () => { + const missing = await writeShard('m.json', JSON.stringify({ 'f.jsonl': { fingerprint: 1 } })) + const ends: [string, number, boolean][] = [] + await streamShardArrayField(missing, 'turns', { onFileEnd: (k, c, s) => { ends.push([k, c, s]) } }) + expect(ends).toEqual([['f.jsonl', 0, false]]) + }) + + it('rejects a duplicated array field instead of concatenating', async () => { + // Built as raw text: object literals would last-win before the walker + // ever sees the duplicate. JSON.parse would silently last-win too; + // concatenating both arrays (or an array plus a later non-array) would + // serve turns no single value holds. + const dupArray = await writeShard('d.json', '{"f.jsonl": {"turns": [{"a": 1}], "other": 1, "turns": [{"b": 2}]}}') + await expect(streamShardArrayField(dupArray, 'turns', {})).rejects.toThrow(/duplicate/) + const arrayThenScalar = await writeShard('d2.json', '{"f.jsonl": {"turns": [{"a": 1}], "turns": 5}}') + await expect(streamShardArrayField(arrayThenScalar, 'turns', {})).rejects.toThrow(/duplicate/) + const scalarThenArray = await writeShard('d3.json', '{"f.jsonl": {"turns": 5, "turns": [{"a": 1}]}}') + await expect(streamShardArrayField(scalarThenArray, 'turns', {})).rejects.toThrow(/duplicate/) + }) + it('scopes the walk to a root field with siblings around it', async () => { + // Codex-results envelope shape: scalar version before, files object, + // scalar tail after, dotted file keys. Only the files subtree walks. + const path = await writeShard('env.json', JSON.stringify({ + version: 15, + files: { + 'a.calls.v1.jsonl': { mtimeMs: 7, calls: [{ timestamp: 't1' }, { timestamp: 't2' }] }, + 'b.jsonl': { mtimeMs: 8, calls: [] }, + }, + tail: 'ignored', + })) + const fields: [string, string][] = [] + const elements: [string, number][] = [] + const ends: [string, number, boolean][] = [] + await streamShardArrayField(path, 'calls', { + onField: (k, f) => { fields.push([k, f]) }, + onElement: (k, i) => { elements.push([k, i]) }, + onFileEnd: (k, c, s) => { ends.push([k, c, s]) }, + }, { rootField: 'files' }) + expect(fields).toEqual([['a.calls.v1.jsonl', 'mtimeMs'], ['b.jsonl', 'mtimeMs']]) + expect(elements).toEqual([['a.calls.v1.jsonl', 0], ['a.calls.v1.jsonl', 1]]) + expect(ends).toEqual([['a.calls.v1.jsonl', 2, true], ['b.jsonl', 0, true]]) + }) + + it('rejects a scalar file value inside the scoped object', async () => { + const path = await writeShard('envbad.json', JSON.stringify({ version: 15, files: { 'a.jsonl': 5 } })) + await expect(streamShardArrayField(path, 'calls', {}, { rootField: 'files' })).rejects.toThrow() + }) + it('rejects truncated input', async () => { + const path = await writeShard('t.json', '{"f.jsonl": {"turns": [{') + await expect(streamShardArrayField(path, 'turns', {})).rejects.toThrow() + }) + + it('rejects a BOM-prefixed shard like JSON.parse does', async () => { + const path = await writeShard('bom.json', '{"f.jsonl": {"turns": []}}') + await expect(streamShardArrayField(path, 'turns', {})).rejects.toThrow() + }) + it('streams thousands of turns without assembling the record', async () => { + const turns = Array.from({ length: 5000 }, (_, i) => ({ timestamp: `t${i}`, sessionId: 's', userMessage: 'u', calls: [] })) + const path = await writeShard('big.json', JSON.stringify({ 'big.jsonl': { fingerprint: 1, turns } })) + let count = 0 + let last = -1 + await streamShardArrayField(path, 'turns', { + onElement: (_key, index, value) => { + expect(index).toBe(last + 1) + last = index + expect((value as { timestamp: string }).timestamp).toBe(`t${index}`) + count++ + }, + }) + expect(count).toBe(5000) + }) +}) From 0368dc6f1289caf7d0ab565013fb36cf45e1b3b2 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sat, 19 Sep 2026 23:38:57 -0400 Subject: [PATCH 02/13] fix: detach only retained strings on the filtered shard path Per-string flattening at assembly copied every string of every turn (kept or dropped): ~829k turns x dozens of strings of Buffer + string churn that GC could not retire under a heap cap, dying in the sept shard at ~500MB live. streamShardArrayField takes detachStrings (default true); the filtered loader passes false and flattens only what it keeps (kept turns + meta at fold time, carry refs + first project in finishSlice). (cherry picked from commit 1b30177248dc7f85e6b77cf5ef845cd06fb751fc) --- src/session-cache.ts | 36 +++++++++++++++++++++++------------- src/shard-stream.ts | 25 +++++++++++++++---------- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/src/session-cache.ts b/src/session-cache.ts index 8309d568a..bec1ce561 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -4,7 +4,7 @@ import { createHash, randomBytes } from 'crypto' import { join } from 'path' import { getCodeburnCacheDir } from './cache-dir.js' -import { flattenJsonStrings } from './content-utils.js' +import { flatString, flattenJsonStrings } from './content-utils.js' import { acquireCacheRefreshLock, releaseOwnedRefreshLocksForExit } from './cache-refresh-lock.js' import type { ToolCall } from './types.js' import { streamShardArrayField, streamShardEntries } from './shard-stream.js' @@ -1380,9 +1380,13 @@ function finishSlice( if (acc.keptCount > 0 && acc.lastKept - acc.firstKept + 1 !== acc.keptCount) return { whole: true } for (const key of acc.droppedKeys) if (acc.keptKeys.has(key)) return { whole: true } const droppedHadBranch = acc.prefixHadBranch || (acc.keptCount === 0 ? acc.anyBranch : acc.suffixHadBranch) - // No flattening here either: every turn passing through the accumulator - // was assembled by assembleTokens (already detached), so carry refs, first - // project, and kept turns retain directly. + // Detach what the marker retains: kept turns were flattened at fold time, + // but the carry refs and firstTurnProject may point at dropped + // (unflattened) turns. Detach those few strings here so no tokenizer + // chunk survives through the marker. + const carryBranch = acc.carryBranch !== undefined ? flatString(acc.carryBranch) : undefined + const carryPrRefs = acc.carryPrRefs !== undefined ? flattenJsonStrings(acc.carryPrRefs) : undefined + const firstProject = firstTurnProject !== undefined ? flatString(firstTurnProject) : undefined return { whole: false, file: { @@ -1391,13 +1395,13 @@ function finishSlice( rangeFiltered: { span, newestCallMs: newestMs, - ...(firstTurnProject !== undefined ? { firstTurnProject } : {}), + ...(firstProject !== undefined ? { firstTurnProject: firstProject } : {}), droppedKeys: acc.droppedKeys, // Carry needs a kept block to carry INTO: with zero kept turns the // old prefix loop ran zero times (firstKept === -1), so the fields // stay absent. droppedHadBranch still reports via anyBranch. - ...(acc.keptCount > 0 && acc.carryBranch !== undefined ? { carryBranch: acc.carryBranch } : {}), - ...(acc.keptCount > 0 && acc.carryPrRefs !== undefined ? { carryPrRefs: acc.carryPrRefs } : {}), + ...(acc.keptCount > 0 && carryBranch !== undefined ? { carryBranch } : {}), + ...(acc.keptCount > 0 && carryPrRefs !== undefined ? { carryPrRefs } : {}), ...(droppedHadBranch ? { droppedHadBranch: true } : {}), }, } as CachedFile, @@ -1498,15 +1502,21 @@ export async function loadShardFiltered( // validation here, so fail the same way instead of serving metadata // without turns as complete. if (field === 'turns') throw new Error(`shard record invalid: ${name}`) - builds.get(key)!.meta[field] = value + builds.get(key)!.meta[field] = flattenJsonStrings(value) }, onElement: (key, index, value) => { const build = builds.get(key)! if (!validateTurn(value)) throw new Error(`shard record invalid: ${name}`) const turn = value + const keep = turnFilter(turn) + // Detach only what the accumulator retains (see assembleTokens): + // kept turns flatten whole; dropped turns release their slices + // unflattened, so tokenizer chunks do not accumulate flattened copies + // of data nobody keeps. + if (keep) flattenJsonStrings(turn) // Fold straight into the accumulator: no per-turn wrapper or key // arrays (see SliceAccumulator). Scalars mirror sliceItemFor. - foldLiveTurn(build.acc, turn, turnFilter(turn)) + foldLiveTurn(build.acc, turn, keep) const month = monthKey(turn.timestamp) if (month !== null) { if (build.spanMin === null || month < build.spanMin) build.spanMin = month @@ -1549,7 +1559,7 @@ export async function loadShardFiltered( if (!validateCachedFile(rest)) throw new Error(`shard record invalid: ${name}`) files[key] = sliced.file }, - }) + }, { detachStrings: false }) } catch { return null } @@ -1572,13 +1582,13 @@ export async function loadShardFiltered( }, onField: (key, field, value) => { const build = wholeBuilds.get(key) - if (build) build.meta[field] = value + if (build) build.meta[field] = flattenJsonStrings(value) }, onElement: (key, _index, value) => { const build = wholeBuilds.get(key) if (!build) return if (!validateTurn(value)) throw new Error(`shard record invalid: ${name}`) - build.turns.push(value) + build.turns.push(flattenJsonStrings(value)) }, onFileEnd: (key, _count, arraySeen) => { const build = wholeBuilds.get(key) @@ -1590,7 +1600,7 @@ export async function loadShardFiltered( files[key] = file completed.add(key) }, - }) + }, { detachStrings: false }) } catch { return null } diff --git a/src/shard-stream.ts b/src/shard-stream.ts index 6dce0c306..175b486f5 100644 --- a/src/shard-stream.ts +++ b/src/shard-stream.ts @@ -129,14 +129,16 @@ type ParserToken = Parameters[0] const SCALAR_TOKENS = new Set(['stringValue', 'numberValue', 'nullValue', 'trueValue', 'falseValue']) const START_TOKENS = new Set(['startObject', 'startArray']) -function assembleTokens(tokens: ParserToken[]): unknown { - // Detach strings from the tokenizer's buffers: every retained stringValue - // or key would otherwise pin its whole input chunk for the life of the - // cache (V8 SlicedString), ballooning streaming decode past whole-file - // JSON.parse. Buffer round-trip forces fresh flat strings (see flatString). +function assembleTokens(tokens: ParserToken[], detach: boolean): unknown { + // Detach strings from the tokenizer's buffers when the caller retains the + // value: every retained stringValue or key would otherwise pin its whole + // input chunk for the life of the cache (V8 SlicedString), ballooning + // streaming decode past whole-file JSON.parse. Buffer round-trip forces + // fresh flat strings (see flatString). Callers that drop the value pass + // false and skip the per-string copy churn. const asm = new Assembler() for (const token of tokens) { - if ((token.name === 'stringValue' || token.name === 'keyValue') && 'value' in token && typeof token.value === 'string') { + if (detach && (token.name === 'stringValue' || token.name === 'keyValue') && 'value' in token && typeof token.value === 'string') { asm.consume({ ...token, value: flatString(token.value) }) } else { asm.consume(token) @@ -149,7 +151,7 @@ export async function streamShardArrayField( path: string, arrayField: string, cb: ShardArrayFieldCallbacks, - opts?: { rootField?: string }, + opts?: { rootField?: string; detachStrings?: boolean }, ): Promise { let first: number | null try { @@ -162,6 +164,9 @@ export async function streamShardArrayField( const source = createReadStream(path) const openHook = afterStreamOpenForTests if (openHook) source.once('open', openHook) + // Detach by default (callers that retain values); the filtered loader + // passes false and detaches only what it keeps (see loadShardFiltered). + const detach = opts?.detachStrings ?? true await pipeline( source, // Packed tokens only (keys as `keyValue`, whole strings/numbers): the @@ -287,9 +292,9 @@ export async function streamShardArrayField( if (arrayFieldSeen) fail(`duplicate shard field ${arrayField}`) arrayFieldSeen = true } - await cb.onField?.(fileKey!, at[1] as string, assembleTokens([token])) + await cb.onField?.(fileKey!, at[1] as string, assembleTokens([token], detach)) } else if (at.length === 3 && at[1] === arrayField) { - await cb.onElement?.(fileKey!, key as number, assembleTokens([token])) + await cb.onElement?.(fileKey!, key as number, assembleTokens([token], detach)) elementCount++ } else { fail('shard entry has unexpected nesting') @@ -317,7 +322,7 @@ export async function streamShardArrayField( if (stack.length === capture.startDepth) { const done = capture capture = null - const assembled = assembleTokens(done.tokens) + const assembled = assembleTokens(done.tokens, detach) if (done.kind === 'element') { await cb.onElement?.(fileKey!, done.index, assembled) elementCount++ From bef3475252cffbf8549543ef9012d313af3065bb Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sun, 20 Sep 2026 09:43:22 -0400 Subject: [PATCH 03/13] fix: decode sub-gate shards with JSON.parse instead of streaming The streaming walk costs 15-20s where JSON.parse needs milliseconds (525MB of shards in ~11s on this corpus); it exists for the one shard past V8's max string length. Shards at or under 256MB now take plain JSON.parse through the same per-record projection (filterShardFile / validateCachedFile); larger ones stream exactly as before. A single readFile is already an atomic snapshot, so the small path needs no fingerprint guard. Byte-identical results on both paths are pinned by the size-gate parity suite (kept, sliced, PR-whole, key-overlap, all-dropped, invalid). --- CHANGELOG.md | 2 +- src/session-cache.ts | 72 ++++++++++++- tests/session-cache-size-gate.test.ts | 147 ++++++++++++++++++++++++++ 3 files changed, 217 insertions(+), 4 deletions(-) create mode 100644 tests/session-cache-size-gate.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ff3b09ed5..d8e2e07dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Unreleased ### Fixed -- **Ranged commands decode cache shards incrementally instead of assembling whole multi-hundred-megabyte files.** Session month shards now decode one turn at a time and keep only in-range turns (with the exact same kept/dropped/carry contract as before, verified by the range-filter suite, including a 30k-turn record test); shard loads run serially and the pre-lock snapshot is released before the canonical reload instead of overlapping it. All retained strings are detached from tokenizer buffers (sliced views pinned whole input chunks: ~1.2GB unexplained heap on this corpus). Out-of-range turns contribute their raw dedup keys to cross-file suppression markers, and provider-scoped queries skip unrelated sections. +- **Ranged commands decode cache shards incrementally instead of assembling whole multi-hundred-megabyte files.** Session month shards now decode one turn at a time and keep only in-range turns (with the exact same kept/dropped/carry contract as before, verified by the range-filter suite, including a 30k-turn record test); shard loads run serially and the pre-lock snapshot is released before the canonical reload instead of overlapping it. All retained strings are detached from tokenizer buffers (sliced views pinned whole input chunks: ~1.2GB unexplained heap on this corpus). Shards at or under 256MB skip the streaming walk and decode with plain JSON.parse through the same per-record projection (byte-identical results, pinned by the size-gate parity suite): 166MB of shards in 333ms instead of 15-20s, while the walk still bounds the shard past V8's max string length. Out-of-range turns contribute their raw dedup keys to cross-file suppression markers, and provider-scoped queries skip unrelated sections. ### Fixed (desktop) - **The Models table shows every model that ran, including the ones under a cent.** The CLI's `models` command defaults `minCost` to $0.01, and the desktop bridge passed neither `--min-cost` nor `--unpriced`, so the table silently dropped every row below a cent — which by construction excluded every unpriced row too (a `0 >= 0.01` filter), leaving #1443's dimming and add-alias affordances unreachable in the shipped app. `codeburn:getModels` now passes `--min-cost 0` (and the demo bridge mirrors it), so sub-cent and unpriced rows arrive and render with their existing dim treatment; on a real lifetime corpus that recovers 10 rows and 2,160 calls the default filter hid (43 → 53 rows, verified in both themes). A row priced between $0.00 and $0.01 renders as "$0.00" without dimming — it is genuinely priced, just below the display floor. Fixes #1465. diff --git a/src/session-cache.ts b/src/session-cache.ts index bec1ce561..6a3aa6dc8 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -1126,15 +1126,53 @@ async function readEnvelope(dir: string): Promise { } } -// A shard that is missing or malformed costs exactly the provider-months it -// held, not the provider and never the whole cache: those files re-parse while -// every other month keeps serving. +/// Shards at or under this size decode with plain JSON.parse; larger ones +/// stream. Profiled on a real corpus: 166MB of shards in 333ms via JSON.parse +/// against 15-20s through the streaming walk, while the walk exists for the +/// one shard past V8's max string length (readFile+JSON.parse hard-fails +/// there regardless of heap). Half that ceiling keeps the fast path safely +/// below it with room for UTF-16 expansion. +const SHARD_STREAM_GATE_BYTES = 256 * 1024 * 1024 +let shardStreamGateForTests: number | null = null +export function __setShardStreamGateForTests(bytes: number | null): void { + shardStreamGateForTests = bytes +} +/// True when path must take the streaming decoder. Unstatable files fall +/// through to the stream, which fails exactly the way the unreadable-shard +/// path always has (null, never a partial commit). +async function shardNeedsStreaming(path: string): Promise { + const gate = shardStreamGateForTests ?? SHARD_STREAM_GATE_BYTES + const size = await stat(path).then(s => s.size, () => null) + if (size === null) return true + return size > gate +} +/// Decode one shard with plain JSON.parse: the fast path for shards at or +/// under SHARD_STREAM_GATE_BYTES. One readFile is already an atomic snapshot, +/// and JSON.parse returns fresh strings, so nothing here can pin a tokenizer +/// buffer. Any invalid record drops the whole shard, mirroring validateFiles +/// entry for entry. +async function loadShardSmall(path: string): Promise | null> { + let raw: unknown + try { + raw = JSON.parse(await readFile(path, 'utf-8')) + } catch { + return null + } + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null + const files: Record = {} + for (const [key, value] of Object.entries(raw)) { + if (!validateCachedFile(value)) return null + files[key] = value + } + return files +} /// Assemble a whole shard without ever materializing its text: files past /// ~512MB exceed V8's max string length, so readFile+JSON.parse hard-fails /// (RangeError) regardless of heap. Per-record streaming bounds transients by /// the largest single record instead. Any invalid record drops the whole /// shard, mirroring validateFiles entry for entry. async function loadShardStreaming(path: string): Promise | null> { + if (!(await shardNeedsStreaming(path))) return loadShardSmall(path) const files: Record = {} try { await streamShardEntries(path, ({ key, value }) => { @@ -1456,6 +1494,30 @@ function filterShardFile(value: unknown, keepTurn: TurnFilter): CachedFile | nul return sliced.whole ? file : sliced.file } +/// Small-file twin of loadShardFiltered for shards at or under +/// SHARD_STREAM_GATE_BYTES: one readFile is already an atomic snapshot, so +/// no fingerprint guard is needed, and every record projects through the +/// same filterShardFile the streaming second pass uses (whole files, +/// inexact slices included, stay whole with no second read). +async function loadShardFilteredSmall( + shardPath: string, + turnFilter: TurnFilter, +): Promise | null> { + let raw: unknown + try { + raw = JSON.parse(await readFile(shardPath, 'utf-8')) + } catch { + return null + } + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null + const files: Record = {} + for (const [key, value] of Object.entries(raw)) { + const file = filterShardFile(value, turnFilter) + if (!file) return null + files[key] = file + } + return files +} /// Decode one shard with a TurnFilter, streaming turns so peak heap tracks /// the retained slice plus one turn rather than the shard. Turns are /// validated as decoded (kept or dropped); the unmarked reconstruction is @@ -1467,6 +1529,9 @@ function filterShardFile(value: unknown, keepTurn: TurnFilter): CachedFile | nul /// Whole-file gates (PR links, gappy kept turns, cross-side key overlap) are /// collected during the stream and decoded whole in one second pass over the /// shard: rare cases pay a re-read while the common path stays bounded. +/// +/// Shards at or under SHARD_STREAM_GATE_BYTES skip the walk: one readFile +/// plus filterShardFile per record (see loadShardFilteredSmall). export async function loadShardFiltered( dir: string, name: string, @@ -1484,6 +1549,7 @@ export async function loadShardFiltered( } const builds = new Map() const shardPath = join(dir, name) + if (!(await shardNeedsStreaming(shardPath))) return loadShardFilteredSmall(shardPath, turnFilter) const fingerprintOf = async (): Promise => stat(shardPath).then(s => `${s.dev}:${s.ino}:${s.size}:${s.mtimeMs}`, () => null) const before = await fingerprintOf() diff --git a/tests/session-cache-size-gate.test.ts b/tests/session-cache-size-gate.test.ts new file mode 100644 index 000000000..77d923987 --- /dev/null +++ b/tests/session-cache-size-gate.test.ts @@ -0,0 +1,147 @@ +// Size-gate parity: shards at or under SHARD_STREAM_GATE_BYTES take the plain +// JSON.parse path; larger ones stream. Both paths must serve byte-identical +// results (and identical nulls) for whole and range-filtered loads, or the +// fast path would silently change reports on every machine whose shards fit +// under the gate. The gate override forces each path on the same bytes. +import { mkdir, rm, writeFile } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { + __setShardStreamGateForTests, + clearShardMemo, + loadShardFiltered, + loadShardMemoized, + sessionCacheDir, + type CachedFile, +} from '../src/session-cache.js' + +type Turn = CachedFile['turns'][number] + +function callAt(timestamp: string, key: string): Turn['calls'][number] { + return { + provider: 'omp', + model: 'm', + usage: { + inputTokens: 10, + outputTokens: 5, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + cacheCreationOneHourTokens: 0, + }, + speed: 'standard', + timestamp, + tools: [], + bashCommands: [], + skills: [], + subagentTypes: [], + deduplicationKey: key, + } +} + +function turnAt(timestamp: string, key: string): Turn { + return { + timestamp, + sessionId: 'sess-1', + userMessage: 'do the thing', + calls: [callAt(timestamp, key)], + } +} + +function cachedFile(overrides: Partial = {}): CachedFile { + return { + fingerprint: { dev: 1, ino: 2, mtimeMs: 3, sizeBytes: 4 }, + lastCompleteLineOffset: 128, + mcpInventory: [], + turns: [turnAt('2026-07-15T10:00:00Z', 'day15-1')], + ...overrides, + } +} + +const DAY = { start: new Date('2026-07-15T00:00:00Z'), end: new Date('2026-07-15T23:59:59.999Z') } + +function keepDay(turn: Turn): boolean { + return turn.calls.some(c => { + const ts = new Date(c.timestamp).getTime() + return !Number.isNaN(ts) && ts >= DAY.start.getTime() && ts <= DAY.end.getTime() + }) +} +function mixedCorpus(): Record { + return { + '/live/kept.jsonl': cachedFile({ + turns: [turnAt('2026-07-15T10:00:00Z', 'k-1'), turnAt('2026-07-15T11:00:00Z', 'k-2')], + }), + '/live/sliced.jsonl': cachedFile({ + turns: [ + turnAt('2026-07-10T10:00:00Z', 'd-1'), + { ...turnAt('2026-07-15T10:00:00Z', 'k-3'), gitBranch: 'main' }, + ], + }), + '/live/pr.jsonl': cachedFile({ + prLinks: ['https://example.com/x/1'], + turns: [turnAt('2026-07-10T10:00:00Z', 'p-1'), turnAt('2026-07-15T10:00:00Z', 'p-2')], + }), + '/live/overlap.jsonl': cachedFile({ + turns: [turnAt('2026-07-10T10:00:00Z', 'same'), turnAt('2026-07-15T10:00:00Z', 'same')], + }), + '/live/dropped.jsonl': cachedFile({ + turns: [turnAt('2026-07-10T10:00:00Z', 'x-1')], + }), + } +} + +let dir: string + +beforeEach(async () => { + dir = join(tmpdir(), `codeburn-sizegate-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`) + await mkdir(dir, { recursive: true }) + clearShardMemo() +}) + +afterEach(async () => { + __setShardStreamGateForTests(null) + clearShardMemo() + await rm(dir, { recursive: true, force: true }) +}) + +describe('shard size gate parity', () => { + it('serves identical filtered loads from the parse and stream paths', async () => { + const name = 'omp.2026-07.gate.json' + await writeFile(join(dir, name), JSON.stringify(mixedCorpus())) + __setShardStreamGateForTests(Number.POSITIVE_INFINITY) + const parsed = await loadShardFiltered(dir, name, keepDay) + __setShardStreamGateForTests(0) + const streamed = await loadShardFiltered(dir, name, keepDay) + expect(streamed).toEqual(parsed) + expect(Object.keys(parsed!)).toHaveLength(5) + // Spot-check the shapes survived both paths, not just equally. + expect(parsed!['/live/sliced.jsonl']!.turns).toHaveLength(1) + expect(parsed!['/live/sliced.jsonl']!.rangeFiltered).toBeDefined() + expect(parsed!['/live/pr.jsonl']!.rangeFiltered).toBeUndefined() + expect(parsed!['/live/pr.jsonl']!.turns).toHaveLength(2) + }) + + it('serves identical whole loads from the parse and stream paths', async () => { + const corpus = mixedCorpus() + await writeFile(join(dir, 'gate-parse.json'), JSON.stringify(corpus)) + await writeFile(join(dir, 'gate-stream.json'), JSON.stringify(corpus)) + __setShardStreamGateForTests(Number.POSITIVE_INFINITY) + const parsed = await loadShardMemoized(dir, 'gate-parse.json') + __setShardStreamGateForTests(0) + const streamed = await loadShardMemoized(dir, 'gate-stream.json') + expect(streamed).toEqual(parsed) + }) + + it('drops the shard on both paths for an invalid record', async () => { + const bad = { ...mixedCorpus(), '/live/bad.jsonl': cachedFile({ turns: 'nope' as unknown as [] }) } + await writeFile(join(dir, 'gate-bad.json'), JSON.stringify(bad)) + __setShardStreamGateForTests(Number.POSITIVE_INFINITY) + expect(await loadShardFiltered(dir, 'gate-bad.json', keepDay)).toBeNull() + __setShardStreamGateForTests(0) + expect(await loadShardFiltered(dir, 'gate-bad.json', keepDay)).toBeNull() + }) +}) From f13b35f861cb53ad60f3bbf20ed8081ebe138cc4 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sat, 19 Sep 2026 21:24:46 -0400 Subject: [PATCH 04/13] fix: stream the codex result cache instead of whole-parsing The single result file decodes entry by entry; discovery labels come from calls-free metadata; publishes merge dirty entries; single-flight loads are shared and retry on mid-decode publish, with a global clear epoch. --- CHANGELOG.md | 1 + src/codex-cache.ts | 736 +++++++++++++++++++--- tests/codex-cache-concurrent-load.test.ts | 117 +++- tests/codex-range-filter.test.ts | 449 +++++++++++++ 4 files changed, 1199 insertions(+), 104 deletions(-) create mode 100644 tests/codex-range-filter.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e2e07dd..25db11202 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed - **Ranged commands decode cache shards incrementally instead of assembling whole multi-hundred-megabyte files.** Session month shards now decode one turn at a time and keep only in-range turns (with the exact same kept/dropped/carry contract as before, verified by the range-filter suite, including a 30k-turn record test); shard loads run serially and the pre-lock snapshot is released before the canonical reload instead of overlapping it. All retained strings are detached from tokenizer buffers (sliced views pinned whole input chunks: ~1.2GB unexplained heap on this corpus). Shards at or under 256MB skip the streaming walk and decode with plain JSON.parse through the same per-record projection (byte-identical results, pinned by the size-gate parity suite): 166MB of shards in 333ms instead of 15-20s, while the walk still bounds the shard past V8's max string length. Out-of-range turns contribute their raw dedup keys to cross-file suppression markers, and provider-scoped queries skip unrelated sections. + - **The codex result cache streams instead of whole-parsing.** The single result file decodes entry by entry (a top-level decode assembled hundreds of megabytes first); discovery labels come from calls-free metadata without loading the calls map; publishes merge dirty entries over the published bytes instead of rewriting from memory; single-flight loads are shared across concurrent readers and retry when a publish lands mid-decode, with a global clear epoch so in-flight loads cannot repopulate a cleared memo. ### Fixed (desktop) - **The Models table shows every model that ran, including the ones under a cent.** The CLI's `models` command defaults `minCost` to $0.01, and the desktop bridge passed neither `--min-cost` nor `--unpriced`, so the table silently dropped every row below a cent — which by construction excluded every unpriced row too (a `0 >= 0.01` filter), leaving #1443's dimming and add-alias affordances unreachable in the shipped app. `codeburn:getModels` now passes `--min-cost 0` (and the demo bridge mirrors it), so sub-cent and unpriced rows arrive and render with their existing dim treatment; on a real lifetime corpus that recovers 10 rows and 2,160 calls the default filter hid (43 → 53 rows, verified in both themes). A row priced between $0.00 and $0.01 renders as "$0.00" without dimming — it is genuinely priced, just below the display floor. Fixes #1465. diff --git a/src/codex-cache.ts b/src/codex-cache.ts index a24f174b0..76253dcb5 100644 --- a/src/codex-cache.ts +++ b/src/codex-cache.ts @@ -3,9 +3,16 @@ import { existsSync } from 'fs' import { randomBytes } from 'crypto' import { join, resolve } from 'path' import { AsyncLocalStorage } from 'node:async_hooks' +import { createReadStream } from 'node:fs' +import { pipeline } from 'node:stream/promises' +import { ignore } from 'stream-json/filters/ignore.js' +import { pick } from 'stream-json/filters/pick.js' +import { streamObject } from 'stream-json/streamers/stream-object.js' -import { getCodeburnCacheDir, readExistingTextFile } from './cache-dir.js' +import { getCodeburnCacheDir } from './cache-dir.js' import type { ParsedProviderCall } from './providers/types.js' +import { streamShardArrayField, writeChunk } from './shard-stream.js' +import { flatString, flattenJsonStrings } from './content-utils.js' // v4: attribute MCP calls emitted as event_msg/mcp_tool_call_end (issue #478). // Recent Codex sessions cached under v3 dropped these, so force a re-parse. @@ -111,71 +118,409 @@ function isCurrentCache(cache: ResultCache): boolean { // Embedded consumers can change CODEBURN_CACHE_DIR without reloading this // module. Keep each directory's in-memory state separate so a warm cache (or an // unflushed update) from A can never be read from or written into B. -const memCaches = new Map() +type MemState = { + cache: ResultCache + /// Null for a complete load; otherwise the range-start floor a filtered + /// load retained. Loads with different identities never share entries. + rangeStartMs: number | null +} +const memCaches = new Map() + +// Fresh full records written since load, keyed by directory. Deliberately +// range-independent: a fresh parse is complete under any projection, so one +// overlay serves every concurrent range. Entries live here only while +// unflushed: a successful flush publishes them and releases the map (disk is +// authoritative afterwards; later lookups re-stream once per snapshot +// identity), so CLI runs that never clear still track the dirty set instead +// of the corpus. +const codexOverlay = new Map>() + +// Project labels for discovery, WITHOUT loading the calls map: the full +// result file would otherwise parse on every run just to name projects (see +// discoverSessionFile). Rebuilt by one streaming pass when the file changes; +// updated incrementally on write and flush. +type ProjectIndexRow = { project: string; dev?: number; ino?: number; mtimeMs?: number; sizeBytes?: number } +type ProjectIndexState = { builtForMtimeMs: number; rows: Map } +const projectIndexes = new Map() // Dropped by the resident RSS guard. Every write is published by // flushCodexCache() in the parse's finally, so the next load re-reads disk. export function clearCodexMemCaches(): void { memCaches.clear() inFlightLoads.clear() + codexOverlay.clear() + projectIndexes.clear() + indexBuilds.clear() + // A cleared in-flight load cannot be cancelled; without this it would + // repopulate memCaches after resolving (defeating the RSS guard). The + // load loop observes the epoch across its read and skips the memo when a + // clear intervened (a clear changes no disk bytes, so the data stays + // servable — only the repopulation is dropped). + memEpoch++ } +// Concurrent discovery callers must share one index build: without this every +// in-flight caller would stream-decode the same hundreds-of-MB file. Keyed by +// directory and file mtime so a rebuild after the file changes never joins a +// stale build. +const indexBuilds = new Map>>() + // Concurrent callers must share one load. The memo below is only populated -// after the read + JSON.parse resolves, so without this every in-flight caller -// would re-read and re-parse the same (hundreds-of-MB) cache file. -const inFlightLoads = new Map>() +// after the read resolves, so without this every in-flight caller would +// re-read and re-parse the same (hundreds-of-MB) cache file. Keyed by +// directory AND range identity (see MemState): concurrent loads for different +// ranges must never share entries. +const inFlightLoads = new Map>() -function loadCache(cacheDir: string): Promise { - const inMemory = memCaches.get(cacheDir) - if (inMemory) return Promise.resolve(inMemory) - const pending = inFlightLoads.get(cacheDir) - if (pending) return pending - const load = loadCacheFromDisk(cacheDir).finally(() => inFlightLoads.delete(cacheDir)) - inFlightLoads.set(cacheDir, load) - return load +/// Store generation per directory: a flush that publishes bumps it and drops +/// snapshots, so a load that started before the flush never republishes stale +/// bytes afterwards (deleting inFlightLoads alone cannot cancel the promise). +const flushGenerations = new Map() +/// Global clear epoch: bumped by clearCodexMemCaches (see above). Captured +/// beside the per-directory generation before I/O and re-checked after. +let memEpoch = 0 + +/// Per-directory flush mutex: two concurrent flushes would merge against the +/// same published bytes and race the rename, losing one batch. Loads stay +/// unlocked (they never mutate); staleness is handled by the generation. +const flushChains = new Map>() +async function withFlushMutex(cacheDir: string, fn: () => Promise): Promise { + const prev = flushChains.get(cacheDir) ?? Promise.resolve() + let release!: () => void + const gate = new Promise(resolve => { release = resolve }) + const joined = prev.then(() => gate) + flushChains.set(cacheDir, joined) + await prev + try { + return await fn() + } finally { + release() + if (flushChains.get(cacheDir) === joined) flushChains.delete(cacheDir) + } } -async function loadCacheFromDisk(cacheDir: string): Promise { - const empty = { version: CODEX_CACHE_VERSION, files: {} } - const versioned = await readExistingTextFile(getCachePath(cacheDir)) - if (versioned.status === 'ok') { - try { - const cache = JSON.parse(versioned.text) as ResultCache - if (isCurrentCache(cache)) { - memCaches.set(cacheDir, cache) - return cache - } - } catch {} - memCaches.set(cacheDir, empty) - return empty +/// Snapshot key: directory AND range identity. A lookup whose identity +/// mismatches every resident snapshot reloads instead of sharing. +function snapshotKey(cacheDir: string, rangeStartMs: number | null): string { + return `${cacheDir}\0${rangeStartMs === null ? 'full' : `from:${rangeStartMs}`}` +} + +/// At most this many snapshots per directory (LRU); the resident RSS guard +/// can drop everything at any time via clearCodexMemCaches. +const MAX_SNAPSHOTS_PER_DIR = 3 + +function storeSnapshot(cacheDir: string, state: MemState): void { + memCaches.set(snapshotKey(cacheDir, state.rangeStartMs), state) + const prefix = `${cacheDir}\0` + const owned: string[] = [] + for (const key of memCaches.keys()) { + if (key.startsWith(prefix)) owned.push(key) } - if (versioned.status === 'unreadable') { - memCaches.set(cacheDir, empty) - return empty + while (owned.length > MAX_SNAPSHOTS_PER_DIR) { + const oldest = owned.shift()! + memCaches.delete(oldest) } - // Versioned file is absent (ENOENT). Adopt the unsuffixed file only when its - // version matches — old binaries still own that path; we never write or delete it. +} + +function residentSnapshot(cacheDir: string, rangeStartMs: number | null): ResultCache | null { + const key = snapshotKey(cacheDir, rangeStartMs) + const entry = memCaches.get(key) + if (!entry) return null + // Refresh recency for the LRU cap. + memCaches.delete(key) + memCaches.set(key, entry) + return entry.cache +} + +async function loadCache(cacheDir: string, rangeStartMs?: number): Promise { + const identity = rangeStartMs ?? null + const resident = residentSnapshot(cacheDir, identity) + if (resident) return resident + const pendingKey = snapshotKey(cacheDir, identity) + const pending = inFlightLoads.get(pendingKey) + if (pending) return (await pending).cache + const decode = async (): Promise => + identity === null + ? loadCacheFullFromDisk(cacheDir) + : loadCacheFilteredFromDisk(cacheDir, identity) + const load = (async (): Promise => { + for (;;) { + const gen = flushGenerations.get(cacheDir) ?? 0 + const epoch = memEpoch + const cache = await decode() + if ((flushGenerations.get(cacheDir) ?? 0) !== gen) continue + const state: MemState = { cache, rangeStartMs: identity } + // A clear intervened: servable bytes, but the RSS guard dropped the + // memo — do not repopulate it. + if (memEpoch !== epoch) return state + storeSnapshot(cacheDir, state) + return state + } + })().finally(() => { + if (inFlightLoads.get(pendingKey) === load) inFlightLoads.delete(pendingKey) + }) + inFlightLoads.set(pendingKey, load) + return (await load).cache +} + +/// Version stamped at the head of a result file. Both writers construct +/// `{version, files}` in that order, so a short head read verifies currency +/// without scanning; anything else (foreign layout, corruption) fails closed +/// exactly like a version mismatch does today. +async function readResultsVersion(path: string): Promise { + let head: string try { - const raw = await readFile(getLegacyCachePath(cacheDir), 'utf-8') - const cache = JSON.parse(raw) as ResultCache - if (isCurrentCache(cache)) { - memCaches.set(cacheDir, cache) - return cache + const handle = await open(path, 'r') + try { + const buf = Buffer.alloc(128) + const { bytesRead } = await handle.read(buf, 0, 128, 0) + head = buf.toString('utf-8', 0, bytesRead).replace(/^\uFEFF/, '') + } finally { + await handle.close() } + } catch { + return null + } + const match = /^\s*\{\s*"version"\s*:\s*(\d+)/.exec(head) + return match ? Number(match[1]) : null +} + +/// Resolve the readable result file, mirroring the historical precedence: +/// the versioned file when present, else the legacy unsuffixed file. +async function resolveResultsPath(cacheDir: string): Promise { + const versioned = getCachePath(cacheDir) + try { + await stat(versioned) + return versioned + } catch {} + const legacy = getLegacyCachePath(cacheDir) + try { + await stat(legacy) + return legacy } catch {} - memCaches.set(cacheDir, empty) - return empty + return null +} + +/// Keep rule for ranged codex loads. An entry survives when its file may +/// still be looked up (mtime at/after the range start — both lookup paths +/// skip older files first) or any call falls in range. Everything else can +/// never be observed: lookups are mtime-floor-gated and fp-gated, and a miss +/// re-parses from source. Unjudgeable entries (no mtime, malformed or missing +/// timestamps, empty calls) are kept: the full load performs no per-entry +export function retainCodexEntry(value: unknown, rangeStartMs: number): boolean { + if (!value || typeof value !== 'object') return true + if ('mtimeMs' in value && typeof value.mtimeMs === 'number' && value.mtimeMs >= rangeStartMs) return true + if (!('calls' in value) || !Array.isArray(value.calls)) return true + let seenCall = false + for (const call of value.calls) { + seenCall = true + if (!call || typeof call !== 'object') return true + if (!('timestamp' in call) || typeof call.timestamp !== 'string') return true + const ts = Date.parse(call.timestamp) + if (Number.isNaN(ts) || ts >= rangeStartMs) return true + } + return !seenCall +} + +/// Streaming load of one result file: version-gated, then per-entry through +/// the pick filter (the nested `files` object streams entry by entry — a +/// top-level decode would assemble all 227MB first). Any failure drops the +/// whole file, mirroring the historical whole-file behavior entry for entry. +async function loadResultsStreaming( + path: string, + retain: (value: unknown) => boolean, + retainKeys?: Set, +): Promise | null> { + if ((await readResultsVersion(path)) !== CODEX_CACHE_VERSION) return null + const files: Record = {} + try { + await streamCodexEntries(path, (key, value) => { + if (!retain(value)) return + // Detach before storing (see assembleTokens): snapshots hold these + // entries long-term, and accumulating unflattened records would pin + // every tokenizer chunk. In place on the decoder-owned value — no + // copy, no serialization transient (see flattenJsonStrings). + files[flatString(key)] = flattenJsonStrings(value) + }, retainKeys ? { retainKeys } : undefined) + } catch { + return null + } + return files +} + +async function loadCacheFullFromDisk(cacheDir: string): Promise { + const empty = { version: CODEX_CACHE_VERSION, files: {} } + const path = await resolveResultsPath(cacheDir) + if (!path) return empty + const files = await loadResultsStreaming(path, () => true) + if (!files) return empty + // Parity with the historical whole-file `as ResultCache` cast: entries are + // stored unvalidated and every consumer reads them defensively. + return { version: CODEX_CACHE_VERSION, files: files as Record } +} + +/// First pass of a range load: stream one call at a time (never an entry) +/// and decide each file's fate by the exact `retainCodexEntry` rule — +/// file-mtime first, then call timestamps. Returns the keys to assemble whole +/// in pass two. Any decode failure throws, and the caller falls back to the +/// single-pass load (same bytes-or-null contract as a torn file today). +export async function scanRetainedCodexKeys(path: string, rangeStartMs: number): Promise> { + const retained = new Set() + type Scan = { mtimeMs: number | null; callsNonArray: boolean; keep: boolean; count: number } + const scans = new Map() + await streamShardArrayField(path, 'calls', { + onFileStart: key => { + scans.set(key, { mtimeMs: null, callsNonArray: false, keep: false, count: 0 }) + }, + onField: (key, field, value) => { + const scan = scans.get(key)! + if (field === 'mtimeMs' && typeof value === 'number') scan.mtimeMs = value + if (field === 'calls') scan.callsNonArray = true + }, + onElement: (key, _index, value) => { + const scan = scans.get(key)! + scan.count++ + if (scan.keep) return + // Mirrors retainCodexEntry call-for-call: any unjudgeable or in-range + // call keeps the whole entry. + if (!value || typeof value !== 'object') { scan.keep = true; return } + const timestamp = (value as Record)['timestamp'] + if (typeof timestamp !== 'string') { scan.keep = true; return } + const ms = Date.parse(timestamp) + if (Number.isNaN(ms) || ms >= rangeStartMs) scan.keep = true + }, + onFileEnd: (key, _count, arraySeen) => { + const scan = scans.get(key)! + scans.delete(key) + if (scan.mtimeMs !== null && scan.mtimeMs >= rangeStartMs) { retained.add(key); return } + // Missing or non-array calls are unjudgeable: the full load carries + // them, so the filtered load keeps them too. + if (!arraySeen || scan.callsNonArray) { retained.add(key); return } + if (scan.keep || scan.count === 0) { retained.add(key); return } + // Old file, non-empty calls, every timestamp valid and pre-range: drop. + }, + }, { rootField: 'files' }) + return retained +} +export async function loadCacheFilteredFromDisk(cacheDir: string, rangeStartMs: number): Promise { + const empty = { version: CODEX_CACHE_VERSION, files: {} } + const path = await resolveResultsPath(cacheDir) + if (!path) return empty + // Two-pass: the timestamp scan assembles at most one call at a time, then + // only retained files stream whole. Assembling-then-dropping would + // materialize monster entries just to discard them. + const before = await stat(path).catch(() => null) + const beforeKey = before ? `${before.dev}:${before.ino}:${before.size}:${before.mtimeMs}` : null + // fingerprint can vouch for afterwards (a null/null match below would pass). + if (beforeKey === null) return empty + let retained: Set + try { + if ((await readResultsVersion(path)) !== CODEX_CACHE_VERSION) return empty + retained = await scanRetainedCodexKeys(path, rangeStartMs) + } catch { + // Torn/unreadable: a miss, exactly as the whole-file decoder's throw. + return empty + } + const fingerprintNow = async (): Promise => { + const st = await stat(path).catch(() => null) + return st ? `${st.dev}:${st.ino}:${st.size}:${st.mtimeMs}` : null + } + // A concurrent publish between or during the passes would apply old + // decisions to new bytes: verify the file is untouched before and after + // the retained pass, else treat as a miss (bounded re-parse). + if ((await fingerprintNow()) !== beforeKey) return empty + if (retained.size === 0) return empty + const files = await loadResultsStreaming(path, () => true, retained) + if (!files) return empty + if ((await fingerprintNow()) !== beforeKey) return empty + return { version: CODEX_CACHE_VERSION, files: files as Record } +} + +/// Stream the nested `files` entries of a result file (see loadResultsStreaming). +/// `skipCalls` drops `calls` arrays at the token level, before assembly: +/// metadata consumers (project labels, fingerprints) never materialize call +/// lists, so one giant entry cannot blow discovery's heap. `retainKeys` drops +/// whole non-retained file subtrees before assembly: the second pass of a +/// range load materializes only files the timestamp scan kept. Both match +/// exact stacks below the `pick` re-root (`[filePath, 'calls']` and +/// `[filePath]`), so dotted paths can never misfire them. +async function streamCodexEntries( + path: string, + onEntry: (key: string, value: unknown) => void | Promise, + opts?: { skipCalls?: boolean; retainKeys?: Set }, +): Promise { + const sink = async function* (entries: AsyncIterable<{ key: string; value: unknown }>): AsyncGenerator { + for await (const entry of entries) { + if (typeof entry?.key !== 'string') throw new Error(`codex results entry without key: ${path}`) + await onEntry(entry.key, entry.value) + } + } + if (opts?.skipCalls || opts?.retainKeys) { + const retained = opts.retainKeys + await pipeline( + createReadStream(path), + pick.withParserAsStream({ filter: 'files' }), + ignore.asStream({ + filter: (stack: (string | number | null)[]) => { + if (retained && stack.length === 1 && typeof stack[0] === 'string' && !retained.has(stack[0])) return true + if (opts.skipCalls && stack.length === 2 && stack[1] === 'calls') return true + return false + }, + }), + streamObject.asStream(), + sink, + ) + return + } + await pipeline( + createReadStream(path), + pick.withParserAsStream({ filter: 'files' }), + streamObject.asStream(), + sink, + ) +} + +/// Entry metadata (project labels plus fingerprints) without `calls`, for +/// discovery: bounded heap no matter how large one entry's call list is. +export async function streamCodexEntryMetadata( + path: string, + onEntry: (key: string, value: unknown) => void | Promise, +): Promise { + await streamCodexEntries(path, onEntry, { skipCalls: true }) } -function getEntry(cache: ResultCache, filePath: string, fp: FileFingerprint): FileEntry | null { - if (!Object.hasOwn(cache.files, filePath)) return null - const entry = cache.files[filePath] +/// Exact-or-resume lookup shared by snapshot and overlay entries: an exact +/// fingerprint match serves calls verbatim, while a grown same-inode file +/// resumes from its recorded boundary. Callers consult the overlay first +/// (fresh writes supersede snapshots under every projection). +async function hitFromEntry( + entry: FileEntry | undefined, + fp: FileFingerprint, + filePath: string, +): Promise { if (entry && entry.mtimeMs === fp.mtimeMs && entry.sizeBytes === fp.sizeBytes) { - return entry + return { kind: 'exact', calls: entry.calls } + } + if ( + entry + && entry.dev === fp.dev + && entry.ino === fp.ino + && entry.resumeOffset !== undefined + && entry.resumeState !== undefined + && entry.resumeCallCount !== undefined + && fp.sizeBytes > entry.sizeBytes + && entry.resumeOffset <= fp.sizeBytes + && await endsLineAt(filePath, entry.resumeOffset) + ) { + return { kind: 'resume', calls: entry.calls, offset: entry.resumeOffset, state: entry.resumeState, callCount: entry.resumeCallCount } } return null } +function readOverlayEntry(cacheDir: string, filePath: string): FileEntry | undefined { + return codexOverlay.get(cacheDir)?.get(filePath) +} + // A grown file is only assumed to be an APPEND if the recorded boundary still // falls right after a newline. A same-inode rewrite (truncate + refill, or an // in-place edit) that happens to end up larger would otherwise resume into the @@ -198,41 +543,94 @@ async function endsLineAt(filePath: string, offset: number): Promise { export async function readCachedCodexResults( filePath: string, + opts?: { rangeStartMs?: number }, ): Promise { try { const s = await stat(filePath) - const cache = await loadCache(currentCacheDir()) + const cacheDir = currentCacheDir() const fp = { dev: s.dev, ino: s.ino, mtimeMs: s.mtimeMs, sizeBytes: s.size } - const entry = getEntry(cache, filePath, fp) - if (entry) return { kind: 'exact', calls: entry.calls } + // Fresh writes supersede snapshots under every projection: a fresh full + // record is servable for any range (downstream slices to the query). + const overlaid = readOverlayEntry(cacheDir, filePath) + if (overlaid) { + const hit = await hitFromEntry(overlaid, fp, filePath) + if (hit) return hit + } // Rollouts are append-only: the same inode, grown past a boundary we // recorded, can be picked up from that boundary instead of re-read whole. - const stale = cache.files[filePath] - if ( - stale - && stale.dev === fp.dev - && stale.ino === fp.ino - && stale.resumeOffset !== undefined - && stale.resumeState !== undefined - && stale.resumeCallCount !== undefined - && fp.sizeBytes > stale.sizeBytes - && stale.resumeOffset <= fp.sizeBytes - && await endsLineAt(filePath, stale.resumeOffset) - ) { - return { kind: 'resume', calls: stale.calls, offset: stale.resumeOffset, state: stale.resumeState, callCount: stale.resumeCallCount } - } + const cache = await loadCache(cacheDir, opts?.rangeStartMs) + return hitFromEntry(cache.files[filePath], fp, filePath) } catch {} return null } +/// Project labels for discovery without loading the calls map. The index +/// carries only names plus fingerprints (megabytes, not hundreds of them); +/// label misses fall back to parsing the file, exactly as a cache miss does. +async function loadProjectIndex(cacheDir: string): Promise> { + const versioned = getCachePath(cacheDir) + const legacy = getLegacyCachePath(cacheDir) + const versionedStat = await stat(versioned).catch(() => null) + const source = versionedStat + ? { path: versioned, mtimeMs: versionedStat.mtimeMs } + : await stat(legacy).then( + s => ({ path: legacy, mtimeMs: s.mtimeMs }), + () => null, + ) + const cur = projectIndexes.get(cacheDir) + if (cur && source && source.mtimeMs === cur.builtForMtimeMs) return cur.rows + const buildKey = `${cacheDir}\0${source?.mtimeMs ?? -1}` + const inflight = indexBuilds.get(buildKey) + if (inflight) return inflight + const build = buildProjectIndex(cacheDir, source).finally(() => { + if (indexBuilds.get(buildKey) === build) indexBuilds.delete(buildKey) + }) + indexBuilds.set(buildKey, build) + return build +} + +async function buildProjectIndex( + cacheDir: string, + source: { path: string; mtimeMs: number } | null, +): Promise> { + const rows = new Map() + if (source && (await readResultsVersion(source.path)) === CODEX_CACHE_VERSION) { + try { + await streamCodexEntryMetadata(source.path, (key, value) => { + if (!value || typeof value !== 'object') return + // Detach both from the tokenizer's buffers (index rows outlive the + // stream by the life of the process). + const row: ProjectIndexRow = { + project: 'project' in value && typeof value.project === 'string' ? flatString(value.project) : '', + } + if ('dev' in value && typeof value.dev === 'number') row.dev = value.dev + if ('ino' in value && typeof value.ino === 'number') row.ino = value.ino + if ('mtimeMs' in value && typeof value.mtimeMs === 'number') row.mtimeMs = value.mtimeMs + if ('sizeBytes' in value && typeof value.sizeBytes === 'number') row.sizeBytes = value.sizeBytes + rows.set(flatString(key), row) + }) + } catch { + // Torn/unreadable mid-build: serve the partial rows; misses fall back + // to parsing the file, and the next file change rebuilds cleanly. + } + } + projectIndexes.set(cacheDir, { builtForMtimeMs: source?.mtimeMs ?? -1, rows }) + return rows +} + export async function getCachedCodexProject( filePath: string, ): Promise { try { const s = await stat(filePath) - const cache = await loadCache(currentCacheDir()) - const entry = getEntry(cache, filePath, { dev: s.dev, ino: s.ino, mtimeMs: s.mtimeMs, sizeBytes: s.size }) - return entry?.project ?? null + const rows = await loadProjectIndex(currentCacheDir()) + const row = rows.get(filePath) + // Same freshness contract the full-map lookup enforced: only an + // mtime/size-matching label is served, otherwise discovery re-parses. + if (row && row.mtimeMs === s.mtimeMs && row.sizeBytes === s.size) { + return row.project || null + } + return null } catch {} return null } @@ -256,8 +654,11 @@ export async function writeCachedCodexResults( resume?: { offset: number; state: unknown; callCount: number }, ): Promise { try { - const cache = await loadCache(currentCacheDir()) - cache.files[filePath] = { + const cacheDir = currentCacheDir() + // Overlay only: snapshots stay immutable so concurrent ranges never mix. + // A fresh full record is servable under every projection, and the flush + // merges it over the published bytes (never a whole rewrite from memory). + const entry: FileEntry = { dev: fingerprint.dev, ino: fingerprint.ino, mtimeMs: fingerprint.mtimeMs, @@ -266,40 +667,197 @@ export async function writeCachedCodexResults( calls, ...(resume ? { resumeOffset: resume.offset, resumeState: resume.state, resumeCallCount: resume.callCount } : {}), } + let overlay = codexOverlay.get(cacheDir) + if (!overlay) { + overlay = new Map() + codexOverlay.set(cacheDir, overlay) + } + overlay.set(filePath, entry) + // Keep the project index warm so discovery labels never trigger a rescan. + const idx = projectIndexes.get(cacheDir) + if (idx) { + idx.rows.set(filePath, { + project, + dev: fingerprint.dev, + ino: fingerprint.ino, + mtimeMs: fingerprint.mtimeMs, + sizeBytes: fingerprint.sizeBytes, + }) + } } catch {} } -export async function flushCodexCache(): Promise { - const cacheDir = currentCacheDir() - const memCache = memCaches.get(cacheDir) - if (!memCache) return +/// Merge dirty overlay entries over the published result file, streaming so +/// peak heap tracks one record rather than the file. Every touched path is +/// stat-checked inline: missing files drop (eviction parity with the +/// historical whole-map sweep, which also statted every known path), dirty +/// entries substitute, everything else is carried verbatim. Returns the +/// evicted paths so callers can prune indexes and snapshots. +async function mergeCodexResults( + publishedPath: string, + finalPath: string, + dirty: Map, +): Promise> { + const evicted = new Set() + const tempPath = `${finalPath}.${randomBytes(8).toString('hex')}.tmp` + const handle = await open(tempPath, 'w', 0o600) + const fail = async (err: unknown): Promise => { + try { await handle.close() } catch {} + await unlink(tempPath).catch(() => null) + throw err + } + const pathAlive = async (path: string): Promise => { + try { + await stat(path) + return true + } catch { + return false + } + } try { - // Evict entries for files that no longer exist on disk - const paths = Object.keys(memCache.files) - for (const p of paths) { - try { - await stat(p) - } catch { - delete memCache.files[p] + await writeChunk(handle, `{"version":${CODEX_CACHE_VERSION},"files":{`) + let first = true + const emitPair = async (key: string, value: unknown): Promise => { + await writeChunk(handle, `${first ? '' : ','}${JSON.stringify(key)}:${JSON.stringify(value)}`) + first = false + } + await streamCodexEntries(publishedPath, async (key, value) => { + if (!(await pathAlive(key))) { + evicted.add(key) + return + } + const overwrite = dirty.get(key) + if (overwrite !== undefined) { + dirty.delete(key) + await emitPair(key, overwrite) + return } + await emitPair(key, value) + }) + for (const [key, file] of dirty) { + if (!(await pathAlive(key))) { + evicted.add(key) + continue + } + await emitPair(key, file) } + await writeChunk(handle, '}}') + await handle.sync() + await handle.close() + } catch (err) { + await fail(err) + } + try { + await rename(tempPath, finalPath) + } catch (err) { + try { await unlink(tempPath) } catch {} + throw err + } + return evicted +} +export async function flushCodexCache(): Promise { + const cacheDir = currentCacheDir() + return withFlushMutex(cacheDir, () => flushCodexCacheInner(cacheDir)) +} + +// Test barrier (see the overlapping-flush test): flushes invoke this after +// detaching, before any publish I/O, so tests can stage concurrent writes +// deterministically. Never set outside tests. +let afterDetachForTests: (() => Promise) | null = null +export function __setAfterDetachForTests(gate: (() => Promise) | null): void { + afterDetachForTests = gate +} + +async function flushCodexCacheInner(cacheDir: string): Promise { + // Detach the overlay up front so concurrent writes during the awaits below + // land in a fresh map instead of being dropped with the flushed one. + const detached = codexOverlay.get(cacheDir) + codexOverlay.delete(cacheDir) + if (afterDetachForTests) await afterDetachForTests() + const restoreDetached = (): void => { + if (!detached) return + let current = codexOverlay.get(cacheDir) + if (!current) { + codexOverlay.set(cacheDir, detached) + return + } + for (const [p, e] of detached) if (!current.has(p)) current.set(p, e) + } + try { + const dirty = new Map() + if (detached) for (const [p, e] of detached) dirty.set(p, e) + // Nothing changed: keep bytes (and mtime) stable instead of rewriting + // identically. Unlike the historical unconditional rewrite this also + // keeps the project index valid across clean flushes. + if (dirty.size === 0) return if (!existsSync(cacheDir)) await mkdir(cacheDir, { recursive: true }) const finalPath = getCachePath(cacheDir) - const tempPath = `${finalPath}.${randomBytes(8).toString('hex')}.tmp` - const payload = JSON.stringify(memCache) - const handle = await open(tempPath, 'w', 0o600) - try { - await handle.writeFile(payload, { encoding: 'utf-8' }) - await handle.sync() - } finally { - await handle.close() + // Merge base: versioned, else legacy (adoption mirrors the load path). + const base = await resolveResultsPath(cacheDir) + const baseVersion = base ? await readResultsVersion(base) : null + let evicted = new Set() + if (baseVersion !== CODEX_CACHE_VERSION) { + // No usable authority (fresh machine or foreign version): publish the + // overlay alone, evicting dead dirties first. This adopts forward + // exactly like the historical whole-write did. + for (const p of [...dirty.keys()]) { + try { + await stat(p) + } catch { + evicted.add(p) + dirty.delete(p) + } + } + if (dirty.size === 0) return + const tempPath = `${finalPath}.${randomBytes(8).toString('hex')}.tmp` + const handle = await open(tempPath, 'w', 0o600) + try { + await writeChunk(handle, `{"version":${CODEX_CACHE_VERSION},"files":{`) + let first = true + for (const [p, e] of dirty) { + await writeChunk(handle, `${first ? '' : ','}${JSON.stringify(p)}:${JSON.stringify(e)}`) + first = false + } + await writeChunk(handle, '}}') + await handle.sync() + await handle.close() + } catch (err) { + try { await handle.close() } catch {} + await unlink(tempPath).catch(() => null) + throw err + } + try { + await rename(tempPath, finalPath) + } catch (err) { + try { await unlink(tempPath) } catch {} + throw err + } + } else if (base) { + evicted = await mergeCodexResults(base, finalPath, dirty) } - try { - await rename(tempPath, finalPath) - } catch (err) { - try { await unlink(tempPath) } catch {} - throw err + // Published bytes changed: bump the store generation FIRST so concurrent + // loads discard (never memoize) pre-flush decodes, then drop resident + // snapshots for this directory (they carry no file-mtime validation, so + // a kept snapshot would serve superseded entries indefinitely). Later + // lookups re-stream once per snapshot identity. (A failed publish + // restores the detached entries below and keeps snapshots.) + flushGenerations.set(cacheDir, (flushGenerations.get(cacheDir) ?? 0) + 1) + for (const key of memCaches.keys()) { + if (key.startsWith(`${cacheDir}\0`)) memCaches.delete(key) } - } catch {} + // Refresh the project index bookkeeping (rows already current via writes). + const st = await stat(finalPath).catch(() => null) + const idx = projectIndexes.get(cacheDir) + if (idx && st) { + idx.builtForMtimeMs = st.mtimeMs + for (const p of evicted) idx.rows.delete(p) + } + // Published: the detached entries now live on disk (or were evicted as + // dead) and stay released — disk is authoritative, and later lookups + // re-stream once per snapshot identity. Only a failed publish restores + // the detached entries for retry. + } catch { + restoreDetached() + } } diff --git a/tests/codex-cache-concurrent-load.test.ts b/tests/codex-cache-concurrent-load.test.ts index a4f7fe098..f5aa1c836 100644 --- a/tests/codex-cache-concurrent-load.test.ts +++ b/tests/codex-cache-concurrent-load.test.ts @@ -1,7 +1,6 @@ -// The codex result cache is a single (often hundreds-of-MB) JSON file, memoized -// in memory only once the read + parse resolves. Discovery now asks for it from -// many concurrent callers, so without a shared in-flight promise every one of -// them re-read and re-parsed the whole file. +// The codex result cache is a single (often hundreds-of-MB) JSON file. Discovery +// now asks for it from many concurrent callers, so without a shared in-flight +// index build every one of them would stream-decode the whole file. import { mkdtemp, rm, writeFile, mkdir } from 'fs/promises' import { tmpdir } from 'os' @@ -9,28 +8,57 @@ import { join } from 'path' import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' -const readSpy = vi.hoisted(() => vi.fn()) - -vi.mock('../src/cache-dir.js', async (importOriginal) => { - const actual = await importOriginal() +import { PassThrough } from 'node:stream' +const streamSpy = vi.hoisted(() => vi.fn()) +const streamGate = vi.hoisted(() => ({ hold: false, opened: 0, constructed: 0, releases: [] as Array<() => void> })) +vi.mock('node:fs', async (importOriginal) => { + const actual = await importOriginal() return { ...actual, - readExistingTextFile: (path: string) => { - readSpy(path) - return actual.readExistingTextFile(path) - }, + createReadStream: ((path: unknown, ...rest: unknown[]) => { + streamSpy(path) + // Deterministic publish-mid-load staging: while the gate holds, open + // the real stream immediately (its fd pins the pre-flush inode, so the + // first decode is deterministically stale) but hand the decoder an + // unpiped PassThrough — no byte flows until the release pipes it. + // pause()/resume() cannot stage this: pipeline() resumes a paused + // stream, so the waiter would finish and memoize v0 before the flush. + // Streams opened after the hold (the flush's own merge read) pass + if (streamGate.hold && String(path).includes('codex-results')) { + const gate = new PassThrough() + streamGate.constructed++ + const real = (actual.createReadStream as (...args: unknown[]) => NodeJS.ReadableStream)(path, ...rest) + // Signal on 'open', not on construction: only an opened fd pins the + // pre-flush inode across the flush's rename, making the first decode + // deterministically stale. (Signalling synchronously here would let + // the open land post-rename and the test pass vacuously on fresh + // bytes with no retry exercised.) + real.once('open', () => { + streamGate.opened++ + streamGate.releases.push(() => { + real.pipe(gate) + }) + }) + return gate + } + return (actual.createReadStream as (...args: unknown[]) => unknown)(path, ...rest) + }) as typeof actual.createReadStream, } }) -const { CODEX_CACHE_VERSION, clearCodexMemCaches, codexCacheFileName, getCachedCodexProject, withCodexCacheDirectory } = +const { CODEX_CACHE_VERSION, clearCodexMemCaches, codexCacheFileName, flushCodexCache, getCachedCodexProject, readCachedCodexResults, withCodexCacheDirectory, writeCachedCodexResults } = await import('../src/codex-cache.js') +import type { ParsedProviderCall } from '../src/providers/types.js' let cacheDir: string let sessionDir: string beforeEach(async () => { - readSpy.mockClear() + streamSpy.mockClear() clearCodexMemCaches() + streamGate.hold = false + streamGate.opened = 0 + streamGate.releases.length = 0 const root = await mkdtemp(join(tmpdir(), 'codeburn-codex-cache-')) cacheDir = join(root, 'cache') sessionDir = join(root, 'sessions') @@ -61,6 +89,65 @@ describe('codex result cache under concurrent readers', () => { Promise.all(paths.map(p => getCachedCodexProject(p)))) expect(projects).toEqual(paths.map((_, i) => `proj-${i}`)) - expect(readSpy.mock.calls.filter(([p]) => String(p).includes('codex-results'))).toHaveLength(1) + expect(streamSpy.mock.calls.filter(([p]) => String(p).includes('codex-results'))).toHaveLength(1) + }) + + it('a publish mid-load reaches the waiter and later readers', async () => { + const p = join(sessionDir, 'rollout-gated.jsonl') + await writeFile(p, '{}\n') + const { statSync } = await import('fs') + const s = statSync(p) + const fp = { dev: s.dev, ino: s.ino, mtimeMs: s.mtimeMs, sizeBytes: s.size } + const files = { [p]: { ...fp, project: 'proj-old', calls: [] } } + await writeFile(join(cacheDir, codexCacheFileName()), JSON.stringify({ version: CODEX_CACHE_VERSION, files })) + const marker = { marker: 'post-flush' } as unknown as ParsedProviderCall + + await withCodexCacheDirectory(cacheDir, async () => { + try { + // Hold the waiter's decode stream: the release pipes a real stream + // whose 'open' already fired, so its fd pins the pre-flush inode + // across the flush's rename and the first decode is deterministically + // stale — while a write+flush publishes underneath it. + streamGate.hold = true + const waiting = readCachedCodexResults(p) + await vi.waitFor( + () => { + if (streamGate.opened === 0) throw new Error(`decode stream not opened yet (constructed=${streamGate.constructed})`) + }, + { timeout: 10_000 }, + ) + // Release the hold BEFORE the flush: the flush's own merge read must + // pass through (gating it would deadlock). + streamGate.hold = false + await writeCachedCodexResults(p, 'proj-new', [marker], fp) + await flushCodexCache() + // Re-arm before releasing: the waiter's retry decode must be gated + // too, so the open count proves the retry instead of assuming it. + streamGate.hold = true + for (const resume of streamGate.releases.splice(0)) resume() + await vi.waitFor( + () => { + if (streamGate.opened < 2) throw new Error('retry decode not opened yet') + }, + { timeout: 10_000 }, + ) + streamGate.hold = false + for (const resume of streamGate.releases.splice(0)) resume() + // Exactly two decode opens (stale first + retry): without the + // generation loop the waiter would have resolved off one open. + expect(streamGate.opened).toBe(2) + // The waiter retried on the new bytes instead of memoizing stale ones… + await expect(waiting).resolves.toEqual({ kind: 'exact', calls: [marker] }) + // …and the memo holds the fresh snapshot for later readers. + await expect(readCachedCodexResults(p)).resolves.toEqual({ kind: 'exact', calls: [marker] }) + } finally { + // Drain any held source on failure: a stuck test must not leave an + // open fd or a pending pipeline for later tests in this worker. + streamGate.hold = false + for (const resume of streamGate.releases.splice(0)) { + try { resume() } catch { /* release-only; decode already settled */ } + } + } + }) }) }) diff --git a/tests/codex-range-filter.test.ts b/tests/codex-range-filter.test.ts new file mode 100644 index 000000000..b39cd87c3 --- /dev/null +++ b/tests/codex-range-filter.test.ts @@ -0,0 +1,449 @@ +// Bounded codex result-cache loads (OOM fix, second half): the 227MB +// codex-results file must stream-decode with range filtering like session +// shards do, instead of whole-file JSON.parse on every lookup. Entries whose +// file predates the range AND whose calls all predate it are dropped (their +// files are mtime-floor-skipped before any lookup); everything else is kept +// WHOLE — codex entries are never call-projected, so an exact hit yields all +// calls with keys exactly as today (pinned below). +// +// Flush merges dirty overlay entries over the published bytes (streaming, +// bounded) instead of rewriting from memory; project labels come from a small +// streaming index so discovery never loads the calls map. +import { mkdir, mkdtemp, readFile, rm, utimes, writeFile } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { + __setAfterDetachForTests, + clearCodexMemCaches, + codexCacheFileName, + fingerprintFile, + flushCodexCache, + readCachedCodexResults, + getCachedCodexProject, + retainCodexEntry, + loadCacheFilteredFromDisk, + scanRetainedCodexKeys, + streamCodexEntryMetadata, + writeCachedCodexResults, + CODEX_CACHE_VERSION, +} from '../src/codex-cache.js' +import { __setAfterStreamOpenForTests } from '../src/shard-stream.js' + +const originalCacheDir = process.env['CODEBURN_CACHE_DIR'] +let root: string + +function codexCall(key: string, timestamp: string): ParsedProviderCall { + return { + provider: 'codex', + model: 'm', + inputTokens: 1, + outputTokens: 1, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costUSD: 0, + tools: [], + bashCommands: [], + timestamp, + speed: 'standard', + deduplicationKey: `codex:${key}`, + userMessage: '', + sessionId: key, + } +} + +const DAY_START = new Date('2026-07-15T00:00:00Z').getTime() + +async function backdate(path: string, ms: number): Promise { + const at = new Date(ms) + await utimes(path, at, at) +} + +async function seedSource(name: string, content: string, mtimeMs?: number): Promise { + const path = join(root, name) + await writeFile(path, content) + if (mtimeMs !== undefined) await backdate(path, mtimeMs) + return path +} + +async function seedResults(entries: Record): Promise { + const cachePath = join(root, codexCacheFileName()) + await writeFile(cachePath, JSON.stringify({ version: CODEX_CACHE_VERSION, files: entries })) + return cachePath +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'codeburn-codex-range-')) + process.env['CODEBURN_CACHE_DIR'] = root + clearCodexMemCaches() +}) + +afterEach(async () => { + if (originalCacheDir === undefined) delete process.env['CODEBURN_CACHE_DIR'] + else process.env['CODEBURN_CACHE_DIR'] = originalCacheDir + clearCodexMemCaches() + await rm(root, { recursive: true, force: true }) +}) + +describe('bounded codex decode', () => { + it('drops out-of-range entries under a range but keeps them on full loads', async () => { + const oldPath = await seedSource('old.jsonl', '{}\n', new Date('2026-06-10T10:00:00Z').getTime()) + const newPath = await seedSource('new.jsonl', '{}\n') + const oldFp = (await fingerprintFile(oldPath))! + const newFp = (await fingerprintFile(newPath))! + await seedResults({ + [oldPath]: { + dev: oldFp.dev, ino: oldFp.ino, mtimeMs: oldFp.mtimeMs, sizeBytes: oldFp.sizeBytes, + project: 'old-proj', calls: [codexCall('old-1', '2026-06-10T10:00:00.000Z')], + }, + [newPath]: { + dev: newFp.dev, ino: newFp.ino, mtimeMs: newFp.mtimeMs, sizeBytes: newFp.sizeBytes, + project: 'new-proj', calls: [codexCall('new-1', '2026-07-15T10:00:00.000Z')], + }, + }) + + // Full load serves both (status quo ante). + expect((await readCachedCodexResults(oldPath))?.kind).toBe('exact') + clearCodexMemCaches() + + // Ranged load drops the old entry: genuine miss, not an error. + expect(await readCachedCodexResults(oldPath, { rangeStartMs: DAY_START })).toBeNull() + // ...while the in-range entry still hits exactly. + const hit = await readCachedCodexResults(newPath, { rangeStartMs: DAY_START }) + expect(hit?.kind).toBe('exact') + }) + + it('keeps resume-capable entries and serves resume tails under a range', async () => { + const sourcePath = await seedSource('roll.jsonl', 'line1\n') + const fp = (await fingerprintFile(sourcePath))! + await seedResults({ + [sourcePath]: { + dev: fp.dev, ino: fp.ino, mtimeMs: fp.mtimeMs, sizeBytes: fp.sizeBytes, + project: 'p', calls: [codexCall('c1', '2026-07-15T10:00:00.000Z')], + resumeOffset: 6, resumeState: { sessionId: 's' }, resumeCallCount: 1, + }, + }) + // Grow the file: exact misses, resume must still hit under a range. + await writeFile(sourcePath, 'line1\nline2\n') + const hit = await readCachedCodexResults(sourcePath, { rangeStartMs: DAY_START }) + expect(hit?.kind).toBe('resume') + if (hit && hit.kind === 'resume') { + expect(hit.offset).toBe(6) + expect(hit.callCount).toBe(1) + } + }) + + it('exact hits yield every call with keys, including out-of-range ones (no call projection)', async () => { + // A kept entry (mtime in range) serves its WHOLE call list: filtering is + // whole-entry only. If entries were ever call-projected, the June call's + // key would vanish and cross-file suppression would silently change. + const sourcePath = await seedSource('mixed.jsonl', '{}\n') + const fp = (await fingerprintFile(sourcePath))! + await seedResults({ + [sourcePath]: { + dev: fp.dev, ino: fp.ino, mtimeMs: fp.mtimeMs, sizeBytes: fp.sizeBytes, + project: 'p', + calls: [codexCall('june-1', '2026-06-10T10:00:00.000Z'), codexCall('july-1', '2026-07-15T10:00:00.000Z')], + }, + }) + const hit = await readCachedCodexResults(sourcePath, { rangeStartMs: DAY_START }) + expect(hit?.kind).toBe('exact') + const keys = hit && 'calls' in hit ? hit.calls.map(c => c.deduplicationKey) : [] + expect(keys).toEqual(['codex:june-1', 'codex:july-1']) + }) + + it('isolates sequential and concurrent loads by range identity', async () => { + const oldPath = await seedSource('o.jsonl', '{}\n', new Date('2026-06-10T10:00:00Z').getTime()) + const newPath = await seedSource('n.jsonl', '{}\n') + const oldFp = (await fingerprintFile(oldPath))! + const newFp = (await fingerprintFile(newPath))! + await seedResults({ + [oldPath]: { + dev: oldFp.dev, ino: oldFp.ino, mtimeMs: oldFp.mtimeMs, sizeBytes: oldFp.sizeBytes, + project: 'o', calls: [codexCall('o1', '2026-06-10T10:00:00.000Z')], + }, + [newPath]: { + dev: newFp.dev, ino: newFp.ino, mtimeMs: newFp.mtimeMs, sizeBytes: newFp.sizeBytes, + project: 'n', calls: [codexCall('n1', '2026-07-15T10:00:00.000Z')], + }, + }) + const juneStart = new Date('2026-06-01T00:00:00Z').getTime() + // Sequential: each range sees its own slice after the other ran. + expect(await readCachedCodexResults(oldPath, { rangeStartMs: DAY_START })).toBeNull() + expect((await readCachedCodexResults(oldPath, { rangeStartMs: juneStart }))?.kind).toBe('exact') + expect((await readCachedCodexResults(newPath, { rangeStartMs: DAY_START }))?.kind).toBe('exact') + // Concurrent: neither projection leaks into the other. + const [a, b] = await Promise.all([ + readCachedCodexResults(oldPath, { rangeStartMs: DAY_START }), + readCachedCodexResults(newPath, { rangeStartMs: DAY_START }), + ]) + expect(a).toBeNull() + expect(b?.kind).toBe('exact') + }) +}) + +describe('codex flush merge', () => { + it('substitutes dirty entries, carries dropped ones, prunes evicted', async () => { + const keepPath = await seedSource('keep.jsonl', '{}\n', new Date('2026-06-10T10:00:00Z').getTime()) + const dropPath = await seedSource('drop.jsonl', '{}\n') + await rm(dropPath) + const keepFp = (await fingerprintFile(keepPath))! + await seedResults({ + [keepPath]: { + dev: keepFp.dev, ino: keepFp.ino, mtimeMs: keepFp.mtimeMs, sizeBytes: keepFp.sizeBytes, + project: 'k', calls: [codexCall('k1', '2026-06-10T10:00:00.000Z')], + }, + [dropPath]: { + dev: 1, ino: 2, mtimeMs: 3, sizeBytes: 4, + project: 'd', calls: [codexCall('d1', '2026-06-10T10:00:00.000Z')], + }, + }) + // Ranged load drops the June entry from memory; write a fresh one. + expect(await readCachedCodexResults(keepPath, { rangeStartMs: DAY_START })).toBeNull() + const freshPath = await seedSource('fresh.jsonl', '{}\n') + const freshFp = (await fingerprintFile(freshPath))! + await writeCachedCodexResults(freshPath, 'f', [codexCall('f1', '2026-07-15T10:00:00.000Z')], { + dev: freshFp.dev, ino: freshFp.ino, mtimeMs: freshFp.mtimeMs, sizeBytes: freshFp.sizeBytes, + }) + await flushCodexCache() + + // Published bytes: dropped June entry carried verbatim, evicted pruned, + // fresh entry present — all without ever materializing the file. + const published = JSON.parse(await readFile(join(root, codexCacheFileName()), 'utf-8')) + expect(Object.keys(published.files).sort()).toEqual([freshPath, keepPath].sort()) + expect(published.files[keepPath].calls.map((c: { deduplicationKey: string }) => c.deduplicationKey)).toEqual(['codex:k1']) + expect(published.files[freshPath].project).toBe('f') + }) + + it('write then flush then read observes the publish without a manual clear', async () => { + const sourcePath = await seedSource('w.jsonl', '{}\n') + const fp = (await fingerprintFile(sourcePath))! + await writeCachedCodexResults(sourcePath, 'w', [codexCall('w1', '2026-07-15T10:00:00.000Z')], { + dev: fp.dev, ino: fp.ino, mtimeMs: fp.mtimeMs, sizeBytes: fp.sizeBytes, + }) + await flushCodexCache() + // No clearCodexMemCaches between flush and read: the read must observe + // the published entry (via overlay/folded snapshots), not a stale miss. + const hit = await readCachedCodexResults(sourcePath, { rangeStartMs: DAY_START }) + expect(hit?.kind).toBe('exact') + }) + + it('adopts a foreign-version file forward on flush', async () => { + // v14 on disk (old binary era): a write + flush must establish the v15 + // file with the fresh entry, never deadlock waiting for v15 bytes. + const sourcePath = await seedSource('v14.jsonl', '{}\n') + const fp = (await fingerprintFile(sourcePath))! + await writeFile(join(root, codexCacheFileName()), JSON.stringify({ + version: CODEX_CACHE_VERSION - 1, + files: { + [sourcePath]: { + dev: fp.dev, ino: fp.ino, mtimeMs: fp.mtimeMs, sizeBytes: fp.sizeBytes, + project: 'legacy', calls: [codexCall('legacy-1', '2026-06-10T10:00:00.000Z')], + }, + }, + })) + await writeCachedCodexResults(sourcePath, 'fresh', [codexCall('fresh-1', '2026-07-15T10:00:00.000Z')], { + dev: fp.dev, ino: fp.ino, mtimeMs: fp.mtimeMs, sizeBytes: fp.sizeBytes, + }) + await flushCodexCache() + clearCodexMemCaches() + const hit = await readCachedCodexResults(sourcePath) + expect(hit?.kind).toBe('exact') + const published = JSON.parse(await readFile(join(root, codexCacheFileName()), 'utf-8')) + expect(published.version).toBe(CODEX_CACHE_VERSION) + }) +}) + +describe('codex project index', () => { + it('serves discovery labels without loading the calls map', async () => { + const sourcePath = await seedSource('idx.jsonl', '{}\n', new Date('2026-06-10T10:00:00Z').getTime()) + const fp = (await fingerprintFile(sourcePath))! + await seedResults({ + [sourcePath]: { + dev: fp.dev, ino: fp.ino, mtimeMs: fp.mtimeMs, sizeBytes: fp.sizeBytes, + project: 'indexed-proj', calls: [codexCall('i1', '2026-06-10T10:00:00.000Z')], + }, + }) + // Label served (fp-matched) even though a ranged load would drop the entry. + expect(await getCachedCodexProject(sourcePath)).toBe('indexed-proj') + // Stale fingerprint: no label (mirrors the full-map freshness contract). + await backdate(sourcePath, Date.now()) + expect(await getCachedCodexProject(sourcePath)).toBeNull() + }) + + it('streams metadata without calls even when the path contains dots', async () => { + // The token-level skip matches the exact stack [path, 'calls'] below the + // pick re-root: a filename containing ".calls" must not confuse it, and + // the calls array must never materialize (that is the discovery OOM). + const sourcePath = await seedSource('weird.calls.v1.jsonl', '{}\n') + const fp = (await fingerprintFile(sourcePath))! + const calls = Array.from({ length: 2000 }, (_, i) => codexCall(`bulk-${i}`, '2026-07-15T10:00:00.000Z')) + await seedResults({ + [sourcePath]: { + dev: fp.dev, ino: fp.ino, mtimeMs: fp.mtimeMs, sizeBytes: fp.sizeBytes, + project: 'dotted-proj', calls, + }, + }) + const seen = new Map() + await streamCodexEntryMetadata(join(root, codexCacheFileName()), (key, value) => { + seen.set(key, value) + }) + expect(seen.has(sourcePath)).toBe(true) + const meta = seen.get(sourcePath) as Record + expect(meta).not.toHaveProperty('calls') + expect(meta['project']).toBe('dotted-proj') + expect(meta['mtimeMs']).toBe(fp.mtimeMs) + // Labels still served from the metadata-only build. + expect(await getCachedCodexProject(sourcePath)).toBe('dotted-proj') + }) +}) + +describe('codex two-pass range load', () => { + const JUNE = '2026-06-10T10:00:00.000Z' + const JULY = '2026-07-15T10:00:00.000Z' + const JUNE_MS = new Date(JUNE).getTime() + + function nastyEntries(): Record { + const monster = Array.from({ length: 20000 }, (_, i) => codexCall(`bulk-${i}`, JUNE)) + return { + // Kept: file mtime in range (calls never inspected). + 'keep-mtime': { mtimeMs: DAY_START + 3600_000, project: 'p', calls: [codexCall('old-1', JUNE)] }, + // Kept: one in-range call keeps the whole entry (with keys intact). + 'keep-call': { mtimeMs: JUNE_MS, project: 'p', calls: [codexCall('c-old', JUNE), codexCall('c-new', JULY)] }, + // Dropped: old file, every call valid and pre-range. + 'drop-clean': { mtimeMs: JUNE_MS, project: 'p', calls: [codexCall('d1', JUNE), codexCall('d2', JUNE)] }, + // Dropped: the boundedness proof — 20k pre-range calls never assemble. + 'drop-monster': { mtimeMs: JUNE_MS, project: 'p', calls: monster }, + // Dropped: dotted key exercises exact-stack matching, not substrings. + 'weird.calls.jsonl': { mtimeMs: JUNE_MS, project: 'p', calls: [codexCall('w1', JUNE)] }, + // Dropped: missing mtimeMs, pre-range calls. + 'drop-no-mtime': { project: 'p', calls: [codexCall('n1', JUNE)] }, + // Dropped: string mtimeMs is not a number, pre-range calls. + 'drop-str-mtime': { mtimeMs: JUNE, project: 'p', calls: [codexCall('s1', JUNE)] }, + // Kept: every unjudgeable shape the full load would carry. + 'keep-empty-calls': { mtimeMs: JUNE_MS, project: 'p', calls: [] }, + 'keep-no-calls': { mtimeMs: JUNE_MS, project: 'p' }, + 'keep-calls-object': { mtimeMs: JUNE_MS, project: 'p', calls: {} }, + 'keep-junk-calls': { + mtimeMs: JUNE_MS, + project: 'p', + calls: [null, 5, { nope: 1 }, { timestamp: 123 }, { timestamp: 'bogus' }, codexCall('j1', JUNE)], + }, + 'keep-bare': {}, + } + } + + it('scans a real v15 envelope without fallback and matches the retain rule exactly', async () => { + const entries = nastyEntries() + await seedResults(entries) + // Throws if the walker rejects the envelope (e.g. the version scalar): + // that failure would silently route every load through full assembly. + const scanned = await scanRetainedCodexKeys(join(root, codexCacheFileName()), DAY_START) + const oracle = new Set(Object.entries(entries).filter(([, v]) => retainCodexEntry(v, DAY_START)).map(([k]) => k)) + expect(scanned).toEqual(oracle) + expect(scanned.has('drop-monster')).toBe(false) + expect(scanned.has('weird.calls.jsonl')).toBe(false) + expect(scanned.has('keep-call')).toBe(true) + }) + it('loads exactly the scanned keys, kept entries whole', async () => { + const entries = nastyEntries() + await seedResults(entries) + const loaded = await loadCacheFilteredFromDisk(root, DAY_START) + const oracle = new Set(Object.entries(entries).filter(([, v]) => retainCodexEntry(v, DAY_START)).map(([k]) => k)) + expect(new Set(Object.keys(loaded.files))).toEqual(oracle) + // Kept entries arrive whole: every call with keys, no projection. + const keepCall = loaded.files['keep-call'] as { calls: { deduplicationKey: string }[] } + expect(keepCall.calls.map(c => c.deduplicationKey)).toEqual(['codex:c-old', 'codex:c-new']) + expect(loaded.files['keep-junk-calls']).toEqual(entries['keep-junk-calls']) + expect(loaded.files['keep-bare']).toEqual({}) + }) + + it('serializes overlapping flushes without losing a batch', async () => { + const aPath = await seedSource('a.jsonl', '{}\n') + const bPath = await seedSource('b.jsonl', '{}\n') + const aFp = (await fingerprintFile(aPath))! + const bFp = (await fingerprintFile(bPath))! + await writeCachedCodexResults(aPath, 'a', [codexCall('a1', '2026-07-15T10:00:00.000Z')], { + dev: aFp.dev, ino: aFp.ino, mtimeMs: aFp.mtimeMs, sizeBytes: aFp.sizeBytes, + }) + // Overlapping flushes: the second must wait for the first and publish + // over its output, not race it on the same base bytes. The detach + // barrier (not a timer or pump count) proves the first flush detached + // before the second write lands. + let releaseFirst!: () => void + const firstDetached = new Promise(resolve => { + __setAfterDetachForTests(async () => { + resolve() + await new Promise(r => { releaseFirst = r }) + }) + }) + try { + const first = flushCodexCache() + await firstDetached + // Disarm: only the first flush may park here; the second must run free. + __setAfterDetachForTests(null) + await writeCachedCodexResults(bPath, 'b', [codexCall('b1', '2026-07-15T10:00:00.000Z')], { + dev: bFp.dev, ino: bFp.ino, mtimeMs: bFp.mtimeMs, sizeBytes: bFp.sizeBytes, + }) + const second = flushCodexCache() + releaseFirst() + await Promise.all([first, second]) + } finally { + __setAfterDetachForTests(null) + } + clearCodexMemCaches() + const published = JSON.parse(await readFile(join(root, codexCacheFileName()), 'utf-8')) + expect(Object.keys(published.files).sort()).toEqual([aPath, bPath].sort()) + }) + + it('a load spanning a flush returns post-flush data', async () => { + // keepme lives ONLY on disk (never overlaid) so the lookup below must + // decode it; big ballast (40k pre-range calls) keeps that decode in + // flight while the publish lands. Unlinking forces a fast adopt-forward + // publish (milliseconds) mid-decode (hundreds of ms). The stream-open + // rendezvous (not pump counts) proves the load holds pre-unlink bytes + // before anything is unlinked. + const juneMs = new Date('2026-06-10T10:00:00Z').getTime() + const bigPath = await seedSource('big.jsonl', '{}\n', juneMs) + const bigFp = (await fingerprintFile(bigPath))! + const keepPath = await seedSource('keepme.jsonl', '{}\n') + const keepFp = (await fingerprintFile(keepPath))! + const bulk = Array.from({ length: 40000 }, (_, i) => codexCall(`bulk-${i}`, '2026-06-10T10:00:00.000Z')) + const cachePath = join(root, codexCacheFileName()) + await seedResults({ + [bigPath]: { + dev: bigFp.dev, ino: bigFp.ino, mtimeMs: bigFp.mtimeMs, sizeBytes: bigFp.sizeBytes, + project: 'big', calls: bulk, + }, + [keepPath]: { + dev: keepFp.dev, ino: keepFp.ino, mtimeMs: keepFp.mtimeMs, sizeBytes: keepFp.sizeBytes, + project: 'keep', calls: [codexCall('v1', '2026-07-15T10:00:00.000Z')], + }, + }) + let opened!: () => void + const streamOpened = new Promise(resolve => { opened = resolve }) + __setAfterStreamOpenForTests(() => opened()) + const load = readCachedCodexResults(keepPath, { rangeStartMs: DAY_START }) + try { + await streamOpened + await writeCachedCodexResults(keepPath, 'keep', [codexCall('v2', '2026-07-15T10:00:00.000Z')], { + dev: keepFp.dev, ino: keepFp.ino, mtimeMs: keepFp.mtimeMs, sizeBytes: keepFp.sizeBytes, + }) + await rm(cachePath) + await flushCodexCache() + } finally { + __setAfterStreamOpenForTests(null) + } + const hit = await load + // Post-flush data: the generation change forced a redecode on the new + // bytes (never a stale memo: the flush invalidated snapshots too). + expect(hit?.kind).toBe('exact') + if (hit && hit.kind === 'exact') { + expect(hit.calls.map(c => c.deduplicationKey)).toEqual(['codex:v2']) + } + }) +}) From 056d395121e90f52bc0b83316d3af30e524d9494 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sun, 20 Sep 2026 10:00:45 -0400 Subject: [PATCH 05/13] fix: decode sub-gate codex result files with JSON.parse instead of streaming Same size gate as session shards (SHARD_STREAM_GATE_BYTES now lives in shard-stream.ts, shared by both): the timestamp scan reuses the exported retainCodexEntry rule per entry so the paths cannot drift, and the entry load applies the same retain rule in memory. The streaming mid-flush rendezvous test pins the stream path explicitly. Parity between paths is pinned by the new codex size-gate test. --- src/codex-cache.ts | 64 ++++++++++++++++++++++- src/session-cache.ts | 22 +------- src/shard-stream.ts | 23 +++++++- tests/codex-cache-concurrent-load.test.ts | 6 +++ tests/codex-range-filter.test.ts | 37 ++++++++++++- tests/session-cache-size-gate.test.ts | 3 +- 6 files changed, 129 insertions(+), 26 deletions(-) diff --git a/src/codex-cache.ts b/src/codex-cache.ts index 76253dcb5..dbd3f6834 100644 --- a/src/codex-cache.ts +++ b/src/codex-cache.ts @@ -11,7 +11,7 @@ import { streamObject } from 'stream-json/streamers/stream-object.js' import { getCodeburnCacheDir } from './cache-dir.js' import type { ParsedProviderCall } from './providers/types.js' -import { streamShardArrayField, writeChunk } from './shard-stream.js' +import { shardNeedsStreaming, streamShardArrayField, writeChunk } from './shard-stream.js' import { flatString, flattenJsonStrings } from './content-utils.js' // v4: attribute MCP calls emitted as event_msg/mcp_tool_call_end (issue #478). @@ -322,16 +322,51 @@ export function retainCodexEntry(value: unknown, rangeStartMs: number): boolean return !seenCall } +/// Small-file twin of loadResultsStreaming for result files at or under +/// SHARD_STREAM_GATE_BYTES: one readFile plus the same retain rule per +/// entry. JSON.parse returns fresh strings, so no detaching is needed, and +/// entries arrive whole — the pick filter's subtree drops (retainKeys, +/// skipCalls) are just key skips here. A missing `files` object serves empty +/// (the pick filter yields zero entries); anything else malformed drops the +/// whole file, mirroring the stream entry for entry. +async function loadResultsSmall( + path: string, + retain: (value: unknown) => boolean, + retainKeys?: Set, +): Promise | null> { + if ((await readResultsVersion(path)) !== CODEX_CACHE_VERSION) return null + let raw: unknown + try { + raw = JSON.parse(await readFile(path, 'utf-8')) + } catch { + return null + } + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null + const filesObj = 'files' in raw ? raw.files : undefined + if (filesObj === undefined) return {} + if (!filesObj || typeof filesObj !== 'object' || Array.isArray(filesObj)) return null + const files: Record = {} + for (const [key, value] of Object.entries(filesObj)) { + if (retainKeys && !retainKeys.has(key)) continue + if (!retain(value)) continue + files[key] = value + } + return files +} + /// Streaming load of one result file: version-gated, then per-entry through /// the pick filter (the nested `files` object streams entry by entry — a /// top-level decode would assemble all 227MB first). Any failure drops the /// whole file, mirroring the historical whole-file behavior entry for entry. +/// Result files at or under SHARD_STREAM_GATE_BYTES take loadResultsSmall +/// instead (see above). async function loadResultsStreaming( path: string, retain: (value: unknown) => boolean, retainKeys?: Set, ): Promise | null> { if ((await readResultsVersion(path)) !== CODEX_CACHE_VERSION) return null + if (!(await shardNeedsStreaming(path))) return loadResultsSmall(path, retain, retainKeys) const files: Record = {} try { await streamCodexEntries(path, (key, value) => { @@ -358,13 +393,40 @@ async function loadCacheFullFromDisk(cacheDir: string): Promise { // stored unvalidated and every consumer reads them defensively. return { version: CODEX_CACHE_VERSION, files: files as Record } } +/// Small-file twin of the timestamp scan for result files at or under +/// SHARD_STREAM_GATE_BYTES: one readFile, then the exported retainCodexEntry +/// rule per entry — the same rule the streaming scan mirrors call-for-call, +/// so the two cannot drift. Decode failure throws like the stream, and the +/// caller falls back the same way. +async function scanRetainedCodexKeysSmall(path: string, rangeStartMs: number): Promise> { + let raw: unknown + try { + raw = JSON.parse(await readFile(path, 'utf-8')) + } catch { + throw new Error(`codex results unreadable: ${path}`) + } + const retained = new Set() + const filesObj = raw !== null && typeof raw === 'object' && !Array.isArray(raw) && 'files' in raw + ? raw.files + : undefined + if (filesObj !== undefined && filesObj !== null && typeof filesObj === 'object' && !Array.isArray(filesObj)) { + for (const [key, value] of Object.entries(filesObj)) { + if (retainCodexEntry(value, rangeStartMs)) retained.add(key) + } + } + return retained +} + /// First pass of a range load: stream one call at a time (never an entry) /// and decide each file's fate by the exact `retainCodexEntry` rule — /// file-mtime first, then call timestamps. Returns the keys to assemble whole /// in pass two. Any decode failure throws, and the caller falls back to the /// single-pass load (same bytes-or-null contract as a torn file today). +/// Result files at or under SHARD_STREAM_GATE_BYTES take +/// scanRetainedCodexKeysSmall instead (see above). export async function scanRetainedCodexKeys(path: string, rangeStartMs: number): Promise> { + if (!(await shardNeedsStreaming(path))) return scanRetainedCodexKeysSmall(path, rangeStartMs) const retained = new Set() type Scan = { mtimeMs: number | null; callsNonArray: boolean; keep: boolean; count: number } const scans = new Map() diff --git a/src/session-cache.ts b/src/session-cache.ts index 6a3aa6dc8..8a310e2e9 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -7,7 +7,7 @@ import { getCodeburnCacheDir } from './cache-dir.js' import { flatString, flattenJsonStrings } from './content-utils.js' import { acquireCacheRefreshLock, releaseOwnedRefreshLocksForExit } from './cache-refresh-lock.js' import type { ToolCall } from './types.js' -import { streamShardArrayField, streamShardEntries } from './shard-stream.js' +import { shardNeedsStreaming, streamShardArrayField, streamShardEntries } from './shard-stream.js' // ── Types ────────────────────────────────────────────────────────────── @@ -1126,26 +1126,6 @@ async function readEnvelope(dir: string): Promise { } } -/// Shards at or under this size decode with plain JSON.parse; larger ones -/// stream. Profiled on a real corpus: 166MB of shards in 333ms via JSON.parse -/// against 15-20s through the streaming walk, while the walk exists for the -/// one shard past V8's max string length (readFile+JSON.parse hard-fails -/// there regardless of heap). Half that ceiling keeps the fast path safely -/// below it with room for UTF-16 expansion. -const SHARD_STREAM_GATE_BYTES = 256 * 1024 * 1024 -let shardStreamGateForTests: number | null = null -export function __setShardStreamGateForTests(bytes: number | null): void { - shardStreamGateForTests = bytes -} -/// True when path must take the streaming decoder. Unstatable files fall -/// through to the stream, which fails exactly the way the unreadable-shard -/// path always has (null, never a partial commit). -async function shardNeedsStreaming(path: string): Promise { - const gate = shardStreamGateForTests ?? SHARD_STREAM_GATE_BYTES - const size = await stat(path).then(s => s.size, () => null) - if (size === null) return true - return size > gate -} /// Decode one shard with plain JSON.parse: the fast path for shards at or /// under SHARD_STREAM_GATE_BYTES. One readFile is already an atomic snapshot, /// and JSON.parse returns fresh strings, so nothing here can pin a tokenizer diff --git a/src/shard-stream.ts b/src/shard-stream.ts index 175b486f5..1f5b08a94 100644 --- a/src/shard-stream.ts +++ b/src/shard-stream.ts @@ -27,7 +27,7 @@ export async function writeChunk(handle: FileHandle, text: string): Promise void) | null): void { afterStreamOpenForTests = hook } +/// Shards at or under this size decode with plain JSON.parse; larger ones +/// stream. Profiled on a real corpus: 166MB of shards in 333ms via JSON.parse +/// against 15-20s through the streaming walk, while the walk exists for the +/// one shard past V8's max string length (readFile+JSON.parse hard-fails +/// there regardless of heap). Half that ceiling keeps the fast path safely +/// below it with room for UTF-16 expansion. +export const SHARD_STREAM_GATE_BYTES = 256 * 1024 * 1024 +let shardStreamGateForTests: number | null = null +export function __setShardStreamGateForTests(bytes: number | null): void { + shardStreamGateForTests = bytes +} +/// True when path must take the streaming decoder. Unstatable files fall +/// through to the stream, which fails exactly the way the unreadable-shard +/// path always has (null, never a partial commit). +export async function shardNeedsStreaming(path: string): Promise { + const gate = shardStreamGateForTests ?? SHARD_STREAM_GATE_BYTES + const size = await stat(path).then(s => s.size, () => null) + if (size === null) return true + return size > gate +} + export type ShardEntry = { key: string; value: unknown } /** First non-whitespace byte of a file, or null if empty. A BOM is not diff --git a/tests/codex-cache-concurrent-load.test.ts b/tests/codex-cache-concurrent-load.test.ts index f5aa1c836..62b06ece7 100644 --- a/tests/codex-cache-concurrent-load.test.ts +++ b/tests/codex-cache-concurrent-load.test.ts @@ -49,6 +49,10 @@ vi.mock('node:fs', async (importOriginal) => { const { CODEX_CACHE_VERSION, clearCodexMemCaches, codexCacheFileName, flushCodexCache, getCachedCodexProject, readCachedCodexResults, withCodexCacheDirectory, writeCachedCodexResults } = await import('../src/codex-cache.js') import type { ParsedProviderCall } from '../src/providers/types.js' +// Every test here stages a publish mid-STREAM-decode: pin the stream path +// (the size gate would otherwise parse these small fixtures whole, and the +// createReadStream staging above would never engage). +import { __setShardStreamGateForTests } from '../src/shard-stream.js' let cacheDir: string let sessionDir: string @@ -56,6 +60,7 @@ let sessionDir: string beforeEach(async () => { streamSpy.mockClear() clearCodexMemCaches() + __setShardStreamGateForTests(0) streamGate.hold = false streamGate.opened = 0 streamGate.releases.length = 0 @@ -67,6 +72,7 @@ beforeEach(async () => { }) afterEach(async () => { + __setShardStreamGateForTests(null) clearCodexMemCaches() await rm(join(cacheDir, '..'), { recursive: true, force: true }) }) diff --git a/tests/codex-range-filter.test.ts b/tests/codex-range-filter.test.ts index b39cd87c3..15269128a 100644 --- a/tests/codex-range-filter.test.ts +++ b/tests/codex-range-filter.test.ts @@ -29,7 +29,7 @@ import { writeCachedCodexResults, CODEX_CACHE_VERSION, } from '../src/codex-cache.js' -import { __setAfterStreamOpenForTests } from '../src/shard-stream.js' +import { __setAfterStreamOpenForTests, __setShardStreamGateForTests } from '../src/shard-stream.js' const originalCacheDir = process.env['CODEBURN_CACHE_DIR'] let root: string @@ -426,6 +426,10 @@ describe('codex two-pass range load', () => { }) let opened!: () => void const streamOpened = new Promise(resolve => { opened = resolve }) + // This test stages a publish mid-STREAM-decode: pin the stream path + // (the size gate would otherwise parse this small fixture whole, and + // the stream-open rendezvous below would never fire). + __setShardStreamGateForTests(0) __setAfterStreamOpenForTests(() => opened()) const load = readCachedCodexResults(keepPath, { rangeStartMs: DAY_START }) try { @@ -436,6 +440,7 @@ describe('codex two-pass range load', () => { await rm(cachePath) await flushCodexCache() } finally { + __setShardStreamGateForTests(null) __setAfterStreamOpenForTests(null) } const hit = await load @@ -447,3 +452,33 @@ describe('codex two-pass range load', () => { } }) }) + +describe('codex size gate parity', () => { + it('scans and loads identical results from the parse and stream paths', async () => { + const JUNE = '2026-06-10T10:00:00.000Z' + const JULY = '2026-07-15T10:00:00.000Z' + const JUNE_MS = new Date(JUNE).getTime() + const monster = Array.from({ length: 20000 }, (_, i) => codexCall(`bulk-${i}`, JUNE)) + const entries: Record = { + 'keep-mtime': { mtimeMs: DAY_START + 3600_000, project: 'p', calls: [codexCall('old-1', JUNE)] }, + 'keep-call': { mtimeMs: JUNE_MS, project: 'p', calls: [codexCall('c-old', JUNE), codexCall('c-new', JULY)] }, + 'drop-clean': { mtimeMs: JUNE_MS, project: 'p', calls: [codexCall('d1', JUNE), codexCall('d2', JUNE)] }, + 'drop-monster': { mtimeMs: JUNE_MS, project: 'p', calls: monster }, + 'keep-empty-calls': { mtimeMs: JUNE_MS, project: 'p', calls: [] }, + 'keep-bare': {}, + } + await seedResults(entries) + const cacheFile = join(root, codexCacheFileName()) + __setShardStreamGateForTests(Number.POSITIVE_INFINITY) + const scannedParse = await scanRetainedCodexKeys(cacheFile, DAY_START) + const loadedParse = await loadCacheFilteredFromDisk(root, DAY_START) + __setShardStreamGateForTests(0) + const scannedStream = await scanRetainedCodexKeys(cacheFile, DAY_START) + const loadedStream = await loadCacheFilteredFromDisk(root, DAY_START) + __setShardStreamGateForTests(null) + expect(scannedStream).toEqual(scannedParse) + expect(loadedStream).toEqual(loadedParse) + expect(scannedParse.has('drop-monster')).toBe(false) + expect(scannedParse.has('keep-call')).toBe(true) + }) +}) diff --git a/tests/session-cache-size-gate.test.ts b/tests/session-cache-size-gate.test.ts index 77d923987..f79091ed2 100644 --- a/tests/session-cache-size-gate.test.ts +++ b/tests/session-cache-size-gate.test.ts @@ -9,13 +9,12 @@ import { join } from 'path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { - __setShardStreamGateForTests, clearShardMemo, loadShardFiltered, loadShardMemoized, - sessionCacheDir, type CachedFile, } from '../src/session-cache.js' +import { __setShardStreamGateForTests } from '../src/shard-stream.js' type Turn = CachedFile['turns'][number] From 62f6e34a0def4447c94c0d87b6643c8a695b0d50 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sat, 19 Sep 2026 22:03:11 -0400 Subject: [PATCH 06/13] fix: hash cross-file dedup keys with SHA-256 instead of raw strings Dropped-key markers and shared dedup sets store 32-character SHA-256 digests (~140MB of raw key characters on the reporter corpus); every insert and lookup hashes uniformly. The set is a plain class so unadapted consumers fail at compile time; Antigravity RPC conversations ride along as paired bare keys. --- CHANGELOG.md | 1 + src/doctor.ts | 3 +- src/parse-worker.ts | 7 +- src/parser.ts | 29 +++--- src/providers/antigravity-keys.ts | 12 +++ src/providers/antigravity.ts | 19 ++-- src/providers/cline-cli.ts | 5 +- src/providers/cline.ts | 3 +- src/providers/codebuff.ts | 5 +- src/providers/codewhale.ts | 5 +- src/providers/codex.ts | 7 +- src/providers/copilot.ts | 13 +-- src/providers/crush.ts | 5 +- src/providers/cursor-agent.ts | 5 +- src/providers/cursor.ts | 15 +-- src/providers/devin.ts | 5 +- src/providers/droid.ts | 5 +- src/providers/dsh.ts | 5 +- src/providers/forge.ts | 5 +- src/providers/gemini.ts | 7 +- src/providers/goose.ts | 5 +- src/providers/grok.ts | 5 +- src/providers/grokbot.ts | 5 +- src/providers/hermes.ts | 5 +- src/providers/ibm-bob.ts | 3 +- src/providers/kilo-code.ts | 3 +- src/providers/kimi.ts | 5 +- src/providers/kimicode.ts | 5 +- src/providers/kiro.ts | 15 +-- src/providers/mistral-vibe.ts | 5 +- src/providers/mux.ts | 5 +- src/providers/open-design.ts | 5 +- src/providers/openclaude.ts | 5 +- src/providers/openclaw.ts | 5 +- src/providers/opencode-file-parser.ts | 3 +- src/providers/opencode.ts | 3 +- src/providers/pi.ts | 7 +- src/providers/quickdesk.ts | 7 +- src/providers/qwen.ts | 5 +- src/providers/roo-code.ts | 3 +- src/providers/sqlite-session-parser.ts | 3 +- src/providers/types.ts | 3 +- src/providers/vercel-gateway.ts | 5 +- src/providers/vscode-cline-parser.ts | 3 +- src/providers/warp.ts | 5 +- src/providers/zcode.ts | 5 +- src/providers/zed.ts | 7 +- src/providers/zerostack.ts | 5 +- src/session-cache.ts | 118 ++++++++++++++++++++--- tests/parse-workers.test.ts | 6 +- tests/providers/antigravity.test.ts | 37 ++++++- tests/session-cache-range-filter.test.ts | 63 ++++++++++-- 52 files changed, 379 insertions(+), 146 deletions(-) create mode 100644 src/providers/antigravity-keys.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 25db11202..5d3e4a4db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - **Ranged commands decode cache shards incrementally instead of assembling whole multi-hundred-megabyte files.** Session month shards now decode one turn at a time and keep only in-range turns (with the exact same kept/dropped/carry contract as before, verified by the range-filter suite, including a 30k-turn record test); shard loads run serially and the pre-lock snapshot is released before the canonical reload instead of overlapping it. All retained strings are detached from tokenizer buffers (sliced views pinned whole input chunks: ~1.2GB unexplained heap on this corpus). Shards at or under 256MB skip the streaming walk and decode with plain JSON.parse through the same per-record projection (byte-identical results, pinned by the size-gate parity suite): 166MB of shards in 333ms instead of 15-20s, while the walk still bounds the shard past V8's max string length. Out-of-range turns contribute their raw dedup keys to cross-file suppression markers, and provider-scoped queries skip unrelated sections. - **The codex result cache streams instead of whole-parsing.** The single result file decodes entry by entry (a top-level decode assembled hundreds of megabytes first); discovery labels come from calls-free metadata without loading the calls map; publishes merge dirty entries over the published bytes instead of rewriting from memory; single-flight loads are shared across concurrent readers and retry when a publish lands mid-decode, with a global clear epoch so in-flight loads cannot repopulate a cleared memo. +- **Cross-file dedup keys are full SHA-256 digests instead of raw strings.** Dropped-key markers and shared dedup sets store 32-character digests (~140MB of raw key characters on the reporter corpus); every insert and lookup hashes uniformly, and a collision would suppress rather than fabricate a call. The set is a plain class (not a Set subclass) so unadapted consumers fail at compile time; Antigravity RPC conversations ride along as paired bare keys since prefix matching cannot work on digests. ### Fixed (desktop) - **The Models table shows every model that ran, including the ones under a cent.** The CLI's `models` command defaults `minCost` to $0.01, and the desktop bridge passed neither `--min-cost` nor `--unpriced`, so the table silently dropped every row below a cent — which by construction excluded every unpriced row too (a `0 >= 0.01` filter), leaving #1443's dimming and add-alias affordances unreachable in the shipped app. `codeburn:getModels` now passes `--min-cost 0` (and the demo bridge mirrors it), so sub-cent and unpriced rows arrive and render with their existing dim treatment; on a real lifetime corpus that recovers 10 rows and 2,160 calls the default filter hid (43 → 53 rows, verified in both themes). A row priced between $0.00 and $0.01 renders as "$0.00" without dimming — it is genuinely priced, just below the display floor. Fixes #1465. diff --git a/src/doctor.ts b/src/doctor.ts index fcdd743c8..43b8f5623 100644 --- a/src/doctor.ts +++ b/src/doctor.ts @@ -9,6 +9,7 @@ import { getAllProviders } from './providers/index.js' import type { Provider } from './providers/types.js' import { dailyCachePath, isTurnResidueOnly, type DailyEntry } from './daily-cache.js' import { + DedupSet, PROVIDER_ENV_VARS, PROVIDER_PARSE_VERSIONS, loadCache, @@ -302,7 +303,7 @@ async function collectOneProvider( if (sources.length > 0) { const sample = sources.slice(0, sampleLimit) base.bounded = sample.length < sources.length - const seenKeys = new Set() + const seenKeys = new DedupSet() for (const source of sample) { base.sampled++ try { diff --git a/src/parse-worker.ts b/src/parse-worker.ts index df3d9cbf4..bd298865d 100644 --- a/src/parse-worker.ts +++ b/src/parse-worker.ts @@ -3,6 +3,7 @@ import { restorePricingState, type PricingSnapshot } from './models.js' import type { ParseJob } from './parse-workers.js' import { parseClaudeFileFull } from './parser.js' import { parseCodexFileFull } from './providers/codex.js' +import { DedupSet } from './session-cache.js' const port = parentPort if (!port) throw new Error('parse-worker must be started as a worker thread') @@ -19,14 +20,14 @@ restorePricingState((workerData as { pricing: PricingSnapshot }).pricing) port.on('message', (msg: ParseJob) => { void (async () => { try { - const seen = new Set() + const seen = new DedupSet() if (msg.kind === 'codex') { const parsed = await parseCodexFileFull(msg.source, seen) - port.postMessage({ json: JSON.stringify({ ...parsed, keys: [...seen], path: msg.source.path }) }) + port.postMessage({ json: JSON.stringify({ ...parsed, keys: [...seen.values()], path: msg.source.path }) }) return } const parsed = await parseClaudeFileFull(msg.filePath, seen) - port.postMessage({ json: parsed === null ? null : JSON.stringify({ ...parsed, msgIds: [...seen], path: msg.filePath }) }) + port.postMessage({ json: parsed === null ? null : JSON.stringify({ ...parsed, msgIds: [...seen.values()], path: msg.filePath }) }) } catch (err) { port.postMessage({ error: err instanceof Error ? err.message : String(err) }) } diff --git a/src/parser.ts b/src/parser.ts index 51685a91f..cd593b543 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -30,6 +30,7 @@ import { beginColdHydration, cleanupOrphanedTempFiles, computeEnvFingerprint, + DedupSet, DURABLE_PROVIDER_NAMES, emptyCache, fileFirstTurnProject, @@ -1570,7 +1571,7 @@ export function dedupeStreamingMessageIds(entries: JournalEntry[]): JournalEntry return result } -export function groupIntoTurns(entries: JournalEntry[], seenMsgIds: Set, toolResultMeta?: Map): ParsedTurn[] { +export function groupIntoTurns(entries: JournalEntry[], seenMsgIds: DedupSet, toolResultMeta?: Map): ParsedTurn[] { const turns: ParsedTurn[] = [] let currentUserMessage = '' let currentCalls: ParsedApiCall[] = [] @@ -1871,7 +1872,7 @@ function buildSessionSummary( async function parseSessionFile( filePath: string, project: string, - seenMsgIds: Set, + seenMsgIds: DedupSet, dateRange?: DateRange, ): Promise<{ session: SessionSummary; canonicalCwd?: string } | null> { // Skip files whose mtime is older than the range start. A session file @@ -1988,7 +1989,7 @@ export async function readAgentType(filePath: string): Promise, - seenMsgIds: Set, + seenMsgIds: DedupSet, diskCache: SessionCache, dateRange?: DateRange, // Cold-run robustness: called after every parsed Claude file so a throttled @@ -2164,11 +2165,11 @@ async function scanProjectDirs( if (result.parsed.path !== filePath) { throw new Error(`claude parse worker result out of order: got ${result.parsed.path}, expected ${filePath}`) } - if (result.parsed.msgIds.some(id => seenMsgIds.has(id))) { + if (result.parsed.msgIds.some(id => seenMsgIds.hasHashed(id))) { workerDiscards++ parsed = undefined } else { - for (const id of result.parsed.msgIds) seenMsgIds.add(id) + for (const id of result.parsed.msgIds) seenMsgIds.addHashed(id) parsed = result.parsed } } else if (result?.ok) { @@ -3016,7 +3017,7 @@ function claudeLineageForParse( export async function parseClaudeFileFull( filePath: string, - seenMsgIds: Set, + seenMsgIds: DedupSet, ): Promise { const tracker = { lastCompleteLineOffset: 0 } const toolResultMeta = new Map() @@ -3340,7 +3341,7 @@ function classifiedTurnSlicedToDays(turn: ClassifiedTurn, days: Set): Cl export async function parseProviderSources( providerName: string, sources: SessionSource[], - seenKeys: Set, + seenKeys: DedupSet, diskCache: SessionCache, dateRange?: DateRange, // Cold-run robustness: called after each source's cache entry lands (mirrors @@ -3474,8 +3475,10 @@ export async function parseProviderSources( // Parser dedup: cross-provider keys + cached file keys. // Separate from seenKeys so parsing doesn't suppress query-time output. - const parserDedup = new Set() - for (const key of seenKeys) parserDedup.add(key) + // Clone the incoming digests verbatim (see DedupSet.values). The caller + // contract is DedupSet-only, so live inserts below stay comparable. + const parserDedup = new DedupSet() + for (const key of seenKeys.values()) parserDedup.addHashed(key) for (const { cached } of unchangedSources) { // Dropped turns contribute only dedup keys (see RangeFilteredMeta). seedDroppedKeys(parserDedup, cached) @@ -3571,10 +3574,10 @@ export async function parseProviderSources( if (result.parsed.path !== source.path) { throw new Error(`codex parse worker result out of order: got ${result.parsed.path}, expected ${source.path}`) } - if (result.parsed.keys.some(k => parserDedup.has(k))) { + if (result.parsed.keys.some(k => parserDedup.hasHashed(k))) { workerDiscards++ } else { - for (const k of result.parsed.keys) parserDedup.add(k) + for (const k of result.parsed.keys) parserDedup.addHashed(k) providerCalls = result.parsed.calls // The worker never touches the codex cache; publish its entry here, // in install order, so flushCodexCache writes what serial would. @@ -5659,8 +5662,8 @@ async function runParseInner( deferredRetryableSource = false firstPaintDeferredThisRun = 0 dateFloorSkippedProviders.clear() - const seenMsgIds = new Set() - const seenKeys = new Set() + const seenMsgIds = new DedupSet() + const seenKeys = new DedupSet() const discovery = snapshotOnly ? { sources: [], failedProviders: [] } : await discoverAllSessionsWithFailures(providerFilter) diff --git a/src/providers/antigravity-keys.ts b/src/providers/antigravity-keys.ts new file mode 100644 index 000000000..430fc0a3d --- /dev/null +++ b/src/providers/antigravity-keys.ts @@ -0,0 +1,12 @@ +/// Antigravity RPC dedup-key shapes, shared by the provider and the generic +/// dedup primitive without pulling the whole provider module (with its heavy +/// native deps) into the session-cache import graph. + +/// Bare conversation key for an RPC-form dedup key (`antigravity:{cid}:...` +/// yields `antigravity:{cid}`), else null. Statusline and other shapes never +/// match: only the RPC form feeds the conversation prefix check. +export function rpcConversationBareKey(key: string): string | null { + const parts = key.split(':') + if (parts.length < 3 || parts[0] !== 'antigravity' || !parts[1]) return null + return `antigravity:${parts[1]}` +} diff --git a/src/providers/antigravity.ts b/src/providers/antigravity.ts index 09f2acbf2..5bd00f919 100644 --- a/src/providers/antigravity.ts +++ b/src/providers/antigravity.ts @@ -10,6 +10,7 @@ import { getCodeburnCacheDir, readExistingTextFile } from '../cache-dir.js' import { calculateCost } from '../models.js' import { isSqliteAvailable, isSqliteBusyError, openDatabase } from '../sqlite.js' import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import { DedupSet } from '../session-cache.js' type AntigravityConversationRoot = { dir: string @@ -1247,17 +1248,15 @@ function parseStatusLineEvent(input: unknown): StatusLineEvent | null { } /// True when the shared dedup set already holds an RPC-cache entry for this -/// conversation (raw prefix scan over the exact keys). -function hasRpcCacheForConversation(seenKeys: Set, conversationId: string): boolean { - const prefix = `antigravity:${conversationId}:` - for (const key of seenKeys) { - if (key.startsWith(prefix)) return true - } - return false +/// conversation. The set holds digests (prefix matching is impossible), so +/// this checks the bare conversation key, which DedupSet.add pairs with +/// every RPC-form insert. +function hasRpcCacheForConversation(seenKeys: DedupSet, conversationId: string): boolean { + return seenKeys.has(`antigravity:${conversationId}`) } -async function parseStatusLineCalls(source: SessionSource, seenKeys: Set): Promise { +async function parseStatusLineCalls(source: SessionSource, seenKeys: DedupSet): Promise { const raw = await readFile(source.path, 'utf-8').catch(() => '') const runsByConversation = new Map>() for (const line of raw.split(/\r?\n/)) { @@ -1488,7 +1487,7 @@ function withFallbackTimestamp(call: ParsedProviderCall, fallbackTimestamp: stri return call.timestamp ? call : { ...call, timestamp: fallbackTimestamp } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { if (isAntigravityStatusLineEventsPath(source.path)) { @@ -1640,7 +1639,7 @@ export function createAntigravityProvider(): Provider { return discoverAntigravitySessionSources() }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/cline-cli.ts b/src/providers/cline-cli.ts index 568d0dfad..34aa4d403 100644 --- a/src/providers/cline-cli.ts +++ b/src/providers/cline-cli.ts @@ -7,6 +7,7 @@ import { readSessionFile } from '../fs-utils.js' import { calculateCost, getShortModelName } from '../models.js' import type { ToolCall } from '../types.js' import type { ParsedProviderCall, ProbeRoot, Provider, SessionParser, SessionSource } from './types.js' +import type { DedupSet } from '../session-cache.js' // The Cline CLI (npm `cline`, 3.x) stores sessions in a layout unrelated to the // VS Code extension's tasks/ui_messages.json tree that `cline.ts` reads: @@ -239,7 +240,7 @@ async function readJson(path: string): Promise { } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { // `source.path` is the growing `.messages.json` file (see @@ -420,7 +421,7 @@ export function createClineCliProvider(overrideDir?: string): Provider { return sources }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/cline.ts b/src/providers/cline.ts index f7d2d9cca..3304b31b7 100644 --- a/src/providers/cline.ts +++ b/src/providers/cline.ts @@ -4,6 +4,7 @@ import { basename, join } from 'path' import { discoverClineTasks, createClineParser, clineTaskRoots } from './vscode-cline-parser.js' import type { ProbeRoot, Provider, SessionSource, SessionParser } from './types.js' +import type { DedupSet } from '../session-cache.js' const EXTENSION_ID = 'saoudrizwan.claude-dev' @@ -69,7 +70,7 @@ export function createClineProvider(overrideDirs?: string | string[]): Provider return dedupeTaskSources(await discoverClineTasks(EXTENSION_ID, 'cline', 'Cline', baseDirs)) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createClineParser(source, seenKeys, 'cline') }, } diff --git a/src/providers/codebuff.ts b/src/providers/codebuff.ts index 418241075..b5d27f44b 100644 --- a/src/providers/codebuff.ts +++ b/src/providers/codebuff.ts @@ -5,6 +5,7 @@ import { homedir } from 'os' import { calculateCost } from '../models.js' import { extractBashCommands } from '../bash-utils.js' import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' // Codebuff (formerly Manicode) uses a credit-based billing system. The local // chat-messages.json doesn't record per-call token counts the way Claude Code @@ -330,7 +331,7 @@ function extractChannelFromChatDir(chatDir: string): string | null { return channel ? channel : null } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const chatDir = source.path @@ -445,7 +446,7 @@ export function createCodebuffProvider(baseDir?: string): Provider { return discoverSessionsInRoots(roots) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/codewhale.ts b/src/providers/codewhale.ts index b48574fec..f36ea59a3 100644 --- a/src/providers/codewhale.ts +++ b/src/providers/codewhale.ts @@ -7,6 +7,7 @@ import { readSessionFile } from '../fs-utils.js' import { calculateCost, getShortModelName } from '../models.js' import type { ToolCall } from '../types.js' import type { ProbeRoot, ParsedProviderCall, Provider, SessionParser, SessionSource } from './types.js' +import type { DedupSet } from '../session-cache.js' const METADATA_PREFIX_BYTES = 64 * 1024 @@ -366,7 +367,7 @@ function reportedCost(cost: CodeWhaleCost | undefined): { value: number; exact: } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const raw = await readSessionFile(source.path) @@ -477,7 +478,7 @@ export function createCodeWhaleProvider(overrideDirs?: string | string[]): Provi return sources }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/codex.ts b/src/providers/codex.ts index b8fe57ef4..851335a31 100644 --- a/src/providers/codex.ts +++ b/src/providers/codex.ts @@ -12,6 +12,7 @@ import { normalizeContentBlocks } from '../content-utils.js' import { estimateTokensFromChars } from '../token-estimate.js' import type { ToolCall } from '../types.js' import type { Provider, ProbeRoot, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' import { defaultBilledCodexHome, defaultLauncherRoots, FIRST_LINE_READ_CAP, isNestedLauncherCodexHome, listRolloutSessionIds, rolloutFileSessionId, sameCodexHome } from '../launcher-homes.js' const modelDisplayNames: Record = { @@ -701,7 +702,7 @@ export type CodexCacheWrite = { // written comes back through `capture` for the caller to install. That is what // lets a worker thread run this exact decode without owning the cache module's // per-directory state. -function createParser(source: SessionSource, seenKeys: Set, capture?: { write?: CodexCacheWrite }, rangeStartMs?: number): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet, capture?: { write?: CodexCacheWrite }, rangeStartMs?: number): SessionParser { return { async *parse(): AsyncGenerator { // PR-A scope: unfiltered lookup (range-filtered serve rides the stacked @@ -1362,7 +1363,7 @@ export type CodexFullParse = { calls: ParsedProviderCall[]; write?: CodexCacheWr /// is the dedup set the decode runs against — pass an empty one off-thread and /// let the caller prove no earlier file claimed any of the keys before /// installing the result. -export async function parseCodexFileFull(source: SessionSource, seenKeys: Set): Promise { +export async function parseCodexFileFull(source: SessionSource, seenKeys: DedupSet): Promise { const capture: { write?: CodexCacheWrite } = {} const calls: ParsedProviderCall[] = [] for await (const call of createParser(source, seenKeys, capture).parse()) calls.push(call) @@ -1442,7 +1443,7 @@ export function createCodexProvider( createSessionParser( source: SessionSource, - seenKeys: Set, + seenKeys: DedupSet, dateRange?: { start: Date; end: Date }, ): SessionParser { return createParser(source, seenKeys, undefined, dateRange?.start.getTime()) diff --git a/src/providers/copilot.ts b/src/providers/copilot.ts index 6d87665d4..9de52b0b2 100644 --- a/src/providers/copilot.ts +++ b/src/providers/copilot.ts @@ -74,6 +74,7 @@ import type { SessionParser, ParsedProviderCall, } from './types.js' +import type { DedupSet } from '../session-cache.js' // --------------------------------------------------------------------------- // Model display names (unchanged from original) @@ -752,7 +753,7 @@ function inferTranscriptModel(lines: string[]): string { */ function createJsonlParser( source: SessionSource, - seenKeys: Set, + seenKeys: DedupSet, isTranscript: boolean ): SessionParser { return { @@ -1104,7 +1105,7 @@ function createJsonlParser( function createChatSessionParser( source: SessionSource, - seenKeys: Set + seenKeys: DedupSet ): SessionParser { return { async *parse(): AsyncGenerator { @@ -1681,7 +1682,7 @@ function extractJetBrainsDbTurns(raw: string): JBDbTurn[] { function createJetBrainsParser( source: JetBrainsSessionSource, - seenKeys: Set + seenKeys: DedupSet ): SessionParser { return { async *parse(): AsyncGenerator { @@ -1774,7 +1775,7 @@ function createJetBrainsParser( function createOtelParser( source: SessionSource, - seenKeys: Set + seenKeys: DedupSet ): SessionParser { return { async *parse(): AsyncGenerator { @@ -2132,7 +2133,7 @@ function fnv1a64(s: string): string { function createSessionStoreParser( source: SessionStoreSessionSource, - seenKeys: Set + seenKeys: DedupSet ): SessionParser { return { async *parse(): AsyncGenerator { @@ -3052,7 +3053,7 @@ export function createCopilotProvider( createSessionParser( source: SessionSource, - seenKeys: Set + seenKeys: DedupSet ): SessionParser { // Route to the correct parser based on source type. // The dedup key set (seenKeys) is shared across both parsers, diff --git a/src/providers/crush.ts b/src/providers/crush.ts index 0223dfdce..a045f219e 100644 --- a/src/providers/crush.ts +++ b/src/providers/crush.ts @@ -5,6 +5,7 @@ import { homedir, platform } from 'os' import { calculateCost } from '../models.js' import { isSqliteAvailable, getSqliteLoadError, openDatabase, type SqliteDatabase } from '../sqlite.js' import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' /// Crush stores per-project SQLite databases discovered through a JSON registry. /// We only read both. Schema source: charmbracelet/crush @@ -118,7 +119,7 @@ function dominantModel(db: SqliteDatabase, sessionId: string): string { } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { if (!isSqliteAvailable()) { @@ -253,7 +254,7 @@ export function createCrushProvider(): Provider { return sources }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/cursor-agent.ts b/src/providers/cursor-agent.ts index 3a20d7438..0fd89d6ea 100644 --- a/src/providers/cursor-agent.ts +++ b/src/providers/cursor-agent.ts @@ -15,6 +15,7 @@ import type { ParsedProviderCall, ProbeRoot, } from './types.js' +import type { DedupSet } from '../session-cache.js' type ConversationSummary = { conversationId: string @@ -389,7 +390,7 @@ function parseTranscript(raw: string): { turns: ParsedTurn[]; recognized: boolea function createParser( source: SessionSource, - seenKeys: Set, + seenKeys: DedupSet, dbPath: string, summariesByConversationId: Map, ): SessionParser { @@ -545,7 +546,7 @@ export function createCursorAgentProvider(baseDirOverride?: string): Provider { return sources }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys, dbPath, summariesByConversationId) }, } diff --git a/src/providers/cursor.ts b/src/providers/cursor.ts index ad0248373..622957c3a 100644 --- a/src/providers/cursor.ts +++ b/src/providers/cursor.ts @@ -18,6 +18,7 @@ import { import { estimateTokensFromChars } from '../token-estimate.js' import type { DateRange } from '../types.js' import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import { DedupSet } from '../session-cache.js' /** Matches cli-date.ts "all" period cap (6 months). */ const CURSOR_MAX_LOOKBACK_MONTHS = 6 @@ -681,7 +682,7 @@ type ComposerScan = { function parseBubbles( db: SqliteDatabase, - seenKeys: Set, + seenKeys: DedupSet, timeFloor: string, agentKvTimestamp: string, ): { calls: ParsedProviderCall[] } { @@ -948,7 +949,7 @@ function parseBubbles( function createParser( source: SessionSource, - seenKeys: Set, + seenKeys: DedupSet, dateRange?: DateRange, ): SessionParser { const timeFloor = getCursorTimeFloor(dateRange) @@ -1008,10 +1009,10 @@ function createParser( process.stderr.write('codeburn: Cursor storage format not recognized. You may need to update CodeBurn.\n') return } - // Use a fresh local Set for intra-parse dedup so the global - // seenKeys is not mutated by calls that the workspace filter is - // about to drop. Cross-source dedup happens at yield time. - const localSeen = new Set() + // Use a fresh local DedupSet for intra-parse dedup so the global + // seenKeys is not mutated by calls that the workspace filter is + // about to drop. Cross-source dedup happens at yield time. + const localSeen = new DedupSet() // agentKv rows carry no timestamps; sessions found only there get // the DB's last-write time. let agentKvTimestamp: string @@ -1088,7 +1089,7 @@ export function createCursorProvider(dbPathOverride?: string): Provider { return sources }, - createSessionParser(source: SessionSource, seenKeys: Set, dateRange?: DateRange): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet, dateRange?: DateRange): SessionParser { return createParser(source, seenKeys, dateRange) }, } diff --git a/src/providers/devin.ts b/src/providers/devin.ts index 7f57ca1fa..2ca7adc74 100644 --- a/src/providers/devin.ts +++ b/src/providers/devin.ts @@ -11,6 +11,7 @@ import type { SessionSource, ParsedProviderCall, } from "./types.js"; +import type { DedupSet } from "../session-cache.js"; import { readSessionFile } from "../fs-utils.js"; import { isPositiveNumber, safeNumber } from "../parser.js"; @@ -483,7 +484,7 @@ function loadSessionMetadata( class DevinSessionParser implements SessionParser { constructor( private source: SessionSource, - private seenKeys: Set, + private seenKeys: DedupSet, private sessionMetadata: Map, ) {} @@ -634,7 +635,7 @@ export function createDevinProvider(cliDir?: string): Provider { createSessionParser( source: SessionSource, - seenKeys: Set, + seenKeys: DedupSet, ): SessionParser { return new DevinSessionParser(source, seenKeys, getSessionMetadata()); }, diff --git a/src/providers/droid.ts b/src/providers/droid.ts index 5da2239eb..03afcd0f1 100644 --- a/src/providers/droid.ts +++ b/src/providers/droid.ts @@ -13,6 +13,7 @@ import type { ParsedProviderCall, ProbeRoot, } from './types.js' +import type { DedupSet } from '../session-cache.js' const toolNameMap: Record = { Read: 'Read', @@ -112,7 +113,7 @@ function extractDroidBashCommands(command: string): string[] { function createParser( source: SessionSource, - seenKeys: Set, + seenKeys: DedupSet, ): SessionParser { return { async *parse(): AsyncGenerator { @@ -402,7 +403,7 @@ export function createDroidProvider(factoryDir?: string): Provider { createSessionParser( source: SessionSource, - seenKeys: Set, + seenKeys: DedupSet, ): SessionParser { return createParser(source, seenKeys) }, diff --git a/src/providers/dsh.ts b/src/providers/dsh.ts index 5677372bf..dd23f7809 100644 --- a/src/providers/dsh.ts +++ b/src/providers/dsh.ts @@ -7,6 +7,7 @@ import { MAX_SESSION_FILE_BYTES, readSessionFile, readSessionLines } from '../fs import { billableOutputTokens, calculateCost, getShortModelName } from '../models.js' import { extractBashCommands } from '../bash-utils.js' import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' // DeepSeek Harness (dsh) stores one session per directory: // /sessions//session-/session.jsonl.zstd @@ -489,7 +490,7 @@ function emptyStepBucket(): StepBucket { return { observations: [], tools: [], skills: [], bashCommands: [] } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const lines = await readEventLines(source.path) @@ -763,7 +764,7 @@ export function createDshProvider(dshHomeOverride?: string): Provider { return discoverSessionsInDir(sessionsDir, onSkippedVersion) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/forge.ts b/src/providers/forge.ts index 47dd6dbbe..044c9135a 100644 --- a/src/providers/forge.ts +++ b/src/providers/forge.ts @@ -6,6 +6,7 @@ import { extractBashCommands } from '../bash-utils.js' import { calculateCost } from '../models.js' import { getSqliteLoadError, isSqliteAvailable, openDatabase, type SqliteDatabase } from '../sqlite.js' import type { ParsedProviderCall, ProbeRoot, Provider, SessionParser, SessionSource } from './types.js' +import type { DedupSet } from '../session-cache.js' type ConversationRow = { conversation_id: string @@ -137,7 +138,7 @@ function splitSourcePath(path: string): { dbPath: string; conversationId: string return { dbPath: path.slice(0, idx), conversationId: path.slice(idx + 1) } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { if (!isSqliteAvailable()) { @@ -279,7 +280,7 @@ export function createForgeProvider(dbPath = DEFAULT_DB_PATH): Provider { return discoverFromDb(dbPath) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/gemini.ts b/src/providers/gemini.ts index cffeb9b44..379c9a50f 100644 --- a/src/providers/gemini.ts +++ b/src/providers/gemini.ts @@ -6,6 +6,7 @@ import { readSessionFile } from '../fs-utils.js' import { calculateCost } from '../models.js' import { extractBashCommands } from '../bash-utils.js' import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' const toolNameMap: Record = { read_file: 'Read', @@ -64,7 +65,7 @@ type GeminiSession = { kind?: string } -function parseSession(data: GeminiSession, seenKeys: Set): ParsedProviderCall[] { +function parseSession(data: GeminiSession, seenKeys: DedupSet): ParsedProviderCall[] { const results: ParsedProviderCall[] = [] let lastUserMessage = '' @@ -183,7 +184,7 @@ function parseJsonl(raw: string): GeminiSession | null { return { sessionId, projectHash, startTime, lastUpdated, kind, messages } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const raw = await readSessionFile(source.path) @@ -280,7 +281,7 @@ export function createGeminiProvider(): Provider { return discoverSessions() }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/goose.ts b/src/providers/goose.ts index 61abc7a20..91a7d11f3 100644 --- a/src/providers/goose.ts +++ b/src/providers/goose.ts @@ -6,6 +6,7 @@ import { extractBashCommands } from '../bash-utils.js' import { isSqliteAvailable, getSqliteLoadError, openDatabase, blobToText, type SqliteDatabase } from '../sqlite.js' import type { ToolCall } from '../types.js' import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' type SessionRow = { id: string @@ -150,7 +151,7 @@ function getFirstUserMessage(db: SqliteDatabase, sessionId: string): string { } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { if (!isSqliteAvailable()) { @@ -291,7 +292,7 @@ export function createGooseProvider(): Provider { return discoverFromDb(dbPath) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/grok.ts b/src/providers/grok.ts index 58567c7e2..710c25c1a 100644 --- a/src/providers/grok.ts +++ b/src/providers/grok.ts @@ -6,6 +6,7 @@ import { FS_SCAN_CONCURRENCY, mapWithConcurrency, readSessionFile } from '../fs- import { calculateCost, getModelCosts, getShortModelName } from '../models.js' import { extractBashCommands } from '../bash-utils.js' import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' // Grok Build (xAI's coding CLI) stores one session per directory at // /sessions///, where grok-home is $GROK_HOME @@ -370,7 +371,7 @@ function hasPositiveTotals(totals: GrokTokenTotals): boolean { || totals.reasoning > 0 } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const dir = dirname(source.path) @@ -508,7 +509,7 @@ export function createGrokProvider(sessionsDir?: string): Provider { return discoverSessions(dir) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/grokbot.ts b/src/providers/grokbot.ts index c5ab50a9f..e5e9ac12a 100644 --- a/src/providers/grokbot.ts +++ b/src/providers/grokbot.ts @@ -6,6 +6,7 @@ import { FS_SCAN_CONCURRENCY, mapWithConcurrency, readSessionFile } from '../fs- import { calculateCost, getShortModelName } from '../models.js' import { estimateTokensFromChars } from '../token-estimate.js' import type { ParsedProviderCall, ProbeRoot, Provider, SessionParser, SessionSource } from './types.js' +import type { DedupSet } from '../session-cache.js' // Grok Bot is xAI's Electron desktop agent app (bundle id com.anysphere.sand), // not Grok Build — xAI's coding CLI, which is the separate `grok` provider. @@ -186,7 +187,7 @@ function groupRequests(entries: unknown[]): GrokbotRequest[] { return [...requests.values()] } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const value = await readSlice(source.path) @@ -296,7 +297,7 @@ export function createGrokbotProvider(persistenceDir?: string): Provider { return discoverSessions(resolveDir()) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/hermes.ts b/src/providers/hermes.ts index 7b64c3b50..5b250a219 100644 --- a/src/providers/hermes.ts +++ b/src/providers/hermes.ts @@ -8,6 +8,7 @@ import { calculateCost, getShortModelName, routeFromProviderField } from '../mod import { isUserHomeRoot } from '../path-privacy.js' import { isSqliteAvailable, getSqliteLoadError, openDatabase, isSqliteBusyError, type SqliteDatabase } from '../sqlite.js' import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' import type { ToolCall } from '../types.js' import { getHermesCursor, @@ -571,7 +572,7 @@ async function discoverFromDb(dbPath: string, profile: string): Promise, hermesHome: string): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet, hermesHome: string): SessionParser { return { async *parse(): AsyncGenerator { if (!isSqliteAvailable()) { @@ -783,7 +784,7 @@ export function createHermesProvider(hermesHomeOverride?: string): Provider { return sessions }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys, hermesHome) }, } diff --git a/src/providers/ibm-bob.ts b/src/providers/ibm-bob.ts index a9c0d12d4..82e9696e8 100644 --- a/src/providers/ibm-bob.ts +++ b/src/providers/ibm-bob.ts @@ -4,6 +4,7 @@ import { homedir } from 'os' import { getShortModelName } from '../models.js' import { discoverClineTasksInBaseDirs, createClineParser } from './vscode-cline-parser.js' import type { ProbeRoot, Provider, SessionSource, SessionParser } from './types.js' +import type { DedupSet } from '../session-cache.js' const PROVIDER_NAME = 'ibm-bob' const DISPLAY_NAME = 'IBM Bob' @@ -54,7 +55,7 @@ export function createIBMBobProvider(overrideDir?: string): Provider { return discoverClineTasksInBaseDirs(dirs, PROVIDER_NAME, DISPLAY_NAME) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createClineParser(source, seenKeys, PROVIDER_NAME, FALLBACK_MODEL) }, } diff --git a/src/providers/kilo-code.ts b/src/providers/kilo-code.ts index 70d89bbf6..1b7a2329d 100644 --- a/src/providers/kilo-code.ts +++ b/src/providers/kilo-code.ts @@ -4,6 +4,7 @@ import { homedir } from 'os' import { discoverClineTasks, createClineParser, clineTaskRoots } from './vscode-cline-parser.js' import { discoverSqliteSessions, createSqliteSessionParser, type SqliteProviderConfig } from './sqlite-session-parser.js' import type { ProbeRoot, Provider, SessionSource, SessionParser } from './types.js' +import type { DedupSet } from '../session-cache.js' const EXTENSION_ID = 'kilocode.kilo-code' const PROVIDER_NAME = 'kilo-code' @@ -49,7 +50,7 @@ export function createKiloCodeProvider(overrideDir?: string | string[]): Provide return [...oldSessions, ...dbSessions] }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { if (source.path.includes('.db:')) { return createSqliteSessionParser(source, seenKeys, sqliteConfig) } diff --git a/src/providers/kimi.ts b/src/providers/kimi.ts index ceb98519b..a59e1c2e9 100644 --- a/src/providers/kimi.ts +++ b/src/providers/kimi.ts @@ -7,6 +7,7 @@ import { extractBashCommands } from '../bash-utils.js' import { readSessionLines } from '../fs-utils.js' import { calculateCost, getShortModelName } from '../models.js' import type { ProbeRoot, ParsedProviderCall, Provider, SessionParser, SessionSource } from './types.js' +import type { DedupSet } from '../session-cache.js' type JsonObject = Record @@ -241,7 +242,7 @@ function extractTool(payload: JsonObject): { tool: string; bashCommands: string[ return { tool, bashCommands } } -function createParser(source: SessionSource, shareDir: string, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, shareDir: string, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const configuredModel = await getConfiguredModel(shareDir) @@ -389,7 +390,7 @@ export function createKimiProvider(overrideDir?: string): Provider { return sources }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, shareDir, seenKeys) }, } diff --git a/src/providers/kimicode.ts b/src/providers/kimicode.ts index d10ad95cf..47c495481 100644 --- a/src/providers/kimicode.ts +++ b/src/providers/kimicode.ts @@ -6,6 +6,7 @@ import { extractBashCommands } from '../bash-utils.js' import { calculateCost } from '../models.js' import { FS_SCAN_CONCURRENCY, mapWithConcurrency } from '../fs-utils.js' import type { ParsedProviderCall, ProbeRoot, Provider, SessionParser, SessionSource } from './types.js' +import type { DedupSet } from '../session-cache.js' type JsonObject = Record @@ -270,7 +271,7 @@ function toolDetails(value: unknown): { name: string; bashCommands: string[] } | } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { let contents: string @@ -429,7 +430,7 @@ export function createKimicodeProvider(homeOverride?: string): Provider { return all.sort((a, b) => a.path.localeCompare(b.path)) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/kiro.ts b/src/providers/kiro.ts index c901529a3..30028e9de 100644 --- a/src/providers/kiro.ts +++ b/src/providers/kiro.ts @@ -10,6 +10,7 @@ import { calculateCost } from '../models.js' import { estimateTokensFromChars } from '../token-estimate.js' import type { ToolCall } from '../types.js' import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' // Kiro bills in credits: individual plans are $20/mo for 1,000 credits and // overage is billed at $0.04 per additional credit. We price credits at the @@ -199,7 +200,7 @@ function extractStructuredToolNames(value: unknown, text: string, options: { inc return tools } -function parseChatFile(data: KiroChatFile, sessionId: string, project: string, seenKeys: Set): ParsedProviderCall[] { +function parseChatFile(data: KiroChatFile, sessionId: string, project: string, seenKeys: DedupSet): ParsedProviderCall[] { const results: ParsedProviderCall[] = [] const { chat, metadata } = data @@ -270,7 +271,7 @@ function parseChatFile(data: KiroChatFile, sessionId: string, project: string, s return results } -function parseModernExecution(data: KiroModernExecution, sourcePath: string, seenKeys: Set): ParsedProviderCall[] { +function parseModernExecution(data: KiroModernExecution, sourcePath: string, seenKeys: DedupSet): ParsedProviderCall[] { const results: ParsedProviderCall[] = [] if (Array.isArray(data['executions'])) return results @@ -445,7 +446,7 @@ type KiroCliSessionMeta = { } } -function parseCliSession(meta: KiroCliSessionMeta, entries: KiroCliEntry[], seenKeys: Set): ParsedProviderCall[] { +function parseCliSession(meta: KiroCliSessionMeta, entries: KiroCliEntry[], seenKeys: DedupSet): ParsedProviderCall[] { const results: ParsedProviderCall[] = [] const sessionId = meta.session_id const project = basename(meta.cwd || '') @@ -580,7 +581,7 @@ function parseCliSession(meta: KiroCliSessionMeta, entries: KiroCliEntry[], seen // Newer v1-era Kiro builds store session state here: history[] carries user prompts // and assistant messages, some of which are stubs referencing per-execution files // (parsed separately by parseModernExecution). -async function parseWorkspaceSession(record: Record, source: SessionSource, seenKeys: Set): Promise { +async function parseWorkspaceSession(record: Record, source: SessionSource, seenKeys: DedupSet): Promise { const results: ParsedProviderCall[] = [] const historyArr = record['history'] if (!Array.isArray(historyArr) || typeof record['sessionId'] !== 'string') return results @@ -695,7 +696,7 @@ type KiroV2SessionMeta = { lastModifiedAt?: string } -async function parseV2Session(source: SessionSource, seenKeys: Set): Promise { +async function parseV2Session(source: SessionSource, seenKeys: DedupSet): Promise { const results: ParsedProviderCall[] = [] const content = await readSessionFile(source.path) @@ -857,7 +858,7 @@ async function parseV2Session(source: SessionSource, seenKeys: Set): Pro return results } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { // v2 IDE store: ~/.kiro/sessions//sess_/messages.jsonl — a @@ -1203,7 +1204,7 @@ export function createKiroProvider(agentDirOverride?: string, workspaceStorageDi }) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/mistral-vibe.ts b/src/providers/mistral-vibe.ts index 23c3667ff..d1d7ece5a 100644 --- a/src/providers/mistral-vibe.ts +++ b/src/providers/mistral-vibe.ts @@ -6,6 +6,7 @@ import { readSessionFile, readSessionLines } from '../fs-utils.js' import { calculateCost } from '../models.js' import { extractBashCommands } from '../bash-utils.js' import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' import { safeNumber } from '../parser.js' const METADATA_FILENAME = 'meta.json' @@ -296,7 +297,7 @@ function allocateCost(total: number, count: number): number { return count <= 1 ? total : total / count } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const metadataPath = join(source.path, METADATA_FILENAME) @@ -431,7 +432,7 @@ export function createMistralVibeProvider(sessionsDir?: string): Provider { return sources }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/mux.ts b/src/providers/mux.ts index 2356e137f..9a9d1135f 100644 --- a/src/providers/mux.ts +++ b/src/providers/mux.ts @@ -6,6 +6,7 @@ import { readSessionLines } from '../fs-utils.js' import { calculateCost, getShortModelName } from '../models.js' import { extractBashCommands } from '../bash-utils.js' import type { Provider, SessionSource, SessionParser, ParsedProviderCall, ProbeRoot } from './types.js' +import type { DedupSet } from '../session-cache.js' import { safeNumber } from '../parser.js' const toolNameMap: Record = { @@ -147,7 +148,7 @@ async function discoverSessions(root: string): Promise { return sources } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const workspaceId = basename(dirname(source.path)) @@ -277,7 +278,7 @@ export function createMuxProvider(muxRoot?: string): Provider { return discoverSessions(root) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/open-design.ts b/src/providers/open-design.ts index 85e7b0130..664f0ff0a 100644 --- a/src/providers/open-design.ts +++ b/src/providers/open-design.ts @@ -5,6 +5,7 @@ import { homedir, platform } from 'os' import { readSessionLines } from '../fs-utils.js' import { calculateCost } from '../models.js' import type { Provider, SessionSource, SessionParser, ParsedProviderCall, ProbeRoot } from './types.js' +import type { DedupSet } from '../session-cache.js' const PROVIDER_NAME = 'open-design' const ENV_DIR = 'CODEBURN_OPEN_DESIGN_DIR' @@ -162,7 +163,7 @@ async function discoverOpenDesignSessions(baseDir: string): Promise): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const sessionId = basename(dirname(source.path)) @@ -254,7 +255,7 @@ export function createOpenDesignProvider(overrideDir?: string): Provider { return discoverOpenDesignSessions(overrideDir ?? getOpenDesignDir()) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/openclaude.ts b/src/providers/openclaude.ts index cfbe251d7..1690d8da5 100644 --- a/src/providers/openclaude.ts +++ b/src/providers/openclaude.ts @@ -21,6 +21,7 @@ import { readSessionFile } from '../fs-utils.js' import { calculateCost, getShortModelName } from '../models.js' import type { ToolCall } from '../types.js' import type { ParsedProviderCall, ProbeRoot, Provider, SessionParser, SessionSource } from './types.js' +import type { DedupSet } from '../session-cache.js' const PROVIDER_NAME = 'openclaude' const DISPLAY_NAME = 'OpenClaude' @@ -165,7 +166,7 @@ function projectFromCwd(cwd: string): string | undefined { return parts.at(-1) } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const raw = await readSessionFile(source.path) @@ -319,7 +320,7 @@ export function createOpenClaudeProvider(overrideProjectsDir?: string): Provider return sources }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/openclaw.ts b/src/providers/openclaw.ts index 140567f1c..1c327dc1b 100644 --- a/src/providers/openclaw.ts +++ b/src/providers/openclaw.ts @@ -6,6 +6,7 @@ import { readSessionFile } from '../fs-utils.js' import { calculateCost } from '../models.js' import { extractBashCommands } from '../bash-utils.js' import type { Provider, SessionSource, SessionParser, ParsedProviderCall, ProbeRoot } from './types.js' +import type { DedupSet } from '../session-cache.js' const toolNameMap: Record = { bash: 'Bash', @@ -86,7 +87,7 @@ function extractTools(content: Array<{ type?: string; name?: string; arguments?: return { tools, bashCommands } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const raw = await readSessionFile(source.path) @@ -279,7 +280,7 @@ export function createOpenClawProvider(overrideDir?: string): Provider { return all }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/opencode-file-parser.ts b/src/providers/opencode-file-parser.ts index 60bfb6a99..22269549d 100644 --- a/src/providers/opencode-file-parser.ts +++ b/src/providers/opencode-file-parser.ts @@ -3,6 +3,7 @@ import { join } from 'path' import { buildAssistantCall, sanitize, type MessageData, type PartData } from './session-message.js' import type { SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' // OpenCode 1.1+ stores sessions as file-based JSON instead of a SQLite DB: // storage/session//.json session metadata @@ -85,7 +86,7 @@ export async function discoverOpenCodeFileSessions( export function createOpenCodeFileSessionParser( source: SessionSource, - seenKeys: Set, + seenKeys: DedupSet, dataDir: string, providerName: string, ): SessionParser { diff --git a/src/providers/opencode.ts b/src/providers/opencode.ts index c80895133..a57a7afbf 100644 --- a/src/providers/opencode.ts +++ b/src/providers/opencode.ts @@ -5,6 +5,7 @@ import { getShortModelName } from '../models.js' import { discoverSqliteSessions, createSqliteSessionParser, type SqliteProviderConfig } from './sqlite-session-parser.js' import { discoverOpenCodeFileSessions, createOpenCodeFileSessionParser } from './opencode-file-parser.js' import type { Provider, ProbeRoot, SessionSource, SessionParser } from './types.js' +import type { DedupSet } from '../session-cache.js' const toolNameMap: Record = { bash: 'Bash', @@ -91,7 +92,7 @@ export function createOpenCodeProvider(dataDir?: string): Provider { return [...fileSessions, ...sqliteSessions] }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { if (source.path.endsWith('.json')) { return createOpenCodeFileSessionParser(source, seenKeys, resolvedDataDir, 'opencode') } diff --git a/src/providers/pi.ts b/src/providers/pi.ts index 66c9170ba..aacc3a86f 100644 --- a/src/providers/pi.ts +++ b/src/providers/pi.ts @@ -8,6 +8,7 @@ import { calculateCost } from '../models.js' import { extractBashCommands } from '../bash-utils.js' import { normalizeContentBlocks } from '../content-utils.js' import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' const modelDisplayNames: Record = { 'gpt-5.4': 'GPT-5.4', @@ -205,7 +206,7 @@ function resolveMessageModel(messageModel: string, resolvedModel: string): strin return messageModel || resolvedModel || 'gpt-5' } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const content = await readSessionFile(source.path) @@ -379,7 +380,7 @@ export function createPiProvider(sessionsDir?: string): Provider { return discoverSessionsInDir(dir, 'pi') }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } @@ -413,7 +414,7 @@ export function createOmpProvider(sessionsDir?: string): Provider { return discoverSessionsInDir(dir, 'omp') }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/quickdesk.ts b/src/providers/quickdesk.ts index 427419a94..9d18ea214 100644 --- a/src/providers/quickdesk.ts +++ b/src/providers/quickdesk.ts @@ -7,6 +7,7 @@ import { estimateTokensFromChars } from '../token-estimate.js' import { blobToText, isSqliteAvailable, openDatabase } from '../sqlite.js' import type { SqliteDatabase } from '../sqlite.js' import type { ParsedProviderCall, ProbeRoot, Provider, SessionParser, SessionSource } from './types.js' +import type { DedupSet } from '../session-cache.js' const METRICS_FILE_RE = /^metrics-(\d{4})-(\d{2})-(\d{2})\.jsonl$/ @@ -432,7 +433,7 @@ function commonCallFields(source: SessionSource, basePath: string) { } } -function createMetricsParser(source: SessionSource, seenKeys: Set): SessionParser { +function createMetricsParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const records = await readMetricsRecords(source.path) @@ -481,7 +482,7 @@ function createMetricsParser(source: SessionSource, seenKeys: Set): Sess } } -function createDatabaseParser(source: SessionSource, seenKeys: Set): SessionParser { +function createDatabaseParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const basePath = basePathFor(source) @@ -545,7 +546,7 @@ export const quickdesk: Provider = { return discoverSources() }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return source.sourceId === 'sessions-db' || basename(source.path) === 'sessions.db' ? createDatabaseParser(source, seenKeys) : createMetricsParser(source, seenKeys) diff --git a/src/providers/qwen.ts b/src/providers/qwen.ts index 55dbf4d11..ea425dba2 100644 --- a/src/providers/qwen.ts +++ b/src/providers/qwen.ts @@ -6,6 +6,7 @@ import { readSessionFile } from '../fs-utils.js' import { calculateCost } from '../models.js' import { extractBashCommands } from '../bash-utils.js' import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' const toolNameMap: Record = { read_file: 'Read', @@ -75,7 +76,7 @@ function extractTools(parts: QwenPart[]): { tools: string[]; bashCommands: strin return { tools, bashCommands } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const raw = await readSessionFile(source.path) @@ -199,7 +200,7 @@ export function createQwenProvider(overrideDir?: string): Provider { return sources }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/roo-code.ts b/src/providers/roo-code.ts index 591941048..3ee8f019e 100644 --- a/src/providers/roo-code.ts +++ b/src/providers/roo-code.ts @@ -1,5 +1,6 @@ import { discoverClineTasks, createClineParser, clineTaskRoots } from './vscode-cline-parser.js' import type { ProbeRoot, Provider, SessionSource, SessionParser } from './types.js' +import type { DedupSet } from '../session-cache.js' const EXTENSION_ID = 'rooveterinaryinc.roo-cline' @@ -24,7 +25,7 @@ export function createRooCodeProvider(overrideDir?: string | string[]): Provider return discoverClineTasks(EXTENSION_ID, 'roo-code', 'Roo Code', overrideDir) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createClineParser(source, seenKeys, 'roo-code') }, } diff --git a/src/providers/sqlite-session-parser.ts b/src/providers/sqlite-session-parser.ts index d9d6bbfb1..2a0c361a0 100644 --- a/src/providers/sqlite-session-parser.ts +++ b/src/providers/sqlite-session-parser.ts @@ -18,6 +18,7 @@ import type { SessionParser, ParsedProviderCall, } from './types.js' +import type { DedupSet } from '../session-cache.js' type MessageRow = { session_id: string @@ -227,7 +228,7 @@ export type SqliteProviderConfig = { export function createSqliteSessionParser( source: SessionSource, - seenKeys: Set, + seenKeys: DedupSet, config: SqliteProviderConfig, ): SessionParser { return { diff --git a/src/providers/types.ts b/src/providers/types.ts index aae70a391..2e270276b 100644 --- a/src/providers/types.ts +++ b/src/providers/types.ts @@ -1,4 +1,5 @@ import type { DateRange, ToolCall } from '../types.js' +import type { DedupSet } from '../session-cache.js' export type SessionSource = { path: string @@ -129,7 +130,7 @@ export type Provider = { // Report once per excluded session, independently of deduplicated warnings. // The callback belongs to this scan, avoiding stale/shared diagnostic counts. discoverSessions(onSkippedVersion?: (version: number) => void): Promise - createSessionParser(source: SessionSource, seenKeys: Set, dateRange?: DateRange): SessionParser + createSessionParser(source: SessionSource, seenKeys: DedupSet, dateRange?: DateRange): SessionParser // The exact directories/dbs discoverSessions() scans, resolved the same way. // Optional: providers that implement it let `codeburn doctor` show and // existence-check the probed paths even when zero sessions are found (so diff --git a/src/providers/vercel-gateway.ts b/src/providers/vercel-gateway.ts index 9b8a8106c..955ffb8fe 100644 --- a/src/providers/vercel-gateway.ts +++ b/src/providers/vercel-gateway.ts @@ -1,6 +1,7 @@ import { getShortModelName } from '../models.js' import type { DateRange } from '../types.js' import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' import { fetchWithTimeout } from '../fetch-utils.js' const REPORT_URL = 'https://ai-gateway.vercel.sh/v1/report' @@ -73,7 +74,7 @@ export async function fetchVercelGatewayReport( function createParser( source: SessionSource, - seenKeys: Set, + seenKeys: DedupSet, dateRange?: DateRange, ): SessionParser { return { @@ -144,7 +145,7 @@ export const vercelGateway: Provider = { createSessionParser( source: SessionSource, - seenKeys: Set, + seenKeys: DedupSet, dateRange?: DateRange, ): SessionParser { return createParser(source, seenKeys, dateRange) diff --git a/src/providers/vscode-cline-parser.ts b/src/providers/vscode-cline-parser.ts index 0b8b65ef5..81d4aeefd 100644 --- a/src/providers/vscode-cline-parser.ts +++ b/src/providers/vscode-cline-parser.ts @@ -4,6 +4,7 @@ import { homedir } from 'os' import { calculateCost } from '../models.js' import type { SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { DedupSet } from '../session-cache.js' type UiMessage = { type?: string @@ -132,7 +133,7 @@ function workspaceToProject(workspace: string): string { return basename(workspace) || workspace } -export function createClineParser(source: SessionSource, seenKeys: Set, providerName: string, fallbackModel = 'cline-auto'): SessionParser { +export function createClineParser(source: SessionSource, seenKeys: DedupSet, providerName: string, fallbackModel = 'cline-auto'): SessionParser { return { async *parse(): AsyncGenerator { const taskDir = source.path diff --git a/src/providers/warp.ts b/src/providers/warp.ts index d02e02f57..01f9c76d3 100644 --- a/src/providers/warp.ts +++ b/src/providers/warp.ts @@ -6,6 +6,7 @@ import { calculateCost, getShortModelName } from '../models.js' import { blobToText, getSqliteLoadError, isBlockedDatabaseError, isSqliteAvailable, openDatabase, type SqliteDatabase } from '../sqlite.js' import { estimateTokensFromChars } from '../token-estimate.js' import type { ProbeRoot, ParsedProviderCall, Provider, SessionParser, SessionSource } from './types.js' +import type { DedupSet } from '../session-cache.js' import { safeNumber } from '../parser.js' const WARP_GROUP_CONTAINER = '2BBY89MBSN.dev.warp' @@ -312,7 +313,7 @@ function validateSchema(db: SqliteDatabase): boolean { } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { if (!isSqliteAvailable()) { @@ -497,7 +498,7 @@ export function createWarpProvider(dbPathOverride?: string): Provider { return sessions }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/zcode.ts b/src/providers/zcode.ts index a273178db..8d8215189 100644 --- a/src/providers/zcode.ts +++ b/src/providers/zcode.ts @@ -4,6 +4,7 @@ import { homedir } from 'os' import { calculateCost } from '../models.js' import { isSqliteAvailable, getSqliteLoadError, openDatabase, type SqliteDatabase } from '../sqlite.js' import type { Provider, SessionSource, SessionParser, ParsedProviderCall, ProbeRoot } from './types.js' +import type { DedupSet } from '../session-cache.js' /// ZCode (CLI v0.14.x) records usage in a single SQLite database at /// ~/.zcode/cli/db/db.sqlite. We read it because the other on-disk sources are @@ -86,7 +87,7 @@ function discover(dbPath: string): SessionSource[] { } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { if (!isSqliteAvailable()) { @@ -222,7 +223,7 @@ export function createZcodeProvider(dbPathOverride?: string): Provider { return discover(dbPath) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/zed.ts b/src/providers/zed.ts index 753912752..946e6728a 100644 --- a/src/providers/zed.ts +++ b/src/providers/zed.ts @@ -6,6 +6,7 @@ import zlib from 'zlib' import { calculateCost } from '../models.js' import { getSqliteLoadError, isSqliteAvailable, openDatabase, type SqliteDatabase } from '../sqlite.js' import type { ParsedProviderCall, ProbeRoot, Provider, SessionParser, SessionSource } from './types.js' +import type { DedupSet } from '../session-cache.js' // Zed's built-in agent stores one row per thread in a single SQLite database; // the `data` blob is zstd-compressed JSON carrying `request_token_usage` @@ -99,7 +100,7 @@ function buildCall(opts: { } } -function parseThreads(db: SqliteDatabase, seenKeys: Set): ParsedProviderCall[] { +function parseThreads(db: SqliteDatabase, seenKeys: DedupSet): ParsedProviderCall[] { const calls: ParsedProviderCall[] = [] let skipped = 0 @@ -170,7 +171,7 @@ function parseThreads(db: SqliteDatabase, seenKeys: Set): ParsedProvider return calls } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { if (!isSqliteAvailable()) { @@ -224,7 +225,7 @@ export function createZedProvider(dbPathOverride?: string): Provider { return [{ path: dbPath, project: 'zed', provider: 'zed' }] }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/providers/zerostack.ts b/src/providers/zerostack.ts index 6c0aa9942..79edccf1d 100644 --- a/src/providers/zerostack.ts +++ b/src/providers/zerostack.ts @@ -5,6 +5,7 @@ import { homedir, platform } from 'os' import { readSessionFile } from '../fs-utils.js' import { calculateCost, getShortModelName } from '../models.js' import type { Provider, SessionSource, SessionParser, ParsedProviderCall, ProbeRoot } from './types.js' +import type { DedupSet } from '../session-cache.js' // zerostack (https://github.com/gi-dellav/zerostack) is a minimal Rust coding // agent. Each session is a single JSON file under /zerostack/sessions/. @@ -72,7 +73,7 @@ async function readSession(path: string): Promise { } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function createParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return { async *parse(): AsyncGenerator { const session = await readSession(source.path) @@ -157,7 +158,7 @@ export function createZerostackProvider(sessionsDir?: string): Provider { return sources }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + createSessionParser(source: SessionSource, seenKeys: DedupSet): SessionParser { return createParser(source, seenKeys) }, } diff --git a/src/session-cache.ts b/src/session-cache.ts index 8a310e2e9..a146da8bd 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -7,6 +7,7 @@ import { getCodeburnCacheDir } from './cache-dir.js' import { flatString, flattenJsonStrings } from './content-utils.js' import { acquireCacheRefreshLock, releaseOwnedRefreshLocksForExit } from './cache-refresh-lock.js' import type { ToolCall } from './types.js' +import { rpcConversationBareKey } from './providers/antigravity-keys.js' import { shardNeedsStreaming, streamShardArrayField, streamShardEntries } from './shard-stream.js' // ── Types ────────────────────────────────────────────────────────────── @@ -148,13 +149,21 @@ export type RangeFilteredMeta = { /// `turns[0].calls[0].project` of the FULL pre-filter list (for orphan /// identity fallbacks that read past the retained turns). firstTurnProject?: string - /// Deduplication keys of the dropped turns, in file walk order. Added to - /// the shared dedup sets wherever the serve loops open the file. Exact if - /// and only if no key appears in both a kept and a dropped turn of this - /// file — the decoder keeps the whole file (unflagged) on any such - /// overlap, so a retained turn can never be suppressed by a dropped later - /// duplicate (or vice versa) differently than the full walk would. + /// Deduplication digests of the dropped turns, in file walk order (see + /// digestDedupKey). Added to the shared dedup sets wherever the serve loops + /// open the file. Exact if and only if no key appears in both a kept and a + /// dropped turn of this file — the decoder keeps the whole file (unflagged) + /// on any such overlap, so a retained turn can never be suppressed by a + /// dropped later duplicate (or vice versa) differently than the full walk + /// would. Digests (not raw keys): raw dropped identities dominate retained + /// heap at corpus scale, and every set/compare in the pipeline hashes both + /// sides uniformly, so membership is exact up to a 256-bit collision bound. droppedKeys: string[] + /// Bare RPC conversation keys (`antigravity:{cid}`, tiny: per conversation, + /// not per call) for consumers that match by conversation rather than by + /// call (the Antigravity RPC prefix check cannot match digests). Seeded + /// through the normal hashed path alongside droppedKeys. + conversations?: string[] /// Branch/PR state carried into the first kept turn, walked from the dropped /// Exact on append-only transcripts, where kept turns always form a suffix; /// files whose dropped turns also carry branch/PR state between kept turns @@ -630,10 +639,15 @@ export function fileFirstTurnProject(file: CachedFile): string | undefined { /// walk adds an out-of-range turn's keys at the same file position. Within a /// file the decoder keeps the record whole on any kept/dropped key overlap, /// so no reordering hazard remains. No-op for unfiltered files. -export function seedDroppedKeys(seen: Set, file: CachedFile): void { +export function seedDroppedKeys(seen: DedupSet, file: CachedFile): void { const marker = file.rangeFiltered if (!marker) return - for (const key of marker.droppedKeys) seen.add(key) + for (const digest of marker.droppedKeys) seen.addHashed(digest) + // Bare conversation keys hash through the normal path exactly like live + // keys do, so prefix-match consumers (Antigravity RPC) keep working. + if (marker.conversations) { + for (const bare of marker.conversations) seen.add(bare) + } } // Save bookkeeping, held beside the cache rather than on it so it never lands in @@ -1237,12 +1251,76 @@ export async function loadShardMemoized(dir: string, name: string): Promise() + add(key: string): this { + this.digests.add(digestDedupKey(key)) + // Pair every RPC-form key with its bare conversation key (see + // rpcConversationBareKey): prefix-match consumers cannot match digests, + // so the bare form rides along wherever the full key goes — live inserts, + // generic cache seeding, and clones alike. + const bare = rpcConversationBareKey(key) + if (bare !== null) this.digests.add(digestDedupKey(bare)) + return this + } + has(key: string): boolean { + return this.digests.has(digestDedupKey(key)) + } + get size(): number { + return this.digests.size + } + /// Stored digests, for seeding another set via addHashed. + /// Mirrors Set.prototype.values so call sites read naturally. + values(): Iterable { + return this.digests + } + /// Insert a pre-hashed digest (marker seeding). Never pass raw keys here. + addHashed(digest: string): this { + this.digests.add(digest) + return this + } + /// Membership test for a pre-hashed digest (worker results carry digests + /// across the thread boundary; hashing them again would never match). + hasHashed(digest: string): boolean { + return this.digests.has(digest) + } +} + type SliceItem = { keep: boolean turn: CachedTurn | null - /// Dedup keys of this turn's calls, for the overlap check and the marker. + /// Digests (see digestDedupKey) of this turn's calls, for the overlap check + /// and the marker. keys: string[] + /// Bare RPC conversation keys referenced by this turn's calls (usually + /// empty; Antigravity RPC keys only). Carried into the marker so consumers + /// that match by conversation survive digesting; tiny (per conversation, + /// not per call). + conversations: string[] gitBranch?: string prRefs?: string[] month: string | null @@ -1258,16 +1336,22 @@ function turnNewestCallMs(turn: CachedTurn): number { } function droppedTurnKeys(turn: CachedTurn, into: string[]): void { - for (const call of turn.calls) into.push(call.deduplicationKey) + for (const call of turn.calls) into.push(digestDedupKey(call.deduplicationKey)) } function sliceItemFor(turn: CachedTurn, keep: boolean): SliceItem { const keys: string[] = [] droppedTurnKeys(turn, keys) + const conversations: string[] = [] + for (const call of turn.calls) { + const bare = rpcConversationBareKey(call.deduplicationKey) + if (bare !== null && !conversations.includes(bare)) conversations.push(bare) + } return { keep, turn: keep ? turn : null, keys, + conversations, gitBranch: turn.gitBranch, prRefs: turn.prRefs, month: monthKey(turn.timestamp), @@ -1286,6 +1370,7 @@ type SliceAccumulator = { kept: CachedTurn[] droppedKeys: string[] keptKeys: Set + conversations: string[] total: number keptCount: number firstKept: number @@ -1303,6 +1388,7 @@ function createSliceAccumulator(): SliceAccumulator { kept: [], droppedKeys: [], keptKeys: new Set(), + conversations: [], total: 0, keptCount: 0, firstKept: -1, @@ -1315,6 +1401,9 @@ function createSliceAccumulator(): SliceAccumulator { anyBranch: false, } } +function pushAccBare(acc: SliceAccumulator, bare: string | null): void { + if (bare !== null && !acc.conversations.includes(bare)) acc.conversations.push(bare) +} /// Fold one live turn with zero per-turn allocation beyond the key strings /// themselves: keys stream straight into the shared arrays, scalars update @@ -1325,7 +1414,8 @@ function foldLiveTurn(acc: SliceAccumulator, turn: CachedTurn, keep: boolean): v const index = acc.total++ acc.kept.push(turn) for (const call of turn.calls) { - acc.keptKeys.add(call.deduplicationKey) + acc.keptKeys.add(digestDedupKey(call.deduplicationKey)) + pushAccBare(acc, rpcConversationBareKey(call.deduplicationKey)) } acc.keptCount++ if (!acc.seenKept) { @@ -1340,7 +1430,8 @@ function foldLiveTurn(acc: SliceAccumulator, turn: CachedTurn, keep: boolean): v } acc.total++ for (const call of turn.calls) { - acc.droppedKeys.push(call.deduplicationKey) + acc.droppedKeys.push(digestDedupKey(call.deduplicationKey)) + pushAccBare(acc, rpcConversationBareKey(call.deduplicationKey)) } if (!acc.seenKept) { if (turn.gitBranch) { @@ -1361,6 +1452,7 @@ function foldItem(acc: SliceAccumulator, item: SliceItem): void { const index = acc.total++ acc.kept.push(item.turn!) for (const key of item.keys) acc.keptKeys.add(key) + for (const bare of item.conversations) pushAccBare(acc, bare) acc.keptCount++ if (!acc.seenKept) { acc.seenKept = true @@ -1373,6 +1465,7 @@ function foldItem(acc: SliceAccumulator, item: SliceItem): void { } acc.total++ for (const key of item.keys) acc.droppedKeys.push(key) + for (const bare of item.conversations) pushAccBare(acc, bare) if (!acc.seenKept) { if (item.gitBranch) { acc.prefixHadBranch = true @@ -1415,6 +1508,7 @@ function finishSlice( newestCallMs: newestMs, ...(firstProject !== undefined ? { firstTurnProject: firstProject } : {}), droppedKeys: acc.droppedKeys, + ...(acc.conversations.length > 0 ? { conversations: acc.conversations } : {}), // Carry needs a kept block to carry INTO: with zero kept turns the // old prefix loop ran zero times (firstKept === -1), so the fields // stay absent. droppedHadBranch still reports via anyBranch. diff --git a/tests/parse-workers.test.ts b/tests/parse-workers.test.ts index 33e7f2705..6c36796ed 100644 --- a/tests/parse-workers.test.ts +++ b/tests/parse-workers.test.ts @@ -10,6 +10,7 @@ import { decideParseWorkers, ParseWorkerPool, parseFilesInOrder, type ClaudeWork import { clearSessionCache, parseAllSessions, parseClaudeFileFull } from '../src/parser.js' import { parseCodexFileFull, type CodexFullParse } from '../src/providers/codex.js' import type { SessionSource } from '../src/providers/types.js' +import { DedupSet } from '../src/session-cache.js' // Two full cold CLI parses of a multi-hundred-file corpus, plus in-process parses // that spawn real threads. @@ -467,12 +468,13 @@ describe('ParseWorkerPool', () => { const afterClose = await pool.submit({ kind: 'codex', source: codexSource }) expect(afterClose.ok).toBe(false) - const seen = new Set() + const seen = new DedupSet() const serial = await parseCodexFileFull(codexSource, seen) if (!fromWorker.ok || !fromWorker.parsed) throw new Error('expected a parsed result') const { keys, path, ...worker } = fromWorker.parsed expect(keys.length).toBeGreaterThan(0) - expect(new Set(keys)).toEqual(seen) + // Both sides carry digests now (the worker posts DedupSet.values()). + expect(new Set(keys)).toEqual(new Set(seen.values())) // Echoed back so the parent can assert the positional worker/file pairing. expect(path).toBe(codexPath) expect(worker).toEqual(JSON.parse(JSON.stringify(serial))) diff --git a/tests/providers/antigravity.test.ts b/tests/providers/antigravity.test.ts index 98294cafb..fca40e674 100644 --- a/tests/providers/antigravity.test.ts +++ b/tests/providers/antigravity.test.ts @@ -5,6 +5,7 @@ import { createRequire } from 'node:module' import { describe, expect, it } from 'vitest' import { isSqliteAvailable } from '../../src/sqlite.js' +import { DedupSet } from '../../src/session-cache.js' import { antigravityAppDataDirFromSourcePath, antigravityCascadeIdFromPath, @@ -393,7 +394,7 @@ describe('antigravity provider helpers', () => { path: getAntigravityStatusLineEventsPath(), project: 'antigravity-cli', provider: 'antigravity', - }, new Set(['antigravity:rpc-covered-conversation:0'])) + }, new DedupSet().add('antigravity:rpc-covered-conversation:0')) const calls = [] for await (const call of parser.parse()) calls.push(call) @@ -404,6 +405,40 @@ describe('antigravity provider helpers', () => { } }) + it('skips statusLine fallback under digests via paired bare keys', async () => { + const dir = await mkdtemp(join(tmpdir(), 'codeburn-antigravity-statusline-rpc-dedup-')) + process.env['CODEBURN_CACHE_DIR'] = dir + try { + expect(await recordAntigravityStatusLinePayload({ + conversation_id: 'rpc-covered-conversation', + session_id: 'session-1', + model: 'Gemini 3.5 Flash (High)', + context_window: { + current_usage: { + input_tokens: 1000, + output_tokens: 100, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + }, + })).toBe(true) + // Production sets hold digests: the RPC key pairs its bare form on + // insert, so the conversation check resolves without prefix scanning. + const seen = new DedupSet() + seen.add('antigravity:rpc-covered-conversation:0') + const parser = createAntigravityProvider().createSessionParser({ + path: getAntigravityStatusLineEventsPath(), + project: 'antigravity-cli', + provider: 'antigravity', + }, seen) + const calls = [] + for await (const call of parser.parse()) calls.push(call) + expect(calls).toEqual([]) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) + it('skips singleton statusLine snapshots and deltas monotonic usage', async () => { const dir = await mkdtemp(join(tmpdir(), 'codeburn-antigravity-statusline-runs-')) process.env['CODEBURN_CACHE_DIR'] = dir diff --git a/tests/session-cache-range-filter.test.ts b/tests/session-cache-range-filter.test.ts index e4ea74351..e645b6a41 100644 --- a/tests/session-cache-range-filter.test.ts +++ b/tests/session-cache-range-filter.test.ts @@ -37,6 +37,8 @@ import { clearLoadCacheMemo, clearShardMemo, computeEnvFingerprint, + digestDedupKey, + DedupSet, loadCache, loadShardFiltered, markCacheDirty, @@ -202,14 +204,14 @@ describe('range-filtered shard load', () => { // June-only file: no turns retained, keys + scalars carried. expect(files['/live/june.jsonl']!.turns).toEqual([]) - expect(files['/live/june.jsonl']!.rangeFiltered?.droppedKeys).toEqual(['june-1']) + expect(files['/live/june.jsonl']!.rangeFiltered?.droppedKeys).toEqual([digestDedupKey('june-1')]) expect(files['/live/june.jsonl']!.fingerprint).toEqual({ dev: 1, ino: 2, mtimeMs: 3, sizeBytes: 4 }) // Span file: prefix carry captured, straddling turn kept WHOLE (both // calls), July-14-only dropped. const span = files['/live/span.jsonl']! expect(span.turns.map(t => t.calls.map(c => c.deduplicationKey))).toEqual([['span-14', 'span-15']]) - expect(span.rangeFiltered?.droppedKeys).toEqual(['juneb-1', 'july14-only']) + expect(span.rangeFiltered?.droppedKeys).toEqual([digestDedupKey('juneb-1'), digestDedupKey('july14-only')]) expect(span.rangeFiltered?.carryBranch).toBe('main') expect(span.rangeFiltered?.carryPrRefs).toEqual(['https://github.com/o/r/pull/1']) expect(span.rangeFiltered?.droppedHadBranch).toBe(true) @@ -217,7 +219,7 @@ describe('range-filtered shard load', () => { // July file: same-month decoys dropped in walk order, query day kept. const july = files['/live/july.jsonl']! expect(july.turns.map(t => t.calls.map(c => c.deduplicationKey))).toEqual([['day15-1']]) - expect(july.rangeFiltered?.droppedKeys).toEqual(['july1-1', 'july16-1', 'july31-1']) + expect(july.rangeFiltered?.droppedKeys).toEqual([digestDedupKey('july1-1'), digestDedupKey('july16-1'), digestDedupKey('july31-1')]) }) it('keeps PR-linked, key-overlap and interleaved files whole', async () => { @@ -261,19 +263,19 @@ describe('range-filtered shard load', () => { rangeFiltered: { span: { bucket: '2026-07', until: '2026-07' }, newestCallMs: 1, - droppedKeys: ['shared-x'], + droppedKeys: [digestDedupKey('shared-x')], }, }) const fileB = cachedFile({ turns: [turnAt('2026-07-15T10:00:00Z', 'shared-x')] }) // Walk order A then B (the serve loops must seed when they open each // file): B's kept turn suppresses exactly as the full walk would, where // A's out-of-range turn would have added the key at A's position. - const seen = new Set() + const seen = new DedupSet() seedDroppedKeys(seen, fileA) const suppressed = fileB.turns[0]!.calls.some(c => seen.has(c.deduplicationKey)) expect(suppressed).toBe(true) // Reversed: B first counts (nothing seeded yet) — order is load-bearing. - const seen2 = new Set() + const seen2 = new DedupSet() const counted = !fileB.turns[0]!.calls.some(c => seen2.has(c.deduplicationKey)) expect(counted).toBe(true) }) @@ -626,9 +628,9 @@ describe('range-filtered load at record scale', () => { expect(file.turns[0]!.calls[0]!.deduplicationKey).toBe('kept-1') const marker = file.rangeFiltered! expect(marker.droppedKeys.length).toBe(29999) - expect(marker.droppedKeys[0]).toBe('june-0') - expect(marker.droppedKeys[15000]).toBe('july1-0') - expect(marker.droppedKeys[29998]).toBe('july1-14998') + expect(marker.droppedKeys[0]).toBe(digestDedupKey('june-0')) + expect(marker.droppedKeys[15000]).toBe(digestDedupKey('july1-0')) + expect(marker.droppedKeys[29998]).toBe(digestDedupKey('july1-14998')) expect(marker.span).toEqual({ bucket: '2026-06', until: '2026-07' }) expect(marker.newestCallMs).toBe(new Date('2026-07-15T12:00:00Z').getTime()) expect(marker).not.toHaveProperty('carryBranch') @@ -650,7 +652,7 @@ describe('range-filtered load at record scale', () => { const file = files['/live/old.jsonl']! expect(file.turns).toEqual([]) const marker = file.rangeFiltered! - expect(marker.droppedKeys).toEqual(['june-0', 'june-1']) + expect(marker.droppedKeys).toEqual([digestDedupKey('june-0'), digestDedupKey('june-1')]) expect(marker).not.toHaveProperty('carryBranch') expect(marker).not.toHaveProperty('carryPrRefs') expect(marker.droppedHadBranch).toBe(true) @@ -673,3 +675,44 @@ describe('range-filtered load at record scale', () => { }) }) +describe('dedup digests', () => { + it('hashes deterministically with avalanche and 32-char shape', async () => { + const a = digestDedupKey('codex:june-1') + expect(a).toBe(digestDedupKey('codex:june-1')) + // Full SHA-256 in single-byte encoding: 32 chars, not necessarily hex. + expect(a).toHaveLength(32) + expect(digestDedupKey('codex:june-2')).not.toBe(a) + }) + + it('pairs RPC bare keys on insert and matches them exactly', async () => { + const seen = new DedupSet() + seen.add('antigravity:cid-1:resp-9') + // Bare form resolves without prefix scanning. + expect(seen.has('antigravity:cid-1')).toBe(true) + // Statusline shapes pair nothing extra. + const before = seen.size + seen.add('antigravity-statusline:cid-1:0:sig') + expect(seen.size).toBe(before + 1) + expect(seen.has('antigravity-statusline:cid-1:0:sig')).toBe(true) + }) + + it('seeds bare conversation keys from marked files', async () => { + // Marker conversations (raw bare keys) hash through the normal path at + // seed time, exactly like live inserts pair them — so a conversation + // cached (and dropped) in a previous run still suppresses its statusline + // twin without prefix scanning digests. + const file = cachedFile({ + turns: [], + rangeFiltered: { + span: { bucket: '2026-07', until: '2026-07' }, + newestCallMs: 1, + droppedKeys: [digestDedupKey('antigravity:cid-9:resp-1')], + conversations: ['antigravity:cid-9'], + }, + }) + const seen = new DedupSet() + seedDroppedKeys(seen, file) + expect(seen.has('antigravity:cid-9:resp-1')).toBe(true) + expect(seen.has('antigravity:cid-9')).toBe(true) + }) +}) \ No newline at end of file From 152009124f0bd51370e7cfccdf38ecdd0eb6ff81 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sat, 19 Sep 2026 22:09:42 -0400 Subject: [PATCH 07/13] fix: parse bare-today headline in aggregate mode overview -p today (all-provider, unfiltered) strips per-call payloads the headline never reads (tool sequences, dedup keys, per-call paths, name arrays; shell commands extracted first for PR launch matching) per provider right after parsing, so peak tracks one provider instead of the corpus. Denylist strip, single narrow-shape gate in the durable builder, memo bypassed both ways. Byte-identical output proven by lite-parity suite. --- CHANGELOG.md | 1 + src/main.ts | 6 +- src/parser.ts | 160 +++++++++++++++++++++++---- src/types.ts | 5 + src/usage-aggregator.ts | 24 +++- tests/overview-lite-parity.test.ts | 140 +++++++++++++++++++++++ tests/parser-aggregate-strip.test.ts | 131 ++++++++++++++++++++++ 7 files changed, 439 insertions(+), 28 deletions(-) create mode 100644 tests/overview-lite-parity.test.ts create mode 100644 tests/parser-aggregate-strip.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d3e4a4db..05aa09ef7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - **Ranged commands decode cache shards incrementally instead of assembling whole multi-hundred-megabyte files.** Session month shards now decode one turn at a time and keep only in-range turns (with the exact same kept/dropped/carry contract as before, verified by the range-filter suite, including a 30k-turn record test); shard loads run serially and the pre-lock snapshot is released before the canonical reload instead of overlapping it. All retained strings are detached from tokenizer buffers (sliced views pinned whole input chunks: ~1.2GB unexplained heap on this corpus). Shards at or under 256MB skip the streaming walk and decode with plain JSON.parse through the same per-record projection (byte-identical results, pinned by the size-gate parity suite): 166MB of shards in 333ms instead of 15-20s, while the walk still bounds the shard past V8's max string length. Out-of-range turns contribute their raw dedup keys to cross-file suppression markers, and provider-scoped queries skip unrelated sections. - **The codex result cache streams instead of whole-parsing.** The single result file decodes entry by entry (a top-level decode assembled hundreds of megabytes first); discovery labels come from calls-free metadata without loading the calls map; publishes merge dirty entries over the published bytes instead of rewriting from memory; single-flight loads are shared across concurrent readers and retry when a publish lands mid-decode, with a global clear epoch so in-flight loads cannot repopulate a cleared memo. - **Cross-file dedup keys are full SHA-256 digests instead of raw strings.** Dropped-key markers and shared dedup sets store 32-character digests (~140MB of raw key characters on the reporter corpus); every insert and lookup hashes uniformly, and a collision would suppress rather than fabricate a call. The set is a plain class (not a Set subclass) so unadapted consumers fail at compile time; Antigravity RPC conversations ride along as paired bare keys since prefix matching cannot work on digests. +- **The bare-today headline parses in aggregate mode.** `overview -p today` (all-provider, unfiltered) now strips per-call payloads the headline never reads (tool sequences, dedup keys, per-call paths, name arrays; shell commands are extracted first for PR launch matching) per provider right after parsing, so peak tracks one provider instead of the corpus. The strip is a denylist, so future fields stay populated. Totals, days, models, categories, tools, and the rendered text are byte-identical to the full parse (proven by the lite-parity suite). The shared parse memo is bypassed in both directions so stripped graphs never mix with full-session consumers. ### Fixed (desktop) - **The Models table shows every model that ran, including the ones under a cent.** The CLI's `models` command defaults `minCost` to $0.01, and the desktop bridge passed neither `--min-cost` nor `--unpriced`, so the table silently dropped every row below a cent — which by construction excluded every unpriced row too (a `0 >= 0.01` filter), leaving #1443's dimming and add-alias affordances unreachable in the shipped app. `codeburn:getModels` now passes `--min-cost 0` (and the demo bridge mirrors it), so sub-cent and unpriced rows arrive and render with their existing dim treatment; on a real lifetime corpus that recovers 10 rows and 2,160 calls the default filter hid (43 → 53 rows, verified in both themes). A row priced between $0.00 and $0.01 renders as "$0.00" without dimming — it is genuinely priced, just below the display floor. Fixes #1465. diff --git a/src/main.ts b/src/main.ts index 37c8f738f..e8d6ccf03 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1002,7 +1002,11 @@ program const { range, label } = customRange ? { range: customRange, label: formatDateRangeLabel(opts.from, opts.to) } : getDateRange(period!) - const durable = await buildDurablePeriod({ range, label }, { provider: opts.provider, project: opts.project, exclude: opts.exclude }) + // Aggregate mode is requested unconditionally here; buildDurablePeriod + // is the single gate that engages it only for the narrow shape needing + // no session payloads downstream (today-only, all-provider, no project + // filters or day selection). Every other shape parses full sessions. + const durable = await buildDurablePeriod({ range, label }, { provider: opts.provider, project: opts.project, exclude: opts.exclude, stripForAggregate: true }) await reportUnmatchedProjectPatterns(durable.knownProjects, opts.project, opts.exclude) const projects = durable.liveProjects const config = await readConfig() diff --git a/src/parser.ts b/src/parser.ts index cd593b543..b86b5aace 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1713,6 +1713,76 @@ function extractCanonicalCwd(entries: JournalEntry[]): string | undefined { return undefined } +/// Aggregate-mode (lite) projection for parsed calls. Denylist: everything +/// carries over except the payloads the overview path never reads downstream +/// (toolSequence, call spawn ids, bash/skill/mcp/subagent name arrays, +/// deduplicationKey), so a future optional field stays populated on the today +/// headline instead of silently dropping to undefined. Breakdowns that read +/// the dropped arrays (tool/mcp/bash/subagent) are computed in +/// buildSessionSummary BEFORE stripping, so they are unaffected. Same type, +/// fresh objects — the full originals stay eligible for release — plus the +/// extracted shell commands PR launch matching needs. +export function stripCallForAggregate(call: ParsedApiCall): ParsedApiCall { + // Denylist: everything carries over except the payloads the overview path + // never reads downstream. workingDirectory/projectPath ride on some calls + // outside the type (session-level identity covers the reports); they are + // heavy (~18MB serialized here) so they drop explicitly rather than by luck. + const { + toolSequence: _droppedSequence, + spawnToolUseIds: _droppedSpawns, + workingDirectory: _droppedWd, + projectPath: _droppedPp, + ...rest + } = call as ParsedApiCall & { workingDirectory?: unknown; projectPath?: unknown } + const lite: ParsedApiCall = { + ...rest, + bashCommands: [], + skills: [], + mcpTools: [], + subagentTypes: [], + deduplicationKey: '', + } + const commands = extractCallCommands(call) + if (commands.length > 0) lite.commands = commands + return lite +} +/// Shell-command strings from a call for PR launch matching, from the lite +/// extraction when present, else straight from the full toolSequence. +export function extractCallCommands(call: ParsedApiCall): string[] { + if (call.commands) return call.commands.filter(command => command.length > 0) + const commands: string[] = [] + for (const step of call.toolSequence ?? []) { + for (const tool of step) { + if (typeof tool.command === 'string' && tool.command.length > 0) commands.push(tool.command) + } + } + return commands +} + +/// Strip one classified turn for aggregate mode (see stripCallForAggregate). +/// userMessage stays: PR candidate prompts and correction scans read it, and +/// at ~115 bytes average it is not the dominator. subCategory drops (its +/// skill breakdown is precomputed and nothing downstream reads it). +export function stripTurnForAggregate(turn: ClassifiedTurn): ClassifiedTurn { + const { subCategory: _dropped, ...rest } = turn + return { ...rest, assistantCalls: turn.assistantCalls.map(stripCallForAggregate) } +} + +/// Strip every session (and subagent anchor) of every project for aggregate +/// mode, MUTATING the session graphs in place. Sessions are always fresh in +/// lite mode (the shared memo is bypassed, so nothing else aliases them); +/// only the turns arrays are replaced, one session at a time, so the transient +/// stays bounded by a single session instead of doubling a whole provider. +/// Totals and breakdowns are precomputed scalars, so they survive unchanged. +export function stripProjectsForAggregate(projects: ProjectSummary[]): ProjectSummary[] { + for (const p of projects) { + for (const s of p.sessions) s.turns = s.turns.map(stripTurnForAggregate) + if (p.subagentAnchors) { + for (const a of p.subagentAnchors) a.turns = a.turns.map(stripTurnForAggregate) + } + } + return projects +} function buildSessionSummary( sessionId: string, project: string, @@ -4872,7 +4942,7 @@ function normalizedWorkingDirectory(path: string | undefined): string | null { return path.trim().replace(/\\/g, '/').replace(/\/+$/, '').toLowerCase() } -function normalizedPrompt(text: string): string { +export function normalizedPrompt(text: string): string { return text.replace(/\s+/g, ' ').trim() } @@ -4945,9 +5015,8 @@ export function correlateCrossProviderPrSessions(projects: ProjectSummary[]): vo if (turn.prRefs?.length) active = turn.prRefs if (active.length === 0) continue for (const call of turn.assistantCalls) { - const commands = (call.toolSequence ?? []) - .flat() - .map(tool => typeof tool.command === 'string' ? normalizedPrompt(tool.command) : '') + const commands = extractCallCommands(call) + .map(command => normalizedPrompt(command)) .filter(command => command.length > 0) if (commands.length === 0) continue const atMs = Date.parse(call.timestamp || turn.timestamp) @@ -5452,14 +5521,27 @@ function deferToBackgroundFill(path: string, fp: { mtimeMs: number }, cached: un return true } -export function parseAllSessions(dateRange?: DateRange, providerFilter?: string): Promise { - const scoped = singlePassParse(dateRange, providerFilter) - if (scoped) return scoped +export type ParseAllSessionsOptions = { + /// Aggregate mode: strip per-call payloads the overview path never reads + /// (see stripProjectsForAggregate) per provider right after its summaries + /// resolve, so peak tracks one provider instead of the corpus. Totals and + /// breakdowns are unaffected (precomputed scalars); the memo is bypassed + /// in both directions so lite graphs never mix with full ones. + stripForAggregate?: boolean +} + +export function parseAllSessions(dateRange?: DateRange, providerFilter?: string, opts?: ParseAllSessionsOptions): Promise { + // Lite graphs must never hit the shared memo (or the single-pass scope): + // their stripped turns would poison full-session consumers on reuse. + if (!opts?.stripForAggregate) { + const scoped = singlePassParse(dateRange, providerFilter) + if (scoped) return scoped + } // Capture synchronously, before the first await. AsyncLocalStorage keeps all // Codex cache reads, dirty writes, and the final flush on this call-time // directory even if an embedding host changes the process env mid-parse. const codexCacheDir = getCodeburnCacheDir() - return withCodexCacheDirectory(codexCacheDir, () => parseAllSessionsInCacheScope(dateRange, providerFilter)) + return withCodexCacheDirectory(codexCacheDir, () => parseAllSessionsInCacheScope(dateRange, providerFilter, opts)) } function canServeCompleteSnapshot(cache: SessionCache, providerFilter?: string, sinceMs?: number): boolean { @@ -5472,7 +5554,11 @@ function canServeCompleteSnapshot(cache: SessionCache, providerFilter?: string, } export async function isCompleteSessionSnapshotAvailable(dateRange: DateRange, providerFilter?: string): Promise { - const diskCache = await loadCache(monthScopeForRange(dateRange.start, dateRange.end)) + // Same projection as the parse itself: this runs ahead of parseAllSessions + // on the dashboard path, and an unfiltered load here would whole-parse the + // month shards before the bounded parse ever starts. A filtered-negative + // only skips the snapshot fast path, never correctness. + const diskCache = await loadCache(monthScopeForRange(dateRange.start, dateRange.end), cacheLoadOpts(dateRange, providerFilter)) return canServeCompleteSnapshot(diskCache, providerFilter, dateRange.start.getTime()) } @@ -5489,14 +5575,18 @@ function cacheLoadOpts(dateRange: DateRange | undefined, providerFilter?: string } } -async function parseAllSessionsInCacheScope(dateRange?: DateRange, providerFilter?: string): Promise { +async function parseAllSessionsInCacheScope(dateRange?: DateRange, providerFilter?: string, opts?: ParseAllSessionsOptions): Promise { + const stripForAggregate = opts?.stripForAggregate === true // Anchor freshness before any config, cache, or session input is read. A // watched-root event that lands while this parse is in flight must remain // newer than the resulting memo instead of being blessed retroactively. const parseStartedAt = Date.now() const claudeDiscoveryRoots = await getClaudeConfigDirs() const key = cacheKey(dateRange, providerFilter, claudeDiscoveryRoots) - const cached = sessionCache.get(key) + // Lite graphs bypass the shared memo in both directions (see + // ParseAllSessionsOptions): a stripped result must never be served to, or + // burst-reused by, a full-session consumer. + const cached = stripForAggregate ? undefined : sessionCache.get(key) if (cached) { const age = Date.now() - cached.createdAt const coveredRange = cached.startMs !== undefined && cached.endMs !== undefined @@ -5521,7 +5611,7 @@ async function parseAllSessionsInCacheScope(dateRange?: DateRange, providerFilte // The signature is the key minus the range: what must match for a burst // reuse (provider, config env, proxy hash) regardless of the now-anchor. const burstSig = cacheKey(undefined, providerFilter, claudeDiscoveryRoots) - if (dateRange) { + if (dateRange && !stripForAggregate) { const reused = burstReuse(dateRange, burstSig) if (reused) return reused } @@ -5555,10 +5645,10 @@ async function parseAllSessionsInCacheScope(dateRange?: DateRange, providerFilte // doubt it proceeds unlocked. if (!isCacheComplete(diskCache, providerFilter, rangeStartMs)) { const hydration = await beginColdHydration(true) - if (hydration.waited) diskCache = await loadCache(loadScope) + if (hydration.waited) diskCache = await loadCache(loadScope, cacheLoadOpts(dateRange, providerFilter)) const isCold = !isCacheComplete(diskCache, providerFilter, rangeStartMs) try { - return await runParse(key, diskCache, dateRange, providerFilter, { isCold, burstSig, parseStartedAt }) + return await runParse(key, diskCache, dateRange, providerFilter, { isCold, burstSig, parseStartedAt, stripForAggregate }) } finally { await hydration.release() } @@ -5570,13 +5660,18 @@ async function parseAllSessionsInCacheScope(dateRange?: DateRange, providerFilte snapshotOnly: true, burstSig, parseStartedAt, + stripForAggregate, }) } // A complete cache refresh is a strict read/reconcile/parse/save transaction. // Keep the snapshot loaded before acquisition: timeout/unavailable paths serve // exactly this complete snapshot and never mutate or invalidate the holder. - const priorSnapshot = diskCache + // `let`: the canonical reloads below release this (and the identical + // `diskCache` binding) BEFORE decoding again. Awaiting the reload while + // either still references the first load keeps two whole filtered caches + // alive at once, which no per-shard bound can prevent. + let priorSnapshot: SessionCache | undefined = diskCache // Heartbeat the WAIT too, not just the parse behind it. This is the one place // a healthy process is deliberately idle for a long stretch, and the desktop // and menubar watchdogs read silence as a dead child - which is how a waiter @@ -5594,20 +5689,26 @@ async function parseAllSessionsInCacheScope(dateRange?: DateRange, providerFilte process.stderr.write(`codeburn: startup timing refresh-lock=${(performance.now() - refreshWaitStarted).toFixed(1)}ms outcome=${refresh.outcome}\n`) } if (refresh.outcome === 'timed-out' || refresh.outcome === 'unavailable') { - return runParse(key, priorSnapshot, dateRange, providerFilter, { readOnly: true, burstSig, parseStartedAt }) + return runParse(key, priorSnapshot ?? diskCache, dateRange, providerFilter, { readOnly: true, burstSig, parseStartedAt, stripForAggregate }) } if (refresh.outcome === 'completed-by-other') { - return runParse(key, await loadCache(loadScope), dateRange, providerFilter, { readOnly: true, burstSig, parseStartedAt }) + const reloadOpts = cacheLoadOpts(dateRange, providerFilter) + priorSnapshot = undefined + diskCache = emptyCache() + return runParse(key, await loadCache(loadScope, reloadOpts), dateRange, providerFilter, { readOnly: true, burstSig, parseStartedAt, stripForAggregate }) } try { // Reload only after ownership is canonical; this closes the lost-update // window between the pre-gate read and the holder's completed publication. - diskCache = await loadCache(loadScope) - return await runParse(key, diskCache, dateRange, providerFilter, { refreshLock: refresh.handle, burstSig, parseStartedAt }) + const reloadOpts = cacheLoadOpts(dateRange, providerFilter) + priorSnapshot = undefined + diskCache = emptyCache() + diskCache = await loadCache(loadScope, reloadOpts) + return await runParse(key, diskCache, dateRange, providerFilter, { refreshLock: refresh.handle, burstSig, parseStartedAt, stripForAggregate }) } catch (err) { if (!(err instanceof RefreshFenceLostError) && !(err instanceof RefreshPublicationUnavailableError)) throw err - return runParse(key, await loadCache(loadScope), dateRange, providerFilter, { readOnly: true, burstSig, parseStartedAt }) + return runParse(key, await loadCache(loadScope, cacheLoadOpts(dateRange, providerFilter)), dateRange, providerFilter, { readOnly: true, burstSig, parseStartedAt, stripForAggregate }) } finally { await refresh.handle.release() } @@ -5623,6 +5724,9 @@ type RunParseOptions = { refreshLock?: RefreshLockHandle burstSig: string parseStartedAt: number + /// Aggregate mode (see ParseAllSessionsOptions): strip per provider and + /// skip the shared memo. Read in runParseInner; never stored. + stripForAggregate?: boolean } /** Thin wrapper so every runParse call site heartbeats for its whole duration, @@ -5650,6 +5754,10 @@ async function runParseInner( options: RunParseOptions, ): Promise { const { isCold = false, readOnly = false, snapshotOnly = false, refreshLock } = options + const stripForAggregate = options.stripForAggregate === true + if (stripForAggregate && process.env['CODEBURN_VERBOSE'] === '1') { + process.stderr.write('codeburn: aggregate strip engaged (lite summaries)\n') + } const timingStarted = performance.now() let timingPrevious = timingStarted const traceTiming = (stage: string, extra = ''): void => { @@ -5726,6 +5834,10 @@ async function runParseInner( if (claudeInScope) { try { claudeProjects = await scanProjectDirs(claudeDirs, seenMsgIds, diskCache, dateRange, saveProgress, readOnly) + // Aggregate mode: strip this provider now so peak tracks one provider + // instead of the corpus (see ParseAllSessionsOptions). Breakdowns and + // totals are precomputed; downstream reads only lite fields. + if (stripForAggregate) claudeProjects = stripProjectsForAggregate(claudeProjects) if (claudeSources.length > 0) emitScanProgress({ kind: 'provider', provider: 'claude', state: 'done', files: claudeSources.length }) } catch (err) { if (!isPermissionError(err)) throw err @@ -5741,7 +5853,8 @@ async function runParseInner( try { const projects = await parseProviderSources(providerName, sources, seenKeys, diskCache, dateRange, saveProgress, readOnly) emitScanProgress({ kind: 'provider', provider: providerName, state: 'done', files: sources.length }) - otherProjects.push(...projects) + if (stripForAggregate) otherProjects.push(...stripProjectsForAggregate(projects)) + else otherProjects.push(...projects) } catch (err) { // A permission-locked provider skips-and-continues; any other error is a // real bug and still aborts (per-file/DB-lock cases are handled deeper). @@ -5771,7 +5884,8 @@ async function runParseInner( // round-trip for every unprocessed provider in the disk cache. if (!snapshotOnly && !section.durable && !DURABLE_PROVIDER_NAMES.has(providerName)) continue const projects = await parseProviderSources(providerName, [], seenKeys, diskCache, dateRange, saveProgress, readOnly) - otherProjects.push(...projects) + if (stripForAggregate) otherProjects.push(...stripProjectsForAggregate(projects)) + else otherProjects.push(...projects) } // The full scan reached the end: this cache is now complete. Mark it and @@ -5869,7 +5983,7 @@ async function runParseInner( // A snapshot is an explicitly stale, source-unvalidated view. Publishing it // into either exact-key or burst reuse can suppress the reconciliation that // the mounted dashboard starts immediately afterward for the full TTL. - if (!snapshotOnly) { + if (!snapshotOnly && !stripForAggregate) { if (dateRange) setCachePutMeta({ startMs: dateRange.start.getTime(), endMs: dateRange.end.getTime(), sig: options.burstSig }) cachePut(key, result, options.parseStartedAt) } diff --git a/src/types.ts b/src/types.ts index 5bc4fb403..b60e8123c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -123,6 +123,11 @@ export type ParsedApiCall = { deduplicationKey: string cacheCreationOneHourTokens?: number toolSequence?: ToolCall[][] + /// Aggregate-mode (lite) extraction: the shell-command strings from + /// toolSequence, pulled out before the sequence itself is stripped (see + /// stripCallForAggregate). Lets PR launch matching run on lite calls; + /// absent on full parses, which read toolSequence directly. + commands?: string[] /// Claude Code: `tool_use` ids of the `Agent`/`Task` subagent-spawn blocks in /// this call's assistant message. Transient (built at parse time, aggregated /// into the turn's `spawnToolUseIds`); never cached per-call. diff --git a/src/usage-aggregator.ts b/src/usage-aggregator.ts index 8a2391cae..949fb7ed4 100644 --- a/src/usage-aggregator.ts +++ b/src/usage-aggregator.ts @@ -296,6 +296,12 @@ export type AggregateOpts = { /// true. The desktop app never renders it, so it passes `--no-timeline` to /// skip the buildGranularHistory pass on every menubar poll. timeline?: boolean + /// Aggregate-mode request (see ParseAllSessionsOptions): strip per-call + /// payloads per provider right after parsing. This builder is the single + /// gate: it engages only for the narrow shape needing no session payloads + /// downstream (today-only, all-provider, no project filters or day + /// selection) and forces full sessions otherwise. + stripForAggregate?: boolean } type ConfigOption = { id: string; label: string; path: string } @@ -780,6 +786,16 @@ export async function buildDurablePeriod(periodInfo: PeriodInfo, opts: Aggregate const rangeStartStr = toDateString(periodInfo.range.start) const rangeEndStr = toDateString(periodInfo.range.end) const isTodayOnly = rangeStartStr === todayStr && rangeEndStr === todayStr + // Aggregate mode engages only on the narrow shape that needs no session + // payloads downstream (see AggregateOpts): anything else forces full + // sessions even if a caller asked for lite. + const strip = opts.stripForAggregate === true + && isTodayOnly + && pf === 'all' + && (opts.project?.length ?? 0) === 0 + && (opts.exclude?.length ?? 0) === 0 + && daysSelection === null + const parseOpts = strip ? { stripForAggregate: true } : undefined // The shared daily cache is hydrated only by an all-provider request. A // provider tab must not turn into a hidden all-provider scan on a cold or @@ -796,12 +812,12 @@ export async function buildDurablePeriod(periodInfo: PeriodInfo, opts: Aggregate let scanRange: DateRange if (pf === 'all') { if (isTodayOnly) { - const raw = fp(await parseAllSessions(todayRange, 'all')) + const raw = fp(await parseAllSessions(todayRange, 'all', parseOpts)) liveProjects = raw scanRange = todayRange todayAllDays = aggregateProjectsIntoDays(raw).filter(d => d.date === todayStr) } else { - const raw = fp(await parseAllSessions(periodInfo.range, 'all')) + const raw = fp(await parseAllSessions(periodInfo.range, 'all', parseOpts)) liveProjects = daysSelection ? filterProjectsByDays(raw, daysSelection.days) : raw scanRange = periodInfo.range // A period that reaches today contains today's turns already, so derive the @@ -813,14 +829,14 @@ export async function buildDurablePeriod(periodInfo: PeriodInfo, opts: Aggregate // JSON daily turn count while the per-call cost/calls still bucket to today. todayAllDays = rangeEndStr >= todayStr ? aggregateProjectsIntoDays(filterProjectsByDays(raw, new Set([todayStr]))).filter(d => d.date === todayStr) - : aggregateProjectsIntoDays(fp(await parseAllSessions(todayRange, 'all'))).filter(d => d.date === todayStr) + : aggregateProjectsIntoDays(fp(await parseAllSessions(todayRange, 'all', parseOpts))).filter(d => d.date === todayStr) } } else { // Provider-filtered: one provider-scoped parse feeds both today's union // slice and the detail/enrichment fields. Scanning every unrelated provider // here made a first hover deserialize the entire multi-gigabyte cache even // though the returned payload contains only `pf`. - const rawProv = fp(await parseAllSessions(isTodayOnly ? todayRange : periodInfo.range, pf)) + const rawProv = fp(await parseAllSessions(isTodayOnly ? todayRange : periodInfo.range, pf, parseOpts)) freshProviderDays = aggregateProjectsIntoDays(rawProv) todayAllDays = rangeEndStr >= todayStr ? aggregateProjectsIntoDays(filterProjectsByDays(rawProv, new Set([todayStr]))).filter(d => d.date === todayStr) diff --git a/tests/overview-lite-parity.test.ts b/tests/overview-lite-parity.test.ts new file mode 100644 index 000000000..bc949350e --- /dev/null +++ b/tests/overview-lite-parity.test.ts @@ -0,0 +1,140 @@ +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { mkdir, rm, writeFile } from 'fs/promises' +import { existsSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' + +import { getDateRange } from '../src/cli-date.js' +import { loadPricing } from '../src/models.js' +import { aggregateProjectsIntoDays } from '../src/day-aggregator.js' +import { buildPeriodData } from '../src/usage-aggregator.js' +import { clearSessionCache, parseAllSessions } from '../src/parser.js' +import { renderOverview } from '../src/overview.js' +import type { DateRange } from '../src/types.js' + +// Lite-vs-full parity for aggregate mode (overview -p today): a stripped +// parse must report exactly what the full parse reports — same totals, days, +// models, categories, tools, and rendered text — while carrying no per-call +// payloads. Isolated env (mirrors cli-durable-totals) so host sessions leak in. + +const ROOT = join(tmpdir(), `codeburn-lite-parity-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`) +const ENV_KEYS = ['HOME', 'CODEBURN_CACHE_DIR', 'CLAUDE_CONFIG_DIR', 'CLAUDE_CONFIG_DIRS', 'CODEX_HOME', 'USERPROFILE', 'KIMI_CODE_HOME', 'CODEBURN_DESKTOP_SESSIONS_DIR'] as const +let savedEnv: Record + +const CODEX_ROOT = vi.hoisted(() => { + const root = `${process.env['TMPDIR'] || '/tmp'}/codeburn-lite-parity-codex-${process.pid}-${Date.now()}` + process.env['CODEX_HOME'] = `${root}/codex` + return root +}) + +function minutesAgo(now: Date, midnight: number, m: number): string { + return new Date(Math.max(midnight, now.getTime() - m * 60_000)).toISOString() +} + +/** One live-today Claude session: user prompt, edit turn, bash turn, chat turn. */ +async function seedTodaySession(): Promise { + const projectDir = join(ROOT, 'home', '.claude', 'projects', 'p') + await mkdir(projectDir, { recursive: true }) + const now = new Date() + const midnight = new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime() + const user = (t: string, text: string): string => JSON.stringify({ + type: 'user', sessionId: 's-lite', timestamp: t, message: { role: 'user', content: text }, + }) + const assistant = (id: string, t: string, model: string, content: unknown[], usage: unknown): string => JSON.stringify({ + type: 'assistant', sessionId: 's-lite', timestamp: t, + message: { id, type: 'message', role: 'assistant', model, content, usage }, + }) + const usage = { input_tokens: 1000, output_tokens: 100 } + const lines = [ + user(minutesAgo(now, midnight, 50), 'add retry logic to the uploader'), + assistant('m1', minutesAgo(now, midnight, 40), 'claude-sonnet-4-5', [ + { type: 'text', text: 'editing' }, + { type: 'tool_use', id: 'tu-1', name: 'Edit', input: { file_path: '/tmp/x', old_string: 'a', new_string: 'b' } }, + ], usage), + user(minutesAgo(now, midnight, 30), 'check git status now'), + assistant('m2', minutesAgo(now, midnight, 20), 'claude-sonnet-4-5', [ + { type: 'text', text: 'running' }, + { type: 'tool_use', id: 'tu-2', name: 'Bash', input: { command: 'git status --short' } }, + ], usage), + user(minutesAgo(now, midnight, 10), 'thanks'), + assistant('m3', minutesAgo(now, midnight, 5), 'claude-sonnet-4-5', [ + { type: 'text', text: 'done' }, + ], usage), + ] + await writeFile(join(projectDir, 's-lite.jsonl'), lines.join('\n') + '\n', 'utf-8') +} + +beforeAll(async () => { + await loadPricing() +}) + +beforeEach(async () => { + savedEnv = Object.fromEntries(ENV_KEYS.map(k => [k, process.env[k]])) + await mkdir(join(ROOT, 'home', '.claude'), { recursive: true }) + await mkdir(join(ROOT, 'cache'), { recursive: true }) + await mkdir(join(ROOT, 'no-desktop-sessions'), { recursive: true }) + await mkdir(join(ROOT, 'no-kimi-home'), { recursive: true }) + process.env['HOME'] = join(ROOT, 'home') + process.env['CODEBURN_CACHE_DIR'] = join(ROOT, 'cache') + process.env['CLAUDE_CONFIG_DIR'] = join(ROOT, 'home', '.claude') + delete process.env['CLAUDE_CONFIG_DIRS'] + delete process.env['CODEX_HOME'] + process.env['USERPROFILE'] = join(ROOT, 'home') + process.env['KIMI_CODE_HOME'] = join(ROOT, 'no-kimi-home') + process.env['CODEBURN_DESKTOP_SESSIONS_DIR'] = join(ROOT, 'no-desktop-sessions') + await rm(CODEX_ROOT, { recursive: true, force: true }) + clearSessionCache() +}) + +afterEach(async () => { + clearSessionCache() + for (const k of ENV_KEYS) { + if (savedEnv[k] === undefined) delete process.env[k] + else process.env[k] = savedEnv[k] + } + if (existsSync(ROOT)) await rm(ROOT, { recursive: true, force: true }) + await rm(CODEX_ROOT, { recursive: true, force: true }) +}) + +describe('overview lite parity', () => { + it('reports byte-identical output with stripped payloads', async () => { + await seedTodaySession() + const range: DateRange = getDateRange('today').range + + clearSessionCache() + const full = await parseAllSessions(range, 'all') + const fullText = renderOverview(full, { label: 'Today', color: false }) + const fullData = buildPeriodData('lite-parity', full) + const fullDays = aggregateProjectsIntoDays(full) + + clearSessionCache() + const lite = await parseAllSessions(range, 'all', { stripForAggregate: true }) + const liteText = renderOverview(lite, { label: 'Today', color: false }) + const liteData = buildPeriodData('lite-parity', lite) + const liteDays = aggregateProjectsIntoDays(lite) + + expect(liteText).toBe(fullText) + expect(liteData.cost).toBe(fullData.cost) + expect(liteData.calls).toBe(fullData.calls) + expect(liteData.inputTokens).toBe(fullData.inputTokens) + expect(liteData.outputTokens).toBe(fullData.outputTokens) + expect(liteDays).toEqual(fullDays) + expect(liteData.models).toEqual(fullData.models) + expect(liteData.categories).toEqual(fullData.categories) + + // Structural: payloads stripped, billing/PR inputs kept. + const liteCalls = lite.flatMap(p => p.sessions).flatMap(s => s.turns).flatMap(t => t.assistantCalls) + const fullCalls = full.flatMap(p => p.sessions).flatMap(s => s.turns).flatMap(t => t.assistantCalls) + expect(liteCalls.length).toBeGreaterThan(0) + expect(liteCalls.length).toBe(fullCalls.length) + for (const c of liteCalls) { + expect(c.toolSequence).toBeUndefined() + expect(c.deduplicationKey).toBe('') + } + const fullCommands = fullCalls.map(c => (c.toolSequence ?? []).flat().map(t => t.command).filter(Boolean)).filter(a => a.length > 0) + const liteCommands = liteCalls.map(c => c.commands ?? []).filter(a => a.length > 0) + expect(liteCommands).toEqual(fullCommands) + const liteTexts = lite.flatMap(p => p.sessions).flatMap(s => s.turns).map(t => t.userMessage) + expect(liteTexts).toContain('add retry logic to the uploader') + }) +}) diff --git a/tests/parser-aggregate-strip.test.ts b/tests/parser-aggregate-strip.test.ts new file mode 100644 index 000000000..0aad5fe3c --- /dev/null +++ b/tests/parser-aggregate-strip.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it } from 'vitest' + +import { extractCallCommands, normalizedPrompt, stripCallForAggregate, stripProjectsForAggregate } from '../src/parser.js' +import type { ParsedApiCall } from '../src/types.js' +import type { ProjectSummary } from '../src/types.js' + +function fullCall(): ParsedApiCall { + return { + provider: 'omp', + model: 'gemini-3.8-flash', + usage: { + inputTokens: 100, + outputTokens: 50, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 10, + cachedInputTokens: 0, + reasoningTokens: 5, + webSearchRequests: 0, + }, + costUSD: 0.01, + tools: ['Edit'], + mcpTools: ['mcp__srv__tool'], + skills: ['s'], + subagentTypes: ['Explore'], + hasAgentSpawn: false, + hasPlanMode: false, + speed: 'standard', + timestamp: '2026-09-19T10:00:00Z', + bashCommands: ['ls'], + deduplicationKey: 'omp:abc123', + workingDirectory: '/repo', + projectPath: '/repo', + toolSequence: [[{ tool: 'Bash', command: 'git status' }]], + savingsUSD: 0.002, + route: 'r1', + supplementaryAccounting: undefined, + } as unknown as ParsedApiCall +} + +describe('stripCallForAggregate', () => { + it('keeps billing scalars and drops payloads', () => { + const lite = stripCallForAggregate(fullCall()) + expect(lite.provider).toBe('omp') + expect(lite.model).toBe('gemini-3.8-flash') + expect(lite.costUSD).toBe(0.01) + expect(lite.usage.inputTokens).toBe(100) + expect(lite.tools).toEqual(['Edit']) + expect(lite.route).toBe('r1') + expect(lite.toolSequence).toBeUndefined() + expect(lite.deduplicationKey).toBe('') + expect(lite.bashCommands).toEqual([]) + expect(lite.skills).toEqual([]) + expect(lite.mcpTools).toEqual([]) + expect(lite.subagentTypes).toEqual([]) + expect(lite).not.toHaveProperty('workingDirectory') + expect(lite).not.toHaveProperty('projectPath') + }) + + it('extracts shell commands for PR launch matching', () => { + expect(stripCallForAggregate(fullCall()).commands).toEqual(['git status']) + expect(extractCallCommands(fullCall())).toEqual(['git status']) + }) + it('matches the legacy toolSequence derivation exactly', () => { + const call = { + ...fullCall(), + toolSequence: [[{ tool: 'Bash', command: ' git status ' }, { tool: 'Edit' }, { tool: 'Bash', command: '' }, { tool: 'Bash', command: ' ' }]], + } as unknown as ParsedApiCall + const legacy = (call.toolSequence ?? []) + .flat() + .map(tool => typeof tool.command === 'string' ? normalizedPrompt(tool.command) : '') + .filter(command => command.length > 0) + const viaHelper = extractCallCommands(call) + .map(command => normalizedPrompt(command)) + .filter(command => command.length > 0) + expect(viaHelper).toEqual(legacy) + expect(viaHelper).toEqual(['git status']) + }) +}) + +describe('stripProjectsForAggregate', () => { + it('preserves totals and breakdowns while stripping turns', () => { + const projects = [{ + project: 'repo', + projectPath: '/repo', + sessions: [{ + sessionId: 's1', + project: 'repo', + firstTimestamp: '2026-09-19T10:00:00Z', + lastTimestamp: '2026-09-19T10:01:00Z', + totalCostUSD: 0.01, + totalSavingsUSD: 0, + totalInputTokens: 100, + totalOutputTokens: 50, + totalReasoningTokens: 5, + totalCacheReadTokens: 10, + totalCacheWriteTokens: 0, + apiCalls: 1, + turns: [{ + userMessage: 'do it', + assistantCalls: [fullCall()], + timestamp: '2026-09-19T10:00:00Z', + sessionId: 's1', + category: 'coding', + retries: 0, + hasEdits: false, + }], + modelBreakdown: {}, + toolBreakdown: { Edit: { calls: 1 } }, + mcpBreakdown: {}, + bashBreakdown: {}, + categoryBreakdown: {}, + skillBreakdown: {}, + subagentBreakdown: {}, + }], + totalCostUSD: 0.01, + totalSavingsUSD: 0, + totalApiCalls: 1, + totalProxiedCostUSD: 0, + }] as unknown as ProjectSummary[] + const [lite] = stripProjectsForAggregate(projects) + expect(lite!.totalCostUSD).toBe(0.01) + expect(lite!.sessions[0]!.toolBreakdown).toEqual({ Edit: { calls: 1 } }) + expect(lite!.sessions[0]!.turns[0]!.userMessage).toBe('do it') + expect(lite!.sessions[0]!.turns[0]!.assistantCalls[0]!.commands).toEqual(['git status']) + expect(lite!.sessions[0]!.turns[0]!.assistantCalls[0]!.toolSequence).toBeUndefined() + // In-place contract: same graphs, payloads stripped (single owner, no + // provider-wide copy transient). + expect(lite).toBe(projects[0]) + expect(projects[0]!.sessions[0]!.turns[0]!.assistantCalls[0]!.toolSequence).toBeUndefined() + }) +}) From 5fc09d60b6396c88c85966ba2d28ca43548a2500 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sat, 19 Sep 2026 22:20:14 -0400 Subject: [PATCH 08/13] fix: strip runtime call extras in aggregate denylist project/prLinks ride on calls outside the ParsedApiCall type and survive a pure type-keyed denylist; they are heavy, so they drop explicitly. Covered by extended strip assertions. --- src/parser.ts | 10 ++++++---- tests/parser-aggregate-strip.test.ts | 4 ++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index b86b5aace..2ad88f379 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1724,16 +1724,18 @@ function extractCanonicalCwd(entries: JournalEntry[]): string | undefined { /// extracted shell commands PR launch matching needs. export function stripCallForAggregate(call: ParsedApiCall): ParsedApiCall { // Denylist: everything carries over except the payloads the overview path - // never reads downstream. workingDirectory/projectPath ride on some calls - // outside the type (session-level identity covers the reports); they are - // heavy (~18MB serialized here) so they drop explicitly rather than by luck. + // never reads downstream. workingDirectory/projectPath/project/prLinks ride + // on some calls outside the type (session/project identity covers the + // reports); they are heavy, so they drop explicitly rather than by luck. const { toolSequence: _droppedSequence, spawnToolUseIds: _droppedSpawns, workingDirectory: _droppedWd, projectPath: _droppedPp, + project: _droppedProject, + prLinks: _droppedPrLinks, ...rest - } = call as ParsedApiCall & { workingDirectory?: unknown; projectPath?: unknown } + } = call as ParsedApiCall & { workingDirectory?: unknown; projectPath?: unknown; project?: unknown; prLinks?: unknown } const lite: ParsedApiCall = { ...rest, bashCommands: [], diff --git a/tests/parser-aggregate-strip.test.ts b/tests/parser-aggregate-strip.test.ts index 0aad5fe3c..408d61ba9 100644 --- a/tests/parser-aggregate-strip.test.ts +++ b/tests/parser-aggregate-strip.test.ts @@ -30,6 +30,8 @@ function fullCall(): ParsedApiCall { deduplicationKey: 'omp:abc123', workingDirectory: '/repo', projectPath: '/repo', + project: 'repo', + prLinks: ['https://github.com/o/r/pull/1'], toolSequence: [[{ tool: 'Bash', command: 'git status' }]], savingsUSD: 0.002, route: 'r1', @@ -54,6 +56,8 @@ describe('stripCallForAggregate', () => { expect(lite.subagentTypes).toEqual([]) expect(lite).not.toHaveProperty('workingDirectory') expect(lite).not.toHaveProperty('projectPath') + expect(lite).not.toHaveProperty('project') + expect(lite).not.toHaveProperty('prLinks') }) it('extracts shell commands for PR launch matching', () => { From 8b5fe5918271d7901b933066ed62b46f5a0efe13 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sat, 19 Sep 2026 22:47:02 -0400 Subject: [PATCH 09/13] fix: drop tool arrays and intern repeated strings in aggregate strip Tool-name strings like 'Bash' repeat ~1.5M times as distinct objects (~90MB heap); the tool breakdown is precomputed before stripping and the edit-time median is unrendered here. Model, provider, speed, category, and project names intern to one shared ref per provider batch. --- src/parser.ts | 55 +++++++++++++++++++--------- tests/parser-aggregate-strip.test.ts | 3 +- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 2ad88f379..750a05094 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1722,11 +1722,15 @@ function extractCanonicalCwd(entries: JournalEntry[]): string | undefined { /// buildSessionSummary BEFORE stripping, so they are unaffected. Same type, /// fresh objects — the full originals stay eligible for release — plus the /// extracted shell commands PR launch matching needs. -export function stripCallForAggregate(call: ParsedApiCall): ParsedApiCall { +export function stripCallForAggregate(call: ParsedApiCall, intern?: Map): ParsedApiCall { // Denylist: everything carries over except the payloads the overview path // never reads downstream. workingDirectory/projectPath/project/prLinks ride // on some calls outside the type (session/project identity covers the // reports); they are heavy, so they drop explicitly rather than by luck. + // tools drops entirely (not just emptied): tool-name strings like 'Bash' + // repeat ~1.5M times as distinct objects (~90MB heap on this corpus); the + // tool breakdown is precomputed in buildSessionSummary and the edit-time + // median is not rendered on this path. const { toolSequence: _droppedSequence, spawnToolUseIds: _droppedSpawns, @@ -1734,10 +1738,19 @@ export function stripCallForAggregate(call: ParsedApiCall): ParsedApiCall { projectPath: _droppedPp, project: _droppedProject, prLinks: _droppedPrLinks, + tools: _droppedTools, ...rest } = call as ParsedApiCall & { workingDirectory?: unknown; projectPath?: unknown; project?: unknown; prLinks?: unknown } const lite: ParsedApiCall = { ...rest, + tools: [], + // Intern low-cardinality strings (model/provider/speed repeat ~1M times + // as distinct slice objects): one shared ref each instead of per-call + // copies. Timestamps/userMessages are unique per call and must NOT enter + // the map (it would grow without bound and pin everything). + model: intern ? interned(intern, call.model) : call.model, + provider: intern ? interned(intern, call.provider) : call.provider, + speed: intern ? (interned(intern, call.speed) as 'standard' | 'fast') : call.speed, bashCommands: [], skills: [], mcpTools: [], @@ -1748,6 +1761,12 @@ export function stripCallForAggregate(call: ParsedApiCall): ParsedApiCall { if (commands.length > 0) lite.commands = commands return lite } +function interned(map: Map, s: string): string { + const hit = map.get(s) + if (hit !== undefined) return hit + map.set(s, s) + return s +} /// Shell-command strings from a call for PR launch matching, from the lite /// extraction when present, else straight from the full toolSequence. export function extractCallCommands(call: ParsedApiCall): string[] { @@ -1760,27 +1779,29 @@ export function extractCallCommands(call: ParsedApiCall): string[] { } return commands } - -/// Strip one classified turn for aggregate mode (see stripCallForAggregate). -/// userMessage stays: PR candidate prompts and correction scans read it, and -/// at ~115 bytes average it is not the dominator. subCategory drops (its -/// skill breakdown is precomputed and nothing downstream reads it). -export function stripTurnForAggregate(turn: ClassifiedTurn): ClassifiedTurn { +export function stripTurnForAggregate(turn: ClassifiedTurn, intern?: Map): ClassifiedTurn { const { subCategory: _dropped, ...rest } = turn - return { ...rest, assistantCalls: turn.assistantCalls.map(stripCallForAggregate) } + // Categories are 14 fixed labels repeated per turn: share one ref each. + const category = (intern ? interned(intern, turn.category) : turn.category) as ClassifiedTurn['category'] + return { ...rest, category, assistantCalls: turn.assistantCalls.map(c => stripCallForAggregate(c, intern)) } } - -/// Strip every session (and subagent anchor) of every project for aggregate -/// mode, MUTATING the session graphs in place. Sessions are always fresh in -/// lite mode (the shared memo is bypassed, so nothing else aliases them); -/// only the turns arrays are replaced, one session at a time, so the transient -/// stays bounded by a single session instead of doubling a whole provider. -/// Totals and breakdowns are precomputed scalars, so they survive unchanged. export function stripProjectsForAggregate(projects: ProjectSummary[]): ProjectSummary[] { + // One interner per provider batch: model/provider/category/project names + // repeat within a provider's sessions, so a batch-scoped map captures + // ~all duplication without growing across providers. + const intern = new Map() for (const p of projects) { - for (const s of p.sessions) s.turns = s.turns.map(stripTurnForAggregate) + p.project = interned(intern, p.project) + if (p.projectPath) p.projectPath = interned(intern, p.projectPath) + for (const s of p.sessions) { + s.project = interned(intern, s.project) + s.turns = s.turns.map(t => stripTurnForAggregate(t, intern)) + } if (p.subagentAnchors) { - for (const a of p.subagentAnchors) a.turns = a.turns.map(stripTurnForAggregate) + for (const a of p.subagentAnchors) { + a.project = interned(intern, a.project) + a.turns = a.turns.map(t => stripTurnForAggregate(t, intern)) + } } } return projects diff --git a/tests/parser-aggregate-strip.test.ts b/tests/parser-aggregate-strip.test.ts index 408d61ba9..e97377883 100644 --- a/tests/parser-aggregate-strip.test.ts +++ b/tests/parser-aggregate-strip.test.ts @@ -45,8 +45,7 @@ describe('stripCallForAggregate', () => { expect(lite.provider).toBe('omp') expect(lite.model).toBe('gemini-3.8-flash') expect(lite.costUSD).toBe(0.01) - expect(lite.usage.inputTokens).toBe(100) - expect(lite.tools).toEqual(['Edit']) + expect(lite.tools).toEqual([]) expect(lite.route).toBe('r1') expect(lite.toolSequence).toBeUndefined() expect(lite.deduplicationKey).toBe('') From 3b8fcf056e188b7b46698f103446842e97f1bd23 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sat, 19 Sep 2026 22:48:50 -0400 Subject: [PATCH 10/13] fix: ignore heap snapshots --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 268256bf4..b9a780a11 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ docs/superpowers/ # Debug / logs *.log +*.heapsnapshot npm-debug.log* # Cache From 68cc6a3b19fe59da4f85c8413c36b5bdc10a441a Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sat, 19 Sep 2026 22:57:55 -0400 Subject: [PATCH 11/13] fix: extract PR prompt prefixes, drop userMessage in aggregate strip userMessage averages ~115B per turn but totals ~40MB serialized (~100MB heap); launch matching needs only the first qualifying 160-char prefix. Precompute turn.promptPrefix at strip time and drop the full text; the matcher reads the prefix first with the userMessage fallback, so attribution is identical. Correction scans tolerate empty messages (their stats are unrendered here). --- src/parser.ts | 16 ++++++++++++---- src/types.ts | 4 ++++ tests/overview-lite-parity.test.ts | 2 +- tests/parser-aggregate-strip.test.ts | 26 +++++++++++++++++++++++--- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 750a05094..1453656f8 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1783,7 +1783,13 @@ export function stripTurnForAggregate(turn: ClassifiedTurn, intern?: Map stripCallForAggregate(c, intern)) } + // PR launch matching reads the first qualifying prompt prefix, not the + // message: precompute it (bounded, 160 chars) and drop the full text + // (~115B average, ~40MB serialized here). Candidates match identically + // because the prefix is exactly what the matcher slices (see PROMPT_*). + const normalized = normalizedPrompt(turn.userMessage) + const promptPrefix = normalized.length >= PROMPT_MIN ? normalized.slice(0, PROMPT_PREFIX) : undefined + return { ...rest, category, userMessage: '', ...(promptPrefix !== undefined ? { promptPrefix } : {}), assistantCalls: turn.assistantCalls.map(c => stripCallForAggregate(c, intern)) } } export function stripProjectsForAggregate(projects: ProjectSummary[]): ProjectSummary[] { // One interner per provider batch: model/provider/category/project names @@ -4993,6 +4999,10 @@ function assignCorrelatedPrs( * Timestamps only narrow prompt comparisons for performance; they can never * create attribution. Conflicting PR evidence is deliberately left unassigned. */ +/// PR candidate-prompt shape, shared with the aggregate strip (which +/// precomputes turn.promptPrefix so lite sessions match identically). +const PROMPT_PREFIX = 160 +const PROMPT_MIN = 80 export function correlateCrossProviderPrSessions(projects: ProjectSummary[]): void { const sessions = projects.flatMap(p => p.sessions) const linked = sessions.filter(s => s.prLinks?.length) @@ -5048,8 +5058,6 @@ export function correlateCrossProviderPrSessions(projects: ProjectSummary[]): vo } } - const PROMPT_PREFIX = 160 - const PROMPT_MIN = 80 const LAUNCH_WINDOW_MS = 15 * 60 * 1000 // Sorted once so each candidate scans only the launches inside its own // window instead of the whole array. Launch order is not observable: the @@ -5069,7 +5077,7 @@ export function correlateCrossProviderPrSessions(projects: ProjectSummary[]): vo for (const session of candidates) { const provider = summaryProvider(session) const prompt = session.turns - .map(t => normalizedPrompt(t.userMessage)) + .map(t => t.promptPrefix ?? normalizedPrompt(t.userMessage)) .find(text => text.length >= PROMPT_MIN) if (!prompt) continue const prefix = prompt.slice(0, PROMPT_PREFIX) diff --git a/src/types.ts b/src/types.ts index b60e8123c..d3496455e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -87,6 +87,10 @@ export type JournalEntry = { export type ParsedTurn = { userMessage: string + /// Aggregate-mode extraction: first 160 chars of the normalized prompt when + /// it qualifies as a PR candidate (see correlateCrossProviderPrSessions). + /// Lets the lite path drop userMessage while keeping launch matching exact. + promptPrefix?: string assistantCalls: ParsedApiCall[] timestamp: string sessionId: string diff --git a/tests/overview-lite-parity.test.ts b/tests/overview-lite-parity.test.ts index bc949350e..e52e2959b 100644 --- a/tests/overview-lite-parity.test.ts +++ b/tests/overview-lite-parity.test.ts @@ -135,6 +135,6 @@ describe('overview lite parity', () => { const liteCommands = liteCalls.map(c => c.commands ?? []).filter(a => a.length > 0) expect(liteCommands).toEqual(fullCommands) const liteTexts = lite.flatMap(p => p.sessions).flatMap(s => s.turns).map(t => t.userMessage) - expect(liteTexts).toContain('add retry logic to the uploader') + expect(liteTexts).toEqual(full.flatMap(p => p.sessions).flatMap(s => s.turns).map(() => '')) }) }) diff --git a/tests/parser-aggregate-strip.test.ts b/tests/parser-aggregate-strip.test.ts index e97377883..b09858532 100644 --- a/tests/parser-aggregate-strip.test.ts +++ b/tests/parser-aggregate-strip.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest' -import { extractCallCommands, normalizedPrompt, stripCallForAggregate, stripProjectsForAggregate } from '../src/parser.js' -import type { ParsedApiCall } from '../src/types.js' +import { extractCallCommands, normalizedPrompt, stripCallForAggregate, stripProjectsForAggregate, stripTurnForAggregate } from '../src/parser.js' +import type { ClassifiedTurn, ParsedApiCall } from '../src/types.js' import type { ProjectSummary } from '../src/types.js' function fullCall(): ParsedApiCall { @@ -123,7 +123,7 @@ describe('stripProjectsForAggregate', () => { const [lite] = stripProjectsForAggregate(projects) expect(lite!.totalCostUSD).toBe(0.01) expect(lite!.sessions[0]!.toolBreakdown).toEqual({ Edit: { calls: 1 } }) - expect(lite!.sessions[0]!.turns[0]!.userMessage).toBe('do it') + expect(lite!.sessions[0]!.turns[0]!.userMessage).toBe('') expect(lite!.sessions[0]!.turns[0]!.assistantCalls[0]!.commands).toEqual(['git status']) expect(lite!.sessions[0]!.turns[0]!.assistantCalls[0]!.toolSequence).toBeUndefined() // In-place contract: same graphs, payloads stripped (single owner, no @@ -131,4 +131,24 @@ describe('stripProjectsForAggregate', () => { expect(lite).toBe(projects[0]) expect(projects[0]!.sessions[0]!.turns[0]!.assistantCalls[0]!.toolSequence).toBeUndefined() }) + it('precomputes qualifying prompt prefixes for launch matching', () => { + const longText = `please review the pull request thoroughly and carefully ${'x'.repeat(200)}` + const turn = { + userMessage: longText, + assistantCalls: [], + timestamp: '2026-09-19T10:00:00Z', + sessionId: 's9', + category: 'coding', + retries: 0, + hasEdits: false, + } as unknown as ClassifiedTurn + const lite = stripTurnForAggregate(turn) + expect(lite.userMessage).toBe('') + expect(lite.promptPrefix).toBe(longText.replace(/\s+/g, ' ').trim().slice(0, 160)) + expect(lite.promptPrefix!.length).toBe(160) + // Short prompts leave no prefix (matcher skips them either way). + const short = stripTurnForAggregate({ ...turn, userMessage: 'do it' }) + expect(short.promptPrefix).toBeUndefined() + expect(short.userMessage).toBe('') + }) }) From d561b42b0607d07a2f4f9c25dbf2efc3be19f30e Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sat, 19 Sep 2026 23:28:01 -0400 Subject: [PATCH 12/13] fix: strip aggregate sessions at build time, not per provider Per-provider stripping left the provider's full summaries resident through its whole parse (death in early provider parsing at cap). buildSessionSummary now strips each session the moment it resolves under an AsyncLocalStorage-gated per-parse store (safe for concurrent mixed parses), so peak never holds more than one session's full turns. --- src/parser.ts | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 1453656f8..cf337f079 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1,4 +1,5 @@ import { homedir } from 'node:os' +import { AsyncLocalStorage } from 'node:async_hooks' import { existsSync } from 'fs' import { lstat, readFile, readdir, stat } from 'fs/promises' import { createHash } from 'crypto' @@ -1941,7 +1942,7 @@ function buildSessionSummary( } } - return { + const summary: SessionSummary = { sessionId, project, firstTimestamp: firstTs || turns[0]?.timestamp || '', @@ -1966,6 +1967,12 @@ function buildSessionSummary( ...(source ? { source } : {}), ...(mcpInventory && mcpInventory.length > 0 ? { mcpInventory } : {}), } + // Aggregate mode: strip this session now (per-session granularity), while + // its full turns are still hot and releasable. Breakdowns above already + // consumed the payloads. Anonymous (store-less) callers keep full turns. + const agg = aggregateStore.getStore() + if (agg) summary.turns = summary.turns.map(t => stripTurnForAggregate(t, agg.intern)) + return summary } async function parseSessionFile( @@ -5755,8 +5762,9 @@ type RunParseOptions = { refreshLock?: RefreshLockHandle burstSig: string parseStartedAt: number - /// Aggregate mode (see ParseAllSessionsOptions): strip per provider and - /// skip the shared memo. Read in runParseInner; never stored. + /// Aggregate mode (see ParseAllSessionsOptions): strip per session inside + /// buildSessionSummary and skip the shared memo. Read in runParseInner; + /// never stored. stripForAggregate?: boolean } @@ -5771,11 +5779,24 @@ async function runParse( ): Promise { startProgressKeepalive() try { + // Aggregate mode strips per session inside buildSessionSummary; the store + // carries one shared string interner for the whole parse (it holds only + // low-cardinality model/provider/category/project names). + if (options.stripForAggregate) { + return await aggregateStore.run({ intern: new Map() }, () => + runParseInner(key, diskCache, dateRange, providerFilter, options)) + } return await runParseInner(key, diskCache, dateRange, providerFilter, options) } finally { stopProgressKeepalive() } } +/// Per-parse aggregate context: when set, buildSessionSummary strips each +/// session the moment it resolves (per-session granularity, so peak never +/// holds a provider's full turns). AsyncLocalStorage (not a module flag) so +/// concurrent parses — e.g. a lite overview racing a full dashboard refresh +/// in one serve process — cannot cross-contaminate. Set once per runParse. +const aggregateStore = new AsyncLocalStorage<{ intern: Map }>() async function runParseInner( key: string, @@ -5865,10 +5886,6 @@ async function runParseInner( if (claudeInScope) { try { claudeProjects = await scanProjectDirs(claudeDirs, seenMsgIds, diskCache, dateRange, saveProgress, readOnly) - // Aggregate mode: strip this provider now so peak tracks one provider - // instead of the corpus (see ParseAllSessionsOptions). Breakdowns and - // totals are precomputed; downstream reads only lite fields. - if (stripForAggregate) claudeProjects = stripProjectsForAggregate(claudeProjects) if (claudeSources.length > 0) emitScanProgress({ kind: 'provider', provider: 'claude', state: 'done', files: claudeSources.length }) } catch (err) { if (!isPermissionError(err)) throw err @@ -5884,8 +5901,7 @@ async function runParseInner( try { const projects = await parseProviderSources(providerName, sources, seenKeys, diskCache, dateRange, saveProgress, readOnly) emitScanProgress({ kind: 'provider', provider: providerName, state: 'done', files: sources.length }) - if (stripForAggregate) otherProjects.push(...stripProjectsForAggregate(projects)) - else otherProjects.push(...projects) + otherProjects.push(...projects) } catch (err) { // A permission-locked provider skips-and-continues; any other error is a // real bug and still aborts (per-file/DB-lock cases are handled deeper). @@ -5915,8 +5931,7 @@ async function runParseInner( // round-trip for every unprocessed provider in the disk cache. if (!snapshotOnly && !section.durable && !DURABLE_PROVIDER_NAMES.has(providerName)) continue const projects = await parseProviderSources(providerName, [], seenKeys, diskCache, dateRange, saveProgress, readOnly) - if (stripForAggregate) otherProjects.push(...stripProjectsForAggregate(projects)) - else otherProjects.push(...projects) + otherProjects.push(...projects) } // The full scan reached the end: this cache is now complete. Mark it and From 4dc5436f52f8f963ff6ea7e6af386987d5c44144 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sun, 20 Sep 2026 00:04:50 -0400 Subject: [PATCH 13/13] fix: skip lock reload when the envelope nonce is unchanged A complete-cache refresh streamed every in-scope shard twice: once before lock acquisition and once after. The pre-lock snapshot stays referenced until the reload replaces it, so peak held two filtered copies plus a 730MB shard transient and died near 500MB on overview -p today. Every saveCache mints a fresh envelope nonce, so an unchanged nonce proves byte-identical shards and the reload is skipped; any publication, torn read, or missing envelope keeps the reload. --- src/parser.ts | 22 +++- src/session-cache.ts | 19 ++++ .../parser-cache-refresh-reload-skip.test.ts | 100 ++++++++++++++++++ 3 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 tests/parser-cache-refresh-reload-skip.test.ts diff --git a/src/parser.ts b/src/parser.ts index cf337f079..8eb2e8059 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -39,10 +39,12 @@ import { fingerprintFile, isCacheComplete, isCacheDirty, + lastLoadCacheNonce, loadCache, markCacheDirty, markProviderComplete, monthScopeForRange, + readCurrentEnvelopeNonce, reconcileFile, saveCache, seedDroppedKeys, @@ -5668,6 +5670,10 @@ async function parseAllSessionsInCacheScope(dateRange?: DateRange, providerFilte const rangeStartMs = dateRange?.start.getTime() const cacheLoadStarted = performance.now() let diskCache = await loadCache(loadScope, cacheLoadOpts(dateRange, providerFilter)) + // Nonce the pre-lock load actually read (see the reload below): captured + // here because no other loadCache runs between this and lock acquisition + // on the complete-cache path. + const preLockEnvelopeNonce = lastLoadCacheNonce() await cleanupOrphanedTempFiles() if (process.env['CODEBURN_VERBOSE'] === '1') { process.stderr.write(`codeburn: startup timing cache-load=${(performance.now() - cacheLoadStarted).toFixed(1)}ms complete=${isCacheComplete(diskCache, providerFilter, rangeStartMs)}\n`) @@ -5739,10 +5745,20 @@ async function parseAllSessionsInCacheScope(dateRange?: DateRange, providerFilte try { // Reload only after ownership is canonical; this closes the lost-update // window between the pre-gate read and the holder's completed publication. + // Every saveCache mints a fresh envelope nonce, so when the nonce is + // unchanged the shards are byte-identical and re-streaming them only + // doubles peak heap (the pre-lock snapshot stays referenced until the + // reload replaces it) for zero new data — reuse it. Any publication, torn + // read, or missing envelope falls back to the reload. const reloadOpts = cacheLoadOpts(dateRange, providerFilter) - priorSnapshot = undefined - diskCache = emptyCache() - diskCache = await loadCache(loadScope, reloadOpts) + const envelopeNonceNow = await readCurrentEnvelopeNonce() + if (envelopeNonceNow === null || envelopeNonceNow !== preLockEnvelopeNonce) { + priorSnapshot = undefined + diskCache = emptyCache() + diskCache = await loadCache(loadScope, reloadOpts) + } else if (process.env['CODEBURN_VERBOSE'] === '1') { + process.stderr.write('codeburn: startup timing reload=skipped (envelope unchanged)\n') + } return await runParse(key, diskCache, dateRange, providerFilter, { refreshLock: refresh.handle, burstSig, parseStartedAt, stripForAggregate }) } catch (err) { if (!(err instanceof RefreshFenceLostError) && !(err instanceof RefreshPublicationUnavailableError)) throw err diff --git a/src/session-cache.ts b/src/session-cache.ts index a146da8bd..ae3d061d7 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -1073,6 +1073,24 @@ async function adoptNewestPriorCache(): Promise { // stays the canonical one after a refresh. let cacheMemo: { dir: string; nonce: string; scope: string; provider: string; cache: SessionCache } | null = null +/// Nonce of the envelope the most recent loadCache call in this process read +/// (null when the directory held no envelope). Lets the refresh path prove no +/// other process published between its pre-lock load and lock acquisition and +/// skip the re-stream: every saveCache mints a fresh nonce, so an unchanged +/// nonce means byte-identical shards. +let lastLoadedEnvelopeNonce: string | null = null + +export function lastLoadCacheNonce(): string | null { + return lastLoadedEnvelopeNonce +} + +/// Current envelope nonce on disk without parsing any shard (small-file read). +/// Null when the envelope is missing or invalid. +export async function readCurrentEnvelopeNonce(): Promise { + const envelope = await readEnvelope(sessionCacheDir()) + return envelope?.nonce ?? null +} + export function clearLoadCacheMemo(): void { cacheMemo = null clearShardMemo() @@ -1816,6 +1834,7 @@ export async function loadCache(scope?: CacheLoadScope, opts?: LoadCacheOptions) if (process.env['CODEBURN_CACHE_SCOPE'] === 'all') scope = undefined const dir = sessionCacheDir() const envelope = await readEnvelope(dir) + lastLoadedEnvelopeNonce = envelope?.nonce ?? null if (!envelope) return afterMissingShardCache() const scopeKey = scope ? `${scope.fromMonth}..${scope.toMonth}` : 'all' // A filtered load must never reuse a memoized full cache (it would serve diff --git a/tests/parser-cache-refresh-reload-skip.test.ts b/tests/parser-cache-refresh-reload-skip.test.ts new file mode 100644 index 000000000..6a3953eb0 --- /dev/null +++ b/tests/parser-cache-refresh-reload-skip.test.ts @@ -0,0 +1,100 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { mkdir, mkdtemp, rm, writeFile } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' + +vi.mock('../src/cache-refresh-lock.js', () => ({ + acquireCacheRefreshLock: async () => { + // A concurrent publication landing while the lock is acquired must force + // the reload: rewrite the envelope nonce inside acquisition. + if (process.env['CB_RELOAD_SKIP_REWRITE_NONCE'] === '1') { + const { readFile, writeFile } = await import('fs/promises') + const { join } = await import('path') + const envelopePath = join(process.env['CODEBURN_CACHE_DIR']!, 'session-cache.v9', 'envelope.json') + const envelope = JSON.parse(await readFile(envelopePath, 'utf-8')) + envelope.nonce = 'deadbeefdeadbeef' + await writeFile(envelopePath, JSON.stringify(envelope)) + } + return { + outcome: 'acquired' as const, + handle: { release: async () => {} }, + } + }, +})) +import { clearSessionCache, parseAllSessions } from '../src/parser.js' +import type { ProjectSummary } from '../src/types.js' + +let root: string +let sessionPath: string + +function output(projects: ProjectSummary[]): number { + return projects.flatMap(p => p.sessions).flatMap(s => s.turns) + .flatMap(t => t.assistantCalls).reduce((sum, call) => sum + call.usage.outputTokens, 0) +} + +async function writeSession(value: number): Promise { + await writeFile(sessionPath, JSON.stringify({ + type: 'assistant', + sessionId: 'sess', + timestamp: '2026-05-15T10:00:00Z', + cwd: '/tmp/proj', + message: { + id: `msg-${value}`, type: 'message', role: 'assistant', model: 'claude-sonnet-4-5', + content: [], usage: { input_tokens: 100, output_tokens: value }, + }, + }) + '\n') +} + +beforeEach(async () => { + clearSessionCache() + root = await mkdtemp(join(tmpdir(), 'cb-reload-skip-')) + const home = join(root, 'home') + const project = join(home, 'projects', 'proj') + await mkdir(project, { recursive: true }) + sessionPath = join(project, 'sess.jsonl') + process.env['CLAUDE_CONFIG_DIR'] = home + process.env['CODEBURN_CACHE_DIR'] = join(root, 'cache') + process.env['CODEBURN_DESKTOP_SESSIONS_DIR'] = join(home, 'desktop-sessions') + process.env['CODEBURN_VERBOSE'] = '1' +}) + +afterEach(async () => { + delete process.env['CODEBURN_VERBOSE'] + clearSessionCache() + await rm(root, { recursive: true, force: true }) +}) + +describe('parseAllSessions refresh reload', () => { + it('reuses the pre-lock snapshot when the envelope nonce is unchanged', async () => { + await writeSession(50) + expect(output(await parseAllSessions(undefined, 'claude'))).toBe(50) + clearSessionCache() + const writes: string[] = [] + const origWrite = process.stderr.write.bind(process.stderr) + process.stderr.write = ((chunk: unknown) => { writes.push(String(chunk)); return true }) as typeof process.stderr.write + try { + expect(output(await parseAllSessions(undefined, 'claude'))).toBe(50) + } finally { + process.stderr.write = origWrite + } + expect(writes.some(w => w.includes('reload=skipped'))).toBe(true) + }) + + it('reloads when another publication changed the envelope nonce', async () => { + await writeSession(50) + expect(output(await parseAllSessions(undefined, 'claude'))).toBe(50) + + clearSessionCache() + process.env['CB_RELOAD_SKIP_REWRITE_NONCE'] = '1' + const writes: string[] = [] + const origWrite = process.stderr.write.bind(process.stderr) + process.stderr.write = ((chunk: unknown) => { writes.push(String(chunk)); return true }) as typeof process.stderr.write + try { + expect(output(await parseAllSessions(undefined, 'claude'))).toBe(50) + } finally { + process.stderr.write = origWrite + delete process.env['CB_RELOAD_SKIP_REWRITE_NONCE'] + } + expect(writes.some(w => w.includes('reload=skipped'))).toBe(false) + }) +})