From af4e5e15e08bff39f5e1a8150f2e8b0a57735912 Mon Sep 17 00:00:00 2001 From: Louise Lau Date: Thu, 2 Jul 2026 10:17:17 +0800 Subject: [PATCH] =?UTF-8?q?feat(recall):=20receipt-aware=20ranking=20+=20t?= =?UTF-8?q?imeline=20=E2=80=94=20ground-truth-weighted=20recall?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recall_approach now knows which past approaches PROVABLY worked, ranks with that, and shows the evolution on a timeline. Same pipeline everywhere (CLI tool + dashboard panel via the shared renderer). - Episode.verified: stamped true at record time (episodes are written on the objective-success path — sandbox compiled + merged, verify/completion gates passed). Old episodes → unknown. - New recall source kind 'receipt': maintain runs with receipts join the corpus — verified autonomous work IS history worth recalling. opened → verified:true, blocked/failed → false. - rankApproaches verifiedBoost: ✓ proven +0.08, ⛔ blocked/failed −0.04, unknown neutral — applied to the effective rank only; the displayed score stays honest raw relevance. Composes with the existing recency tilt + MMR diversity + stemming. - Visualization: outcome marks on every label (🎯 task ✓ / 🧾 receipt ⛔) + renderTimeline — the last 5 dated approaches, oldest → newest with ○/● glyphs and per-entry outcome marks; omitted when <2 matches carry a date. Appended by renderApproachDiffs, so the dashboard Recall panel and the CLI tool get it with zero extra wiring. - Both gathers (recall_approach tool + dashboard recall.query) feed episodes' verified flag and the maintain-receipt source. --- src/agent/loop.ts | 3 +++ src/cli/dashboard-control.ts | 23 +++++++++++++++-- src/context/approach-diff.ts | 30 ++++++++++++++++++++-- src/context/approach-recall.ts | 16 +++++++++--- src/context/episodic-memory.ts | 3 +++ src/tools/builtin/recall-approach.ts | 32 ++++++++++++++++++++++-- test/approach-diff.test.ts | 37 ++++++++++++++++++++++++++++ test/approach-recall.test.ts | 11 +++++++++ 8 files changed, 146 insertions(+), 9 deletions(-) diff --git a/src/agent/loop.ts b/src/agent/loop.ts index 3c469ca..8a6b843 100644 --- a/src/agent/loop.ts +++ b/src/agent/loop.ts @@ -750,6 +750,9 @@ export class AgentLoop { filesChanged: changedFiles, toolsUsed: [...this.sessionToolNames], toolCalls: this.totalToolCalls, + // This branch IS the objective-success path (sandbox compiled + merged, verify + + // completion gates passed) — recall can boost these over unverified history. + verified: true, }); } catch (e: any) { logger.debug('Episode record skipped', { err: e?.message }); diff --git a/src/cli/dashboard-control.ts b/src/cli/dashboard-control.ts index a592bd7..c66f804 100644 --- a/src/cli/dashboard-control.ts +++ b/src/cli/dashboard-control.ts @@ -113,11 +113,30 @@ export async function dispatchAction(name: string, params: any, cwd: string): Pr const store = getSessionStore(); const worklog = (() => { try { return store.getWorklog(cwd, 100).map((w: any) => ({ kind: 'worklog' as const, text: w.entry, when: String(w.created_at ?? '').slice(0, 10), at: w.created_at, detail: w.kind })); } catch { return []; } })(); const episodes = await (async () => { - try { const { readEpisodes } = await import('../context/episodic-memory.js'); return (await readEpisodes(cwd)).map((e: any) => ({ kind: 'episode' as const, text: `${e.prompt} ${e.summary}`, when: String(e.ts ?? '').slice(0, 10), at: e.ts, files: e.filesChanged, detail: e.summary })); } + try { const { readEpisodes } = await import('../context/episodic-memory.js'); return (await readEpisodes(cwd)).map((e: any) => ({ kind: 'episode' as const, text: `${e.prompt} ${e.summary}`, when: String(e.ts ?? '').slice(0, 10), at: e.ts, files: e.filesChanged, detail: e.summary, verified: e.verified })); } catch { return []; } })(); const facts = (() => { try { return store.getFactsForCwd(cwd, 200).map((f: string) => ({ kind: 'fact' as const, text: f, when: '', detail: 'fact' })); } catch { return []; } })(); - const matches = rankApproaches(q, [...episodes, ...worklog, ...facts], { topK: 4, nowMs: Date.now(), diversity: 0.35 }); + const receipts = await (async () => { + try { + const { getScheduleStore } = await import('../schedule/store.js'); + const { parseMaintainScope } = await import('../schedule/recipes.js'); + const sched = getScheduleStore(); + const out: any[] = []; + for (const s of sched.list().filter((s: any) => s.recipe === 'maintain' && s.cwd === cwd)) { + const scope = parseMaintainScope(s.prompt).scope; + for (const r of sched.recentRuns(s.id, 50)) { + if (!r.receipt) continue; + try { + const rc = JSON.parse(r.receipt); + out.push({ kind: 'receipt' as const, at: r.started_at, when: String(r.started_at ?? '').slice(0, 10), text: `maintain ${scope}: ${rc.summary || rc.reason || rc.status}`, files: Array.isArray(rc.filesChanged) ? rc.filesChanged : [], detail: scope, verified: rc.status === 'opened' ? true : rc.status === 'blocked' || rc.status === 'failed' ? false : undefined }); + } catch { /* skip */ } + } + } + return out; + } catch { return []; } + })(); + const matches = rankApproaches(q, [...episodes, ...worklog, ...facts, ...receipts], { topK: 4, nowMs: Date.now(), diversity: 0.35 }); return { ok: true, message: renderApproachDiffs(q, matches) }; } case 'memory.add': { diff --git a/src/context/approach-diff.ts b/src/context/approach-diff.ts index 98de7ae..26adb9c 100644 --- a/src/context/approach-diff.ts +++ b/src/context/approach-diff.ts @@ -80,7 +80,9 @@ function head(text: string, max = 160): string { } function tag(m: ApproachMatch): string { - return m.kind === 'episode' ? '🎯 task' : m.kind === 'fact' ? '🧠 fact' : `📝 ${m.detail ?? 'worklog'}`; + const base = m.kind === 'episode' ? '🎯 task' : m.kind === 'fact' ? '🧠 fact' : m.kind === 'receipt' ? '🧾 receipt' : `📝 ${m.detail ?? 'worklog'}`; + // Ground-truth outcome, when known: ✓ = gates/receipt proved it worked; ⛔ = it was blocked. + return m.verified === true ? `${base} ✓` : m.verified === false ? `${base} ⛔` : base; } function fileLinks(files: string[] | undefined, refFiles?: Set): string { @@ -89,9 +91,32 @@ function fileLinks(files: string[] | undefined, refFiles?: Set): string return ` files: ${shown.join(', ')}${files.length > 4 ? ` (+${files.length - 4})` : ''}`; } +/** + * A compact chronological timeline of the matched approaches — oldest → newest, so the EVOLUTION + * of how this problem was attacked reads at a glance. Entries without a parseable timestamp are + * skipped; capped at `k` (default 5, per the "last 5 approaches with dates" spec). PURE. + */ +export function renderTimeline(matches: ApproachMatch[], k = 5): string[] { + const dated = matches + .map(m => ({ m, t: m.at ? Date.parse(m.at) : NaN })) + .filter(x => Number.isFinite(x.t)) + .sort((a, b) => a.t - b.t) + .slice(-k); + if (dated.length < 2) return []; + const lines = ['', `Timeline (oldest → newest):`]; + dated.forEach(({ m }, i) => { + const date = m.at!.slice(0, 10); + const glyph = i === dated.length - 1 ? '●' : '○'; + const mark = m.verified === true ? ' ✓' : m.verified === false ? ' ⛔' : ''; + lines.push(` ${glyph} ${date}${mark} ${head(m.text, 76)}`); + }); + return lines; +} + /** * Render top-K matches as a visual comparison: the best approach in full, then each alternative - * diffed against it, then the common core across all. PURE. Falls back to a plain list note when + * diffed against it, then the common core across all — and a chronological timeline so the + * evolution of the approach reads at a glance. PURE. Falls back to a plain list note when * there's only one match (nothing to diff). */ export function renderApproachDiffs(query: string, matches: ApproachMatch[], opts: { topK?: number } = {}): string { @@ -125,5 +150,6 @@ export function renderApproachDiffs(query: string, matches: ApproachMatch[], opt const core = commonCore(top.map(m => m.text)); if (core.length >= 2) lines.push('', `Stable core across all ${top.length}: ${core.join(', ')} — this is how you consistently approach it here.`); } + lines.push(...renderTimeline(top)); return lines.join('\n'); } diff --git a/src/context/approach-recall.ts b/src/context/approach-recall.ts index 82f2ae2..df0c3a5 100644 --- a/src/context/approach-recall.ts +++ b/src/context/approach-recall.ts @@ -34,7 +34,7 @@ export function semanticTokens(text: string): string[] { } export interface ApproachSource { - kind: 'episode' | 'worklog' | 'fact'; + kind: 'episode' | 'worklog' | 'fact' | 'receipt'; /** The searchable text (prompt + summary, or the worklog entry). */ text: string; when: string; @@ -44,6 +44,9 @@ export interface ApproachSource { files?: string[]; /** A short human label (the summary, or the worklog kind). */ detail?: string; + /** Objectively-verified outcome: true = gates/receipt proved it worked; false = it was blocked + * or failed; undefined = unknown (old episodes, facts). Drives the success-weighted ranking. */ + verified?: boolean; } export interface ApproachMatch extends ApproachSource { score: number } @@ -58,6 +61,13 @@ function recencyBoost(at: string | undefined, nowMs: number | undefined): number return Math.max(0, 0.06 * (1 - ageDays / 180)); } +/** Success weighting from ground truth: an approach that PROVABLY worked (gates passed / receipt + * status opened) should edge out an equally-relevant unverified one; one that was blocked/failed + * should rank a notch below. Unknown (old episodes, facts) is neutral. PURE. */ +function verifiedBoost(verified: boolean | undefined): number { + return verified === true ? 0.08 : verified === false ? -0.04 : 0; +} + /** * Rank past approaches against a query by lexical similarity, with a mild recency tilt so the most * RECENT relevant approach surfaces first on close calls. PURE. Returns top-K ≥ minScore. @@ -81,7 +91,7 @@ export function rankApproaches( for (const s of sources) { const tf = termFreq(semanticTokens(s.text)); const score = cosineSim(qv, tf); - if (score >= minScore) scored.push({ m: { ...s, score }, eff: score + recencyBoost(s.at, opts.nowMs), tf }); + if (score >= minScore) scored.push({ m: { ...s, score }, eff: score + recencyBoost(s.at, opts.nowMs) + verifiedBoost(s.verified), tf }); } scored.sort((a, b) => b.eff - a.eff); if (lambda === 0 || scored.length <= 1) return scored.slice(0, topK).map(x => x.m); @@ -109,7 +119,7 @@ export function formatApproaches(query: string, matches: ApproachMatch[]): strin if (matches.length === 0) return `No past work on this project resembles "${query}".`; const lines = [`How you approached similar work before — "${query}":`, '']; for (const m of matches) { - const tag = m.kind === 'episode' ? '🎯 task' : m.kind === 'fact' ? '🧠 fact' : `📝 ${m.detail ?? 'worklog'}`; + const tag = m.kind === 'episode' ? '🎯 task' : m.kind === 'fact' ? '🧠 fact' : m.kind === 'receipt' ? '🧾 receipt' : `📝 ${m.detail ?? 'worklog'}`; const head = m.text.replace(/\s+/g, ' ').trim().slice(0, 140); const files = m.files?.length ? ` (touched: ${m.files.slice(0, 4).join(', ')})` : ''; lines.push(`- [${tag} · ${m.when}] ${head}${files}`); diff --git a/src/context/episodic-memory.ts b/src/context/episodic-memory.ts index 17915de..9b5607a 100644 --- a/src/context/episodic-memory.ts +++ b/src/context/episodic-memory.ts @@ -30,6 +30,9 @@ export interface Episode { /** Tool calls it took to finish — a cleanliness signal (fewer = tidier path). Optional * so episodes recorded before this field still load. */ toolCalls?: number; + /** True when the episode was recorded on the objectively-verified path (sandbox compiled, + * verify + completion gates passed, squash-merged). Absent on older episodes → unknown. */ + verified?: boolean; } export interface EpisodeMatch extends Episode { score: number } diff --git a/src/tools/builtin/recall-approach.ts b/src/tools/builtin/recall-approach.ts index 582187c..8e9a353 100644 --- a/src/tools/builtin/recall-approach.ts +++ b/src/tools/builtin/recall-approach.ts @@ -40,7 +40,7 @@ export class RecallApproachTool extends Tool> { const episodes = await (async () => { try { const { readEpisodes } = await import('../../context/episodic-memory.js'); - return (await readEpisodes(cwd)).map(e => ({ kind: 'episode' as const, text: `${e.prompt} ${e.summary}`, when: relTime(e.ts), at: e.ts, files: e.filesChanged, detail: e.summary })); + return (await readEpisodes(cwd)).map(e => ({ kind: 'episode' as const, text: `${e.prompt} ${e.summary}`, when: relTime(e.ts), at: e.ts, files: e.filesChanged, detail: e.summary, verified: e.verified })); } catch { return []; } })(); @@ -49,7 +49,35 @@ export class RecallApproachTool extends Tool> { catch { return []; } })(); - const matches = rankApproaches(args.query, [...episodes, ...worklog, ...facts], { topK: args.limit ?? 5, nowMs: Date.now(), diversity: 0.35 }); + // Maintain receipts — verified autonomous work IS history worth recalling, with ground-truth + // outcomes: opened = proven success (boosted), blocked = the guardrail declined (down-weighted). + const receipts = await (async () => { + try { + const { getScheduleStore } = await import('../../schedule/store.js'); + const { parseMaintainScope } = await import('../../schedule/recipes.js'); + const sched = getScheduleStore(); + const out: { kind: 'receipt'; text: string; when: string; at: string; files?: string[]; detail?: string; verified?: boolean }[] = []; + for (const s of sched.list().filter((s: any) => s.recipe === 'maintain' && s.cwd === cwd)) { + const scope = parseMaintainScope(s.prompt).scope; + for (const r of sched.recentRuns(s.id, 50)) { + if (!r.receipt) continue; + try { + const rc = JSON.parse(r.receipt); + const files: string[] = Array.isArray(rc.filesChanged) ? rc.filesChanged : []; + out.push({ + kind: 'receipt', at: r.started_at, when: relTime(r.started_at), + text: `maintain ${scope}: ${rc.summary || rc.reason || rc.status}`, + files, detail: scope, + verified: rc.status === 'opened' ? true : rc.status === 'blocked' || rc.status === 'failed' ? false : undefined, + }); + } catch { /* skip malformed receipt */ } + } + } + return out; + } catch { return []; } + })(); + + const matches = rankApproaches(args.query, [...episodes, ...worklog, ...facts, ...receipts], { topK: args.limit ?? 5, nowMs: Date.now(), diversity: 0.35 }); // Visual comparison for the top approaches (best match + how the others differed + stable // core); anything past the visualized window is listed compactly so nothing is hidden. diff --git a/test/approach-diff.test.ts b/test/approach-diff.test.ts index 4708af2..8ecd23d 100644 --- a/test/approach-diff.test.ts +++ b/test/approach-diff.test.ts @@ -91,6 +91,43 @@ describe('renderApproachDiffs', () => { expect(renderApproachDiffs('xyz', [])).toMatch(/No past work/); }); + it('shows ground-truth outcome marks and a receipt tag', () => { + const out = renderApproachDiffs('auth', [ + { kind: 'episode', text: 'add auth login middleware', when: '2d ago', score: 0.9, verified: true }, + { kind: 'receipt', text: 'maintain unused-imports: auth files cleaned', when: '3d ago', score: 0.5, verified: false }, + ]); + expect(out).toContain('🎯 task ✓'); // verified episode + expect(out).toContain('🧾 receipt ⛔'); // blocked maintain run + }); + + it('renders a chronological timeline (oldest → newest, capped, dated-only, outcome marks)', () => { + const T = (text: string, at: string, verified?: boolean): ApproachMatch => + ({ kind: 'episode', text, when: at.slice(0, 10), at, score: 0.5, verified }); + const matches = [ + T('third attempt with refresh tokens', '2026-06-20T10:00:00Z', true), + T('first attempt basic auth', '2026-03-01T10:00:00Z'), + T('second attempt jwt middleware', '2026-05-10T10:00:00Z', false), + { kind: 'fact' as const, text: 'undated fact', when: '', score: 0.4 }, // no date → skipped + ]; + const out = renderApproachDiffs('auth', matches); + expect(out).toContain('Timeline (oldest → newest):'); + const tl = out.slice(out.indexOf('Timeline')); + expect(tl.indexOf('2026-03-01')).toBeLessThan(tl.indexOf('2026-05-10')); + expect(tl.indexOf('2026-05-10')).toBeLessThan(tl.indexOf('2026-06-20')); + expect(tl).toContain('2026-06-20 ✓'); + expect(tl).toContain('2026-05-10 ⛔'); + expect(tl).not.toContain('undated fact'); + expect(tl).toContain('● 2026-06-20'); // newest gets the filled glyph + }); + + it('timeline is omitted when fewer than 2 matches carry a date', () => { + const out = renderApproachDiffs('auth', [ + { kind: 'episode', text: 'one dated', when: '2d', at: '2026-06-20T00:00:00Z', score: 0.9 }, + { kind: 'fact', text: 'undated', when: '', score: 0.5 }, + ]); + expect(out).not.toContain('Timeline'); + }); + it('identical vocabulary → "same approach" note instead of empty ± lines', () => { const out = renderApproachDiffs('auth', [ M('episode', 'add auth login', 0.9), diff --git a/test/approach-recall.test.ts b/test/approach-recall.test.ts index 2d5f732..f4da9b6 100644 --- a/test/approach-recall.test.ts +++ b/test/approach-recall.test.ts @@ -69,6 +69,17 @@ describe('rankApproaches', () => { expect(stem('categories')).toBe('category'); }); + it('receipt-aware ranking: a VERIFIED approach beats an equally-relevant unverified one; blocked sinks', () => { + const three: ApproachSource[] = [ + { kind: 'episode', text: 'add login auth flow', when: 'unknown' }, // no signal + { kind: 'episode', text: 'add login auth flow', when: 'proven', verified: true }, // gates passed + { kind: 'receipt', text: 'add login auth flow', when: 'blocked', verified: false }, // guardrail declined + ]; + const m = rankApproaches('add login auth flow', three, { minScore: 0.1 }); + expect(m.map(x => x.when)).toEqual(['proven', 'unknown', 'blocked']); // ✓ > unknown > ⛔ + expect(m[0]!.score).toBeGreaterThan(0.9); // displayed score stays honest relevance + }); + it('on near-equal relevance, the more RECENT approach surfaces first', () => { const now = Date.parse('2026-07-01T00:00:00Z'); const iso = (d: number) => new Date(now - d * 86_400_000).toISOString();