diff --git a/cmd/meat/main.go b/cmd/meat/main.go index 280397a..112f871 100644 --- a/cmd/meat/main.go +++ b/cmd/meat/main.go @@ -213,7 +213,11 @@ func run(ctx context.Context, o runOpts) error { cacheStore(o.cacheDir, key, res) o.render(res) - fmt.Fprintf(o.stderr, "\nmeat: tokens in=%d out=%d in %s\n", res.InputTokens, res.OutputTokens, elapsed) + cached := "" + if res.CacheReadTokens > 0 || res.CacheWriteTokens > 0 { + cached = fmt.Sprintf(" (cache read=%d write=%d)", res.CacheReadTokens, res.CacheWriteTokens) + } + fmt.Fprintf(o.stderr, "\nmeat: tokens in=%d%s out=%d in %s\n", res.InputTokens, cached, res.OutputTokens, elapsed) return nil } diff --git a/meat/anthropic.go b/meat/anthropic.go index 035f062..1a6b09f 100644 --- a/meat/anthropic.go +++ b/meat/anthropic.go @@ -71,7 +71,7 @@ func NewAnthropicFromEnv(ctx context.Context, model string) (*AnthropicModel, er type antReq struct { Model string `json:"model"` MaxTokens int `json:"max_tokens"` - System string `json:"system"` + System []antBlock `json:"system"` Messages []antMessage `json:"messages"` Tools []antTool `json:"tools,omitempty"` } @@ -82,16 +82,26 @@ type antMessage struct { } type antBlock struct { - Type string `json:"type"` - Text string `json:"text,omitempty"` - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Input json.RawMessage `json:"input,omitempty"` - ToolUseID string `json:"tool_use_id,omitempty"` - Content string `json:"content,omitempty"` - IsError bool `json:"is_error,omitempty"` + Type string `json:"type"` + Text string `json:"text,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Input json.RawMessage `json:"input,omitempty"` + ToolUseID string `json:"tool_use_id,omitempty"` + Content string `json:"content,omitempty"` + IsError bool `json:"is_error,omitempty"` + CacheControl *antCacheControl `json:"cache_control,omitempty"` } +// antCacheControl marks a cache breakpoint: the request prefix up to and +// including the marked block is cached. +type antCacheControl struct { + Type string `json:"type"` +} + +// The 5-minute lifetime outlasts one Abridge run's 4-minute budget. +var ephemeralCache = &antCacheControl{Type: "ephemeral"} + type antTool struct { Name string `json:"name"` Description string `json:"description"` @@ -102,8 +112,10 @@ type antResp struct { Content []antBlock `json:"content"` StopReason string `json:"stop_reason"` Usage struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + CacheCreationInputTokens int `json:"cache_creation_input_tokens"` + CacheReadInputTokens int `json:"cache_read_input_tokens"` } `json:"usage"` Error *struct { Type string `json:"type"` @@ -117,13 +129,17 @@ func (m *AnthropicModel) Generate(ctx context.Context, system string, messages [ return nil, fmt.Errorf("meat: AnthropicModel.APIKey is empty") } + // The loop re-sends the whole conversation every turn, so most input is a + // replay of an identical prefix. Cache the system prompt (stable across + // runs) and the numbered diff (stable across turns). reqBody := antReq{ Model: cmpOr(m.Model, DefaultAnthropicModel), MaxTokens: maxOutputTokens, - System: system, + System: []antBlock{{Type: "text", Text: system, CacheControl: ephemeralCache}}, Messages: toAntMessages(messages), Tools: toAntTools(tools), } + markCacheBreakpoints(reqBody.Messages) body, err := json.Marshal(reqBody) if err != nil { return nil, err @@ -153,9 +169,14 @@ func (m *AnthropicModel) Generate(ctx context.Context, system string, messages [ return nil, fmt.Errorf("anthropic response truncated at max_tokens (%d); the diff may be too large to abridge in one pass", maxOutputTokens) } + // InputTokens stays the total processed, so it stays comparable to + // pre-caching runs; the breakdown splits it by billing rate. out := &Response{ - InputTokens: resp.Usage.InputTokens, - OutputTokens: resp.Usage.OutputTokens, + InputTokens: resp.Usage.InputTokens + + resp.Usage.CacheCreationInputTokens + resp.Usage.CacheReadInputTokens, + OutputTokens: resp.Usage.OutputTokens, + CacheWriteTokens: resp.Usage.CacheCreationInputTokens, + CacheReadTokens: resp.Usage.CacheReadInputTokens, } for _, b := range resp.Content { switch b.Type { @@ -252,6 +273,24 @@ func toAntTools(tools []Tool) []antTool { return out } +// markCacheBreakpoints marks the first message (the numbered diff) and the +// last, which rolls forward each turn to absorb accumulated tool results. +// Anthropic allows four breakpoints per request; the system prompt uses one. +func markCacheBreakpoints(msgs []antMessage) { + mark := func(m *antMessage) { + if len(m.Content) > 0 { + m.Content[len(m.Content)-1].CacheControl = ephemeralCache + } + } + if len(msgs) == 0 { + return + } + mark(&msgs[0]) + if len(msgs) > 1 { + mark(&msgs[len(msgs)-1]) + } +} + func toAntMessages(messages []Message) []antMessage { out := make([]antMessage, 0, len(messages)) for _, m := range messages { diff --git a/meat/anthropic_test.go b/meat/anthropic_test.go index 3874687..21ce5a5 100644 --- a/meat/anthropic_test.go +++ b/meat/anthropic_test.go @@ -52,9 +52,15 @@ func TestAbridge_AnthropicEditPlanEndToEnd(t *testing.T) { w.WriteHeader(http.StatusBadRequest) return } - if !strings.Contains(req.System, "edit plan") { + if len(req.System) != 1 || !strings.Contains(req.System[0].Text, "edit plan") { t.Errorf("system prompt does not describe edit plans") } + if req.System[0].CacheControl == nil { + t.Errorf("system prompt is not a cache breakpoint") + } + if len(req.Messages) > 0 && req.Messages[0].Content[len(req.Messages[0].Content)-1].CacheControl == nil { + t.Errorf("numbered diff is not a cache breakpoint") + } if len(req.Tools) != 2 || req.Tools[0].Name != "preview_plan" || req.Tools[1].Name != "submit" || strings.Contains(string(req.Tools[1].InputSchema), "smart_diff") || !strings.Contains(string(req.Tools[1].InputSchema), `"fold"`) { t.Errorf("unexpected plan tool schema: %+v", req.Tools) } diff --git a/meat/chunk.go b/meat/chunk.go index ebae190..8a96d94 100644 --- a/meat/chunk.go +++ b/meat/chunk.go @@ -966,6 +966,8 @@ func abridgeChunked(ctx context.Context, model Model, req Request) (*Result, err } merged.InputTokens += res.InputTokens merged.OutputTokens += res.OutputTokens + merged.CacheWriteTokens += res.CacheWriteTokens + merged.CacheReadTokens += res.CacheReadTokens if strings.TrimSpace(res.SmartDiff) != "" { piece := res.SmartDiff if chunk.sectionID >= 0 { diff --git a/meat/diff.go b/meat/diff.go index aab57a1..589df33 100644 --- a/meat/diff.go +++ b/meat/diff.go @@ -128,10 +128,42 @@ func numberedDiff(diff string) string { if len(lines) == 0 { return "" } + return renderNumbered(lines, nil) +} + +// numberedDiffEliding collapses every hidden line, keeping original 1-based +// numbers so the model's coordinates still address the immutable input. Each +// hidden run becomes one marker row, so the model can see something sits +// between two visible lines and will not fold across it. +func numberedDiffEliding(diff string, hidden []bool) string { + lines := splitSourceLines(diff) + if len(lines) == 0 { + return "" + } + return renderNumbered(lines, hidden) +} + +// Marker for a collapsed run of automatically removed import rows. +const elidedImportRun = "(imports, removed automatically)" + +func renderNumbered(lines []sourceLine, hidden []bool) string { width := len(strconv.Itoa(len(lines))) var b strings.Builder - for i, line := range lines { - fmt.Fprintf(&b, "%*d|%s\n", width, i+1, line.text) + for i := 0; i < len(lines); i++ { + if hidden != nil && hidden[i] { + run := i + for run+1 < len(lines) && hidden[run+1] { + run++ + } + if run == i { + fmt.Fprintf(&b, "%*d|%s\n", width, i+1, elidedImportRun) + } else { + fmt.Fprintf(&b, "%*d-%d|%s\n", width, i+1, run+1, elidedImportRun) + } + i = run + continue + } + fmt.Fprintf(&b, "%*d|%s\n", width, i+1, lines[i].text) } return b.String() } diff --git a/meat/imports.go b/meat/imports.go index ce3eebf..de5962a 100644 --- a/meat/imports.go +++ b/meat/imports.go @@ -338,6 +338,18 @@ func hiddenLineRanges(hidden []bool) []lineRange { return ranges } +// mandatoryHiddenMask reports which lines the compiler removes regardless of +// the model's plan. Those rows are elided from the prompt: their fate is +// already decided, so sending them is wasted. +func mandatoryHiddenMask(diff string) []bool { + lines := splitSourceLines(diff) + if len(lines) == 0 { + return nil + } + layout := analyzeDiff(lines) + return mandatoryRemovalMask(len(lines), mandatoryImportRemovalPlan(lines, layout)) +} + func mandatoryRemovalMask(lines int, ranges []lineRange) []bool { hidden := make([]bool, lines) for _, r := range ranges { diff --git a/meat/meat.go b/meat/meat.go index 95071cf..374c4c4 100644 --- a/meat/meat.go +++ b/meat/meat.go @@ -100,8 +100,13 @@ type Result struct { // Summary is a one-line, high-level description of the change. Summary string `json:"summary"` // InputTokens and OutputTokens are the cumulative token usage across the run. + // InputTokens counts all input processed, including cached. InputTokens int `json:"input_tokens"` OutputTokens int `json:"output_tokens"` + // CacheWriteTokens and CacheReadTokens split InputTokens by billing rate. + // Zero on providers without an explicit prompt cache. + CacheWriteTokens int `json:"cache_write_tokens,omitempty"` + CacheReadTokens int `json:"cache_read_tokens,omitempty"` } // noToolCallNudge is sent when the model produced text but no tool call. Like @@ -134,7 +139,7 @@ func Abridge(ctx context.Context, model Model, req Request) (*Result, error) { // abridgeOne runs the agent loop on one single-run-sized diff: the whole // input when it fits, or one chunk of a split diff. func abridgeOne(ctx context.Context, model Model, req Request, opts runOptions) (*Result, error) { - numbered := numberedDiff(req.UnifiedDiff) + numbered := numberedDiffEliding(req.UnifiedDiff, mandatoryHiddenMask(req.UnifiedDiff)) maxTurns := req.MaxTurns if maxTurns <= 0 { @@ -158,7 +163,8 @@ func abridgeOne(ctx context.Context, model Model, req Request, opts runOptions) progress = func(string) {} } - var inTok, outTok int + var inTok, outTok, cacheWrite, cacheRead int + var previewIDs []string var retentionNudged bool var fallback *Result @@ -167,14 +173,15 @@ func abridgeOne(ctx context.Context, model Model, req Request, opts runOptions) resp, err := model.Generate(ctx, systemPrompt, messages, tools) if err != nil { if fallback != nil && callerCtx.Err() == nil { - fallback.InputTokens = inTok - fallback.OutputTokens = outTok + setUsage(fallback, inTok, outTok, cacheWrite, cacheRead) return fallback, nil } return nil, fmt.Errorf("meat: model generate: %w", err) } inTok += resp.InputTokens outTok += resp.OutputTokens + cacheWrite += resp.CacheWriteTokens + cacheRead += resp.CacheReadTokens messages = append(messages, Message{Role: RoleAssistant, Content: resp.Content}) @@ -185,6 +192,9 @@ func abridgeOne(ctx context.Context, model Model, req Request, opts runOptions) } progress(describeToolCall(b)) out, isErr := tb.run(ctx, b.ToolName, b.ToolInput) + if b.ToolName == "preview_plan" { + previewIDs = append(previewIDs, b.ID) + } results = append(results, Block{ Type: "tool_result", ToolUseID: b.ID, @@ -195,11 +205,10 @@ func abridgeOne(ctx context.Context, model Model, req Request, opts runOptions) if tb.submitSeen { candidate := &Result{ - SmartDiff: tb.smartDiff, - Summary: tb.submitted.Summary, - InputTokens: inTok, - OutputTokens: outTok, + SmartDiff: tb.smartDiff, + Summary: tb.submitted.Summary, } + setUsage(candidate, inTok, outTok, cacheWrite, cacheRead) canRefine := !retentionNudged && turn+1 < maxTurns && tb.submittedPlan != nil && retentionPressure(tb.submittedPlan.stats) if canRefine { @@ -207,6 +216,7 @@ func abridgeOne(ctx context.Context, model Model, req Request, opts runOptions) retentionNudged = true tb.clearSubmission() messages = append(messages, Message{Role: RoleUser, Content: results}) + elideSupersededPreviews(messages, previewIDs) continue } return candidate, nil @@ -222,25 +232,54 @@ func abridgeOne(ctx context.Context, model Model, req Request, opts runOptions) continue } messages = append(messages, Message{Role: RoleUser, Content: results}) + elideSupersededPreviews(messages, previewIDs) } if fallback != nil { if err := callerCtx.Err(); err != nil { return nil, fmt.Errorf("meat: caller context: %w", err) } - fallback.InputTokens = inTok - fallback.OutputTokens = outTok + setUsage(fallback, inTok, outTok, cacheWrite, cacheRead) return fallback, nil } return nil, fmt.Errorf("meat: agent did not submit within %d turns", maxTurns) } +// Replaces a preview_plan result that a later preview superseded. +const supersededPreviewStub = "(superseded preview elided; plans are complete restatements against the ORIGINAL diff)" + +// elideSupersededPreviews stubs every preview_plan result but the newest. Each +// embeds a full rendered diff, and since plans are complete restatements rather +// than incremental edits, an older preview cannot inform a later turn. +func elideSupersededPreviews(messages []Message, previewIDs []string) { + if len(previewIDs) < 2 { + return + } + stale := make(map[string]bool, len(previewIDs)-1) + for _, id := range previewIDs[:len(previewIDs)-1] { + stale[id] = true + } + for _, m := range messages { + for i, b := range m.Content { + if b.Type == "tool_result" && stale[b.ToolUseID] && m.Content[i].ToolResult != supersededPreviewStub { + m.Content[i].ToolResult = supersededPreviewStub + } + } + } +} + +// setUsage stamps cumulative token counters onto a result. +func setUsage(r *Result, in, out, cacheWrite, cacheRead int) { + r.InputTokens, r.OutputTokens = in, out + r.CacheWriteTokens, r.CacheReadTokens = cacheWrite, cacheRead +} + // The static model-visible user-prompt fragments. Every string the model can // see that is not derived from the input diff lives in a named const so // promptSurface can hash the complete frozen surface. const ( userPromptIntro = "Abridge the following unified diff into a reading diff by submitting a complete remove/replace/fold plan against the numbered original lines. Meat applies your plan to the original diff; you do not write the resulting diff yourself. Coordinates are 1-based and always refer to the original numbering. The `N|` gutter is display-only and is not part of a line's source text. Use preview_plan to inspect sizeable drafts before submit.\n" - userPromptImports = "Imports/includes/requires/use declarations are removed automatically, including multiline blocks and recognized imports inside embedded source strings. They may appear in the numbered input but never in a preview or result. Do not spend edit coordinates on them, never fold across them into behavioral rows, and do not mention them in the summary.\n" + userPromptImports = "Rows shown as (imports, removed automatically) stand for import/include/require/use declarations already elided from the input and absent from every preview and result. Do not spend edit coordinates on them, never fold across one into behavioral rows, and do not mention them in the summary.\n" userPromptMoves = "Meat detected exact source-evidenced moves across hunks/files: %s. Give both sides of each pair identical keep/remove/fold/replace treatment, including matching fold boundaries and equivalent local elisions; automatically removed rows need none. Asymmetric plans are rejected.\n" userPromptTools = "Use read_file/grep on the surrounding source only when it changes your judgment about what is load-bearing (or whether a file is generated), then preview or submit.\n" userPromptNoTools = "Judge from the diff text alone, then preview or submit.\n" diff --git a/meat/meat_test.go b/meat/meat_test.go index d971aa4..64e53f6 100644 --- a/meat/meat_test.go +++ b/meat/meat_test.go @@ -761,7 +761,7 @@ func TestSurfaceFixturesCoverBothMoveBranches(t *testing.T) { } func TestRubricHashPinned(t *testing.T) { - const pinned = "441f5e6e28ad3add" + const pinned = "24409308abf9d0ea" if h := RubricHash(); h != pinned { t.Errorf("RubricHash() = %q, pinned %q; the model-visible prompt surface changed — review it against the freeze policy on systemPrompt, then update the pin", h, pinned) } diff --git a/meat/model.go b/meat/model.go index 3f051fc..3e4c397 100644 --- a/meat/model.go +++ b/meat/model.go @@ -62,9 +62,14 @@ type Tool struct { // Response is a single model reply. type Response struct { - Content []Block + Content []Block + // InputTokens counts all input processed, including cached. InputTokens int OutputTokens int + // CacheWriteTokens and CacheReadTokens split InputTokens by billing rate. + // Zero on providers without an explicit prompt cache. + CacheWriteTokens int + CacheReadTokens int } // Model is the minimal LLM interface meat needs. It is provider-agnostic: the diff --git a/meat/rubric.go b/meat/rubric.go index 6b04ef4..2e9d5c9 100644 --- a/meat/rubric.go +++ b/meat/rubric.go @@ -10,7 +10,7 @@ import ( // abridgeProtocolVersion covers the machine-side edit protocol as well as the // prose rubric. Changing the submit schema or edit semantics must invalidate // cached results even when the high-level advice is unchanged. -const abridgeProtocolVersion = "source-edit-plan-v10-frozen-prompt-surface" +const abridgeProtocolVersion = "source-edit-plan-v12-elided-import-rows" // surfaceFixtureDiff is a canonical input containing an exact cross-file move, // used to render every branch of the model-visible prompt surface for hashing. @@ -68,13 +68,14 @@ func promptSurface() string { b.WriteString(abridgeProtocolVersion) add(systemPrompt) - numbered := numberedDiff(surfaceFixtureDiff) + numbered := numberedDiffEliding(surfaceFixtureDiff, mandatoryHiddenMask(surfaceFixtureDiff)) add(buildUserPrompt(Request{UnifiedDiff: surfaceFixtureDiff, RepoRoot: "/repo"}, runOptions{}, numbered)) add(buildUserPrompt(Request{UnifiedDiff: surfaceFixtureDiff}, runOptions{}, numbered)) - numberedNoMove := numberedDiff(surfaceFixtureNoMoveDiff) + numberedNoMove := numberedDiffEliding(surfaceFixtureNoMoveDiff, mandatoryHiddenMask(surfaceFixtureNoMoveDiff)) add(buildUserPrompt(Request{UnifiedDiff: surfaceFixtureNoMoveDiff, RepoRoot: "/repo"}, runOptions{}, numberedNoMove)) add(buildUserPrompt(Request{UnifiedDiff: surfaceFixtureNoMoveDiff}, runOptions{}, numberedNoMove)) add(noToolCallNudge) + add(supersededPreviewStub) for _, tools := range [][]Tool{ (&toolbox{root: "/repo"}).tools(), @@ -91,7 +92,12 @@ func promptSurface() string { // builder: a fixture with more detected moves than maxMoveHints, so // "and N more" appears in an actual user prompt. overflowDiff := surfaceOverflowDiff() - add(buildUserPrompt(Request{UnifiedDiff: overflowDiff}, runOptions{}, numberedDiff(overflowDiff))) + add(buildUserPrompt(Request{UnifiedDiff: overflowDiff}, runOptions{}, numberedDiffEliding(overflowDiff, mandatoryHiddenMask(overflowDiff)))) + + // The import-elision branch: a fixture with real import rows, so the + // collapsed-run marker appears in an actual rendered user prompt. + importDiff := surfaceImportDiff() + add(buildUserPrompt(Request{UnifiedDiff: importDiff}, runOptions{}, numberedDiffEliding(importDiff, mandatoryHiddenMask(importDiff)))) // Plan feedback exactly as the model receives it: rendered by the real // preview_plan tool handler, which applies tool-output truncation. The @@ -155,6 +161,21 @@ func surfaceOverflowDiff() string { return b.String() } +// surfaceImportDiff renders both the single-line and multi-line elision +// markers on the hashed prompt surface. +func surfaceImportDiff() string { + return "diff --git a/x.go b/x.go\n--- a/x.go\n+++ b/x.go\n" + + "@@ -1,7 +1,8 @@\n" + + " import (\n" + + "-\t\"math/rand\"\n" + + "+\t\"crypto/rand\"\n" + + "+\t\"encoding/hex\"\n" + + " )\n" + + " \n" + + "-\treturn fmt.Sprintf(\"%x\", b)\n" + + "+\treturn hex.EncodeToString(b)\n" +} + // surfaceOversizeDiff builds a valid diff whose identity preview exceeds // maxToolOutput, forcing the preview_plan handler to truncate. func surfaceOversizeDiff() string {