diff --git a/CHANGELOG.md b/CHANGELOG.md index d39dd01..894e138 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Added + +- **Configurable compaction model.** `modelSystem.compaction` selects the model + used to summarize the conversation during context compaction (both automatic + compaction and `/compact`). It defaults to the main conversation model; set it + to a `ModelRef` object (`{ "backend": "...", "model": "..." }`) to compact with + a different (for example cheaper or longer-context) model. When the configured backend is not ready, compaction + inherits the main model and prints a warning in the compaction notice; if the + chosen model errors mid-compaction, it retries with the main model before + falling back to the deterministic trim, reporting the failure instead of + silently degrading. The configured model is shown in `/route status` and + `/route template`. + ## [1.2.0] - 2026-07-08 ### Added diff --git a/README.md b/README.md index 7d51a08..3ee11fd 100644 --- a/README.md +++ b/README.md @@ -853,7 +853,10 @@ token counts, and reported OpenRouter costs stay visible. `/route` lets you describe a model stack that blends local and frontier models: separate orchestrators for low/medium/high planning, coders for low/medium/high implementation, play and production review models, a security -review model, and one selector model that chooses the route for a task. +review model, a compaction model that summarizes the transcript when context is +compacted, and one selector model that chooses the route for a task. `/route +status` shows the configured stack (including the compaction model) and `/route +template` prints the JSON shape to paste into `agent.config.json`. ```text /route template @@ -879,6 +882,25 @@ For whole-goal decomposition, `/plan route ` uses `modelSystem.planner` or a planner override to create `.small-harness/plan.json`; `/plan execute` then runs each ready node with the configured low/medium/high coder model. +Context compaction (summarizing the conversation when the prompt budget fills, +both automatically and via `/compact`) uses the main conversation model by +default. Set `modelSystem.compaction` to a `ModelRef` object (a `{ "backend": +"...", "model": "..." }` pair, the same shape as `planner`/`selector`) to +summarize with a different model, for example a cheaper or longer-context one: + +```json +{ + "modelSystem": { + "compaction": { "backend": "openrouter", "model": "anthropic/claude-3.5-haiku" } + } +} +``` + +Prompt-budget sizing still tracks the main model (that is the context being +fit); only the summary call uses the compaction model. If the compaction +backend is not ready (for example a missing API key), compaction falls back to +the main model and prints a warning. + --- ## Configuration diff --git a/src/commands/context_cmds.rs b/src/commands/context_cmds.rs index 8afeef2..c4f9385 100644 --- a/src/commands/context_cmds.rs +++ b/src/commands/context_cmds.rs @@ -308,6 +308,9 @@ pub(super) async fn cmd_compact(args: &str, state: &mut AppState) -> Result<()> " {GREEN}✓{RESET} {DIM}{}{RESET}", state.context_guard_notice.as_deref().unwrap_or("") ); + if let Some(warning) = result.warning { + println!(" {YELLOW}!{RESET} {DIM}{warning}{RESET}"); + } println!(" {DIM}session → {}{RESET}", state.session_path.display()); Ok(()) } diff --git a/src/commands/route.rs b/src/commands/route.rs index ad24cb3..44b42d5 100644 --- a/src/commands/route.rs +++ b/src/commands/route.rs @@ -171,6 +171,7 @@ fn print_route_status(stack: &ModelSystemConfig) { } print_model_ref("planner", stack.planner.as_ref()); print_model_ref("selector", stack.selector.as_ref()); + print_model_ref("compaction", stack.compaction.as_ref()); print_tier_set("orchestrator", &stack.orchestrators); print_tier_set("coder", &stack.coders); print_review_set("review", &stack.reviewers); @@ -215,6 +216,11 @@ fn print_route_template() { "thinkingDepth": "deep", "notes": "Chooses the model route for a task." }}, + "compaction": {{ + "backend": "openrouter", + "model": "anthropic/claude-3.5-haiku", + "notes": "Summarizes the transcript when context is compacted. Omit to inherit the main model." + }}, "orchestrators": {{ "low": {{ "backend": "ollama", "model": "qwen2.5-coder:7b" }}, "medium": {{ "backend": "openrouter", "model": "qwen/qwen-2.5-coder-32b-instruct" }}, diff --git a/src/context_guard.rs b/src/context_guard.rs index 8a966aa..b79e8f2 100644 --- a/src/context_guard.rs +++ b/src/context_guard.rs @@ -31,6 +31,11 @@ pub struct CompactResult { pub after_ratio: f64, pub method: CompactMethod, pub conversation_summary: Option, + /// Set when a configured compaction model could not be used as-is, so a + /// fallback ran: an unavailable backend inherits the main model; a dedicated + /// model that errors or returns empty is retried with the main model and, if + /// that also fails, the deterministic trim. Surfaced in the compaction notice. + pub warning: Option, } #[derive(Debug, Clone)] @@ -42,6 +47,44 @@ pub struct ContextGuardConfig { pub model_context_tokens: usize, } +/// Which model summarizes the transcript during compaction. Compaction inherits +/// the main conversation model unless `modelSystem.compaction` selects another. +#[derive(Debug, Clone, Default)] +pub enum CompactionModel { + /// Use the main conversation model (default, and when nothing is configured). + #[default] + Inherit, + /// A configured, ready model dedicated to compaction. + Use { + backend: BackendDescriptor, + model: String, + label: String, + }, + /// A model was configured but its backend is not ready (e.g. missing API + /// key). Compaction inherits the main model and warns. + Unavailable { label: String, reason: String }, +} + +/// Resolve `modelSystem.compaction` into a [`CompactionModel`]. Pure: it does no +/// I/O and never warns; the not-ready warning is surfaced when compaction runs. +pub fn classify_compaction_model(config: &AgentConfig) -> CompactionModel { + let Some(model_ref) = config.model_system.compaction() else { + return CompactionModel::Inherit; + }; + let backend = config.backend_descriptor_for(model_ref.backend); + match crate::backends::validate(&backend) { + Ok(()) => CompactionModel::Use { + backend, + model: model_ref.model.clone(), + label: model_ref.label(), + }, + Err(e) => CompactionModel::Unavailable { + label: model_ref.label(), + reason: e.to_string(), + }, + } +} + #[derive(Debug, Clone)] pub struct ContextGuardParams { pub effective_limit_bytes: usize, @@ -50,6 +93,7 @@ pub struct ContextGuardParams { pub keep_messages: usize, pub summarize_budget_bytes: usize, pub conversation_summary: Option, + pub compaction: CompactionModel, } #[derive(Debug, Clone)] @@ -83,6 +127,7 @@ struct CompactRequest<'a> { backend: &'a BackendDescriptor, model: &'a str, conversation_summary: Option<&'a str>, + compaction: &'a CompactionModel, } pub fn default_model_context_tokens(model: &str, is_local: bool) -> usize { @@ -150,6 +195,7 @@ pub fn guard_params_from( summarize_budget_bytes: (guard.effective_limit_bytes as f64 * SUMMARIZE_BUDGET_FRACTION) as usize, conversation_summary, + compaction: classify_compaction_model(config), } } @@ -532,8 +578,17 @@ async fn compact_messages_core(req: CompactRequest<'_>) -> Result backend, model, conversation_summary, + compaction, } = req; + // Compaction summarizes with a dedicated model when one is configured and + // ready; otherwise it inherits the main model. Prompt-budget sizing above is + // always based on the main model, since that is the context we are fitting. + let (summary_backend, summary_model): (&BackendDescriptor, &str) = match compaction { + CompactionModel::Use { backend, model, .. } => (backend, model.as_str()), + CompactionModel::Inherit | CompactionModel::Unavailable { .. } => (backend, model), + }; + let active_system_prompt = merge_system_prompt(system_prompt, conversation_summary); let budget_before = measure_prompt_budget(&active_system_prompt, messages, tool_defs); let before_messages = messages.len(); @@ -548,6 +603,7 @@ async fn compact_messages_core(req: CompactRequest<'_>) -> Result after_ratio: before_ratio, method: CompactMethod::None, conversation_summary: conversation_summary.map(str::to_string), + warning: None, }); } @@ -561,10 +617,14 @@ async fn compact_messages_core(req: CompactRequest<'_>) -> Result after_ratio: before_ratio, method: CompactMethod::None, conversation_summary: conversation_summary.map(str::to_string), + warning: None, }); }; let use_tier2 = transcript_json_bytes(&older) > summarize_budget_bytes; + // Only warn about an unavailable compaction model when we would actually have + // called it (the LLM-summary path); the deterministic trim uses no model. + let mut warning = None; let (method, new_summary) = if use_tier2 { let addition = deterministic_summary(&older); ( @@ -572,11 +632,56 @@ async fn compact_messages_core(req: CompactRequest<'_>) -> Result append_summary(conversation_summary, &addition), ) } else { - match summarize_transcript(http, backend, model, conversation_summary, &older).await { + if let CompactionModel::Unavailable { label, reason } = compaction { + warning = Some(format!( + "compaction model {label} unavailable ({reason}); summarized with {model}" + )); + } + let mut result = summarize_transcript( + http, + summary_backend, + summary_model, + conversation_summary, + &older, + ) + .await; + + // A dedicated compaction model that errors or returns empty retries with + // the main conversation model before falling back to the deterministic + // trim, so a broken compaction backend never silently loses the summary. + let retried_main = if let CompactionModel::Use { label, .. } = compaction { + if matches!(&result, Ok(summary) if !summary.trim().is_empty()) { + false + } else { + let detail = match &result { + Err(e) => e.to_string(), + _ => "returned an empty summary".to_string(), + }; + warning = Some(format!( + "compaction model {label} failed ({detail}); summarized with {model} instead" + )); + result = + summarize_transcript(http, backend, model, conversation_summary, &older).await; + true + } + } else { + false + }; + + match result { Ok(summary) if !summary.trim().is_empty() => { (CompactMethod::LlmSummary, summary.trim().to_string()) } - Ok(_) | Err(_) => { + _ => { + // The summary model — and the main-model retry, if one ran — both + // failed, so fall back to the deterministic trim. + if retried_main { + if let CompactionModel::Use { label, .. } = compaction { + warning = Some(format!( + "compaction model {label} and main model {model} both failed; trimmed deterministically" + )); + } + } let addition = deterministic_summary(&older); ( CompactMethod::DeterministicTrim, @@ -601,6 +706,7 @@ async fn compact_messages_core(req: CompactRequest<'_>) -> Result after_ratio, method, conversation_summary: Some(new_summary), + warning, }) } @@ -612,6 +718,7 @@ pub async fn compact_messages( let keep = keep.unwrap_or(guard.keep_messages); let summarize_budget = (guard.effective_limit_bytes as f64 * SUMMARIZE_BUDGET_FRACTION) as usize; + let compaction = classify_compaction_model(ctx.config); compact_messages_core(CompactRequest { messages: ctx.messages, @@ -624,6 +731,7 @@ pub async fn compact_messages( backend: ctx.backend, model: ctx.model, conversation_summary: ctx.conversation_summary, + compaction: &compaction, }) .await } @@ -687,16 +795,7 @@ pub async fn maybe_auto_compact( rewrite_session_transcript(session_dir, session_path, ctx.messages)?; Ok(Some(CompactNotice { - line: format!( - " \x1b[32m✓\x1b[0m \x1b[2m{}\x1b[0m", - compact_notice( - result.before_messages, - result.after_messages, - result.before_ratio, - result.after_ratio, - result.method - ) - ), + line: compaction_notice_line(&result), conversation_summary: result.conversation_summary.clone(), transcript_rewritten: true, })) @@ -737,6 +836,7 @@ pub async fn maybe_compact_messages( backend, model, conversation_summary: guard.conversation_summary.as_deref(), + compaction: &guard.compaction, }) .await?; @@ -745,21 +845,31 @@ pub async fn maybe_compact_messages( } Ok(Some(CompactNotice { - line: format!( - " \x1b[32m✓\x1b[0m \x1b[2m{}\x1b[0m", - compact_notice( - result.before_messages, - result.after_messages, - result.before_ratio, - result.after_ratio, - result.method - ) - ), + line: compaction_notice_line(&result), conversation_summary: result.conversation_summary.clone(), transcript_rewritten: true, })) } +/// The one-line compaction notice, with an optional warning line appended when a +/// configured compaction model could not be used. +fn compaction_notice_line(result: &CompactResult) -> String { + let mut line = format!( + " \x1b[32m✓\x1b[0m \x1b[2m{}\x1b[0m", + compact_notice( + result.before_messages, + result.after_messages, + result.before_ratio, + result.after_ratio, + result.method + ) + ); + if let Some(warning) = &result.warning { + line.push_str(&format!("\n \x1b[33m!\x1b[0m \x1b[2m{warning}\x1b[0m")); + } + line +} + pub fn context_status_lines( config: &AgentConfig, model: &str, @@ -972,6 +1082,7 @@ mod tests { backend: &backend, model: "mock", conversation_summary: Some(&existing_summary), + compaction: &CompactionModel::Inherit, }) .await .expect("deterministic compaction"); @@ -1012,4 +1123,129 @@ mod tests { ); assert_eq!(default_model_context_tokens("gpt-4", false), 128_000); } + + #[test] + fn compaction_inherits_main_model_by_default() { + let config = AgentConfig::default(); + assert!(matches!( + classify_compaction_model(&config), + CompactionModel::Inherit + )); + } + + #[test] + fn compaction_uses_configured_ready_backend() { + // Ollama needs no API key, so its backend is always ready. + let mut config = AgentConfig::default(); + config.model_system.compaction = + crate::model_system::ModelRef::parse_spec("ollama:qwen2.5-coder:7b"); + match classify_compaction_model(&config) { + CompactionModel::Use { model, .. } => assert_eq!(model, "qwen2.5-coder:7b"), + other => panic!("expected Use, got {other:?}"), + } + } + + #[test] + fn compaction_marks_not_ready_backend_unavailable() { + // openrouter is not ready without an API key. Force the key empty for + // this test regardless of the ambient environment, then restore it. + let saved = std::env::var("OPENROUTER_API_KEY").ok(); + std::env::set_var("OPENROUTER_API_KEY", ""); + + let mut config = AgentConfig::default(); + config.model_system.compaction = + crate::model_system::ModelRef::parse_spec("openrouter:anthropic/claude-3.5-haiku"); + let classified = classify_compaction_model(&config); + + match saved { + Some(value) => std::env::set_var("OPENROUTER_API_KEY", value), + None => std::env::remove_var("OPENROUTER_API_KEY"), + } + + match classified { + CompactionModel::Unavailable { label, reason } => { + assert_eq!(label, "openrouter:anthropic/claude-3.5-haiku"); + assert!(reason.contains("OPENROUTER_API_KEY"), "reason: {reason}"); + } + other => panic!("expected Unavailable, got {other:?}"), + } + } + + #[tokio::test] + async fn compaction_use_model_failure_surfaces_warning() { + // A dedicated compaction model whose summarize call fails must not + // silently degrade to the deterministic trim; the failure is surfaced. + let mut messages = vec![ + ChatMessage::System { + content: "sys".into(), + }, + ChatMessage::User { + content: "old request".into(), + }, + ChatMessage::Assistant { + content: Some("old answer".into()), + tool_calls: vec![], + }, + ChatMessage::User { + content: "current request".into(), + }, + ]; + // Port 9 (discard) refuses the connection, so summarize_transcript errors. + let backend = BackendDescriptor { + name: BackendName::Ollama, + base_url: "http://127.0.0.1:9/v1".into(), + api_key: "test".into(), + is_local: true, + openrouter: crate::backends::OpenRouterConfig::default(), + }; + let compaction = CompactionModel::Use { + backend: backend.clone(), + model: "compactor".into(), + label: "ollama:compactor".into(), + }; + let http = Client::new(); + let result = compact_messages_core(CompactRequest { + messages: &mut messages, + system_prompt: "sys", + tool_defs: &[], + keep_messages: 1, + limit_bytes: 1000, + // Large budget keeps this on the LLM-summary path (not tier-2 trim). + summarize_budget_bytes: 1_000_000, + http: &http, + backend: &backend, + model: "mock", + conversation_summary: None, + compaction: &compaction, + }) + .await + .expect("compaction runs"); + + assert_eq!(result.method, CompactMethod::DeterministicTrim); + let warning = result.warning.expect("failure surfaced as warning"); + assert!(warning.contains("ollama:compactor"), "warning: {warning}"); + assert!(warning.contains("failed"), "warning: {warning}"); + } + + #[test] + fn compaction_notice_appends_warning_when_present() { + let result = CompactResult { + compacted: true, + before_messages: 10, + after_messages: 4, + before_ratio: 0.9, + after_ratio: 0.3, + method: CompactMethod::LlmSummary, + conversation_summary: Some("summary".into()), + warning: Some("compaction model openrouter:x unavailable".into()), + }; + let line = compaction_notice_line(&result); + assert!(line.contains("compaction model openrouter:x unavailable")); + + let no_warning = CompactResult { + warning: None, + ..result + }; + assert!(!compaction_notice_line(&no_warning).contains("unavailable")); + } } diff --git a/src/hooks/state.rs b/src/hooks/state.rs index dc6a36e..043e65e 100644 --- a/src/hooks/state.rs +++ b/src/hooks/state.rs @@ -22,10 +22,8 @@ pub fn hook_state_file_path() -> Option { pub fn hook_state_file_path_from_env(xdg: Option<&str>, home: Option<&str>) -> Option { let base = if let Some(xdg) = xdg { PathBuf::from(xdg) - } else if let Some(home) = home { - PathBuf::from(home).join(".config") } else { - return None; + PathBuf::from(home?).join(".config") }; Some(base.join("small-harness").join("hooks-state.json")) } diff --git a/src/model_system.rs b/src/model_system.rs index 0109052..0de4e83 100644 --- a/src/model_system.rs +++ b/src/model_system.rs @@ -235,6 +235,10 @@ pub struct ModelSystemConfig { pub planner: Option, #[serde(default)] pub selector: Option, + /// Model used to summarize/compact the conversation transcript. When unset, + /// compaction inherits the main conversation model. + #[serde(default)] + pub compaction: Option, #[serde(default)] pub orchestrators: ModelTierSet, #[serde(default)] @@ -249,12 +253,17 @@ impl ModelSystemConfig { pub fn any_configured(&self) -> bool { self.planner.is_some() || self.selector.is_some() + || self.compaction.is_some() || self.orchestrators.any_configured() || self.coders.any_configured() || self.reviewers.any_configured() || self.security_reviewer.is_some() } + pub fn compaction(&self) -> Option<&ModelRef> { + self.compaction.as_ref() + } + pub fn coder(&self, complexity: TaskComplexity) -> Option<&ModelRef> { self.coders.get(complexity) } @@ -329,4 +338,30 @@ mod tests { Some("qwen2.5-coder:7b") ); } + + #[test] + fn compaction_model_is_configurable_and_detected() { + let empty = ModelSystemConfig::default(); + assert!(empty.compaction().is_none()); + + let configured = ModelSystemConfig { + compaction: ModelRef::parse_spec("openrouter:anthropic/claude-3.5-haiku"), + ..Default::default() + }; + assert!(configured.any_configured()); + let compaction = configured.compaction().expect("compaction set"); + assert_eq!(compaction.backend, BackendName::Openrouter); + assert_eq!(compaction.model, "anthropic/claude-3.5-haiku"); + } + + #[test] + fn compaction_model_round_trips_through_json() { + let json = + r#"{"compaction":{"backend":"openrouter","model":"anthropic/claude-3.5-haiku"}}"#; + let cfg: ModelSystemConfig = serde_json::from_str(json).expect("parse"); + assert_eq!( + cfg.compaction().map(|m| m.model.as_str()), + Some("anthropic/claude-3.5-haiku") + ); + } }