diff --git a/crates/buzz-agent/src/config.rs b/crates/buzz-agent/src/config.rs index afbda5379d..3e40112762 100644 --- a/crates/buzz-agent/src/config.rs +++ b/crates/buzz-agent/src/config.rs @@ -687,6 +687,161 @@ pub enum OpenAiApi { Auto, } +/// One provider the agent can send a turn to — a provider slot's full +/// coordinates. The primary lives in [`Config`] itself; [`Config::fallback`] +/// holds the ordered alternates tried when the primary cannot serve the turn. +/// +/// Each endpoint carries its own model deliberately: a fallback answers with +/// the model its own slot configures, never the primary's. Sending the +/// primary's model id to a different provider is a guaranteed 404. +#[derive(Debug, Clone, PartialEq)] +pub struct Endpoint { + pub provider: Provider, + pub api_key: String, + pub model: String, + pub base_url: String, + pub openai_api: OpenAiApi, +} + +impl Endpoint { + /// Stable identity for circuit-breaker bookkeeping. Keyed on the upstream + /// (provider + base URL) rather than the model: a quota wall or a dead host + /// applies to every model behind it, and keying per-model would make the + /// breaker re-learn the same outage for each one. + pub fn circuit_key(&self) -> String { + format!( + "{:?}|{}", + self.provider, + self.base_url.trim_end_matches('/') + ) + } + + /// Trailing-slash-insensitive identity of the wire destination. Two slots + /// with the same provider and base URL are the same upstream, so one + /// cannot stand in for the other when that upstream is down. + fn same_upstream_as(&self, other: &Self) -> bool { + self.provider == other.provider + && self.base_url.trim_end_matches('/') == other.base_url.trim_end_matches('/') + } +} + +/// A provider slot as read from the environment, before validation. Borrowed +/// so [`derive_fallback_chain`] stays pure and testable without mutating the +/// process environment (env-mutating tests race under a threaded runner). +#[derive(Debug, Clone, Copy)] +struct RawSlot<'a> { + /// Canonical id accepted in `BUZZ_AGENT_FALLBACK_PROVIDERS`. + id: &'a str, + provider: Provider, + api_key: Option<&'a str>, + model: Option<&'a str>, + base_url: &'a str, + openai_api: OpenAiApi, +} + +/// Auto-derivation order when `BUZZ_AGENT_FALLBACK_PROVIDERS` is unset: the +/// flat-rate Ollama Cloud slot first, then OpenRouter's free tier. The primary +/// is removed from whatever this produces. +const DEFAULT_FALLBACK_ORDER: [&str; 2] = ["openai", "openrouter"]; + +/// Map a `BUZZ_AGENT_FALLBACK_PROVIDERS` entry to a slot id, accepting the +/// same spellings as `BUZZ_AGENT_PROVIDER`. +fn canonical_slot_id(raw: &str) -> Result<&'static str, String> { + match raw { + "anthropic" => Ok("anthropic"), + "openai" | "openai-compat" => Ok("openai"), + "openrouter" => Ok("openrouter"), + // Known providers that cannot serve as a fallback. Databricks resolves + // its bearer through OAuth PKCE against a host-specific discovery + // document; there is no way to express a second one in the current env + // surface. Say so rather than silently dropping what the operator asked + // for and leaving them believing failover is armed. + "databricks" | "databricks_v2" | "databricks-v2" => Err(format!( + "config: BUZZ_AGENT_FALLBACK_PROVIDERS={raw} is not supported as a fallback \ + (use anthropic, openai, or openrouter)" + )), + other => Err(format!( + "config: BUZZ_AGENT_FALLBACK_PROVIDERS={other} not supported \ + (use anthropic, openai, openrouter, or none)" + )), + } +} + +/// Build the ordered failover chain. +/// +/// - `Some("none")` disables failover entirely — the pre-failover behavior. +/// - `Some(list)` takes the named slots in the given order. An unknown id is a +/// startup error, matching how `BUZZ_AGENT_PROVIDER` treats a typo. +/// - `None` auto-derives from [`DEFAULT_FALLBACK_ORDER`]. +/// +/// A candidate is dropped when its key or model is absent (nothing to call) or +/// when it targets the same upstream as `primary` (it would fail the same way). +/// Dropping is a warning, never an error: a missing fallback must not stop the +/// agent from starting on a working primary. +fn derive_fallback_chain( + requested: Option<&str>, + primary: &Endpoint, + slots: &[RawSlot<'_>], +) -> Result, String> { + let requested = requested.map(str::trim).filter(|s| !s.is_empty()); + if requested.is_some_and(|r| r.eq_ignore_ascii_case("none")) { + return Ok(Vec::new()); + } + + let (ids, explicit) = match requested { + Some(list) => { + let mut ids = Vec::new(); + for entry in list.split(',').map(str::trim).filter(|s| !s.is_empty()) { + ids.push(canonical_slot_id(&entry.to_ascii_lowercase())?); + } + (ids, true) + } + None => (DEFAULT_FALLBACK_ORDER.to_vec(), false), + }; + + let mut chain: Vec = Vec::new(); + for id in ids { + let Some(slot) = slots.iter().find(|s| s.id == id) else { + continue; + }; + let (Some(api_key), Some(model)) = ( + slot.api_key.map(str::trim).filter(|s| !s.is_empty()), + slot.model.map(str::trim).filter(|s| !s.is_empty()), + ) else { + // Only worth saying out loud when the operator named it — an + // absent slot under auto-derivation is the ordinary case. + if explicit { + tracing::warn!( + provider = id, + "config: fallback provider skipped — its API key or model is not set" + ); + } + continue; + }; + let candidate = Endpoint { + provider: slot.provider, + api_key: api_key.to_owned(), + model: model.to_owned(), + base_url: slot.base_url.to_owned(), + openai_api: slot.openai_api, + }; + if candidate.same_upstream_as(primary) { + if explicit { + tracing::warn!( + provider = id, + "config: fallback provider skipped — same upstream as the primary" + ); + } + continue; + } + if chain.iter().any(|e| e.same_upstream_as(&candidate)) { + continue; + } + chain.push(candidate); + } + Ok(chain) +} + #[derive(Debug, Clone)] pub struct Config { pub provider: Provider, @@ -737,6 +892,11 @@ pub struct Config { pub api_key: String, pub model: String, pub base_url: String, + /// Ordered alternates tried when the primary cannot serve a turn (quota, + /// credentials, upstream 5xx, transport). Empty means no failover — the + /// single-provider behavior. Derived from the provider slots present in + /// the environment; override with `BUZZ_AGENT_FALLBACK_PROVIDERS`. + pub fallback: Vec, pub anthropic_api_version: String, /// OpenAI endpoint selection. See [`OpenAiApi`]. pub openai_api: OpenAiApi, @@ -822,6 +982,62 @@ impl Config { OpenAiApi::Chat, // OpenRouter uses Chat Completions only ), }; + // Every provider slot the environment defines, read once so the chain + // derivation stays pure. Each slot keeps its OWN model var — the + // `BUZZ_AGENT_MODEL` override applies to the primary only, or a + // cutover would send the primary's model id to a provider that has + // never heard of it. `OPENAI_COMPAT_API` is parsed leniently here for + // the same reason it is read conditionally above: a stray value must + // not break a deployment whose primary is not OpenAI. + let anthropic_key = env("ANTHROPIC_API_KEY"); + let anthropic_model = env("ANTHROPIC_MODEL"); + let anthropic_base = env_or("ANTHROPIC_BASE_URL", "https://api.anthropic.com"); + let openai_key = env("OPENAI_COMPAT_API_KEY"); + let openai_model = env("OPENAI_COMPAT_MODEL"); + let openai_base = env_or("OPENAI_COMPAT_BASE_URL", "https://api.openai.com/v1"); + let openai_slot_api = + parse_openai_api(env("OPENAI_COMPAT_API").as_deref()).unwrap_or(OpenAiApi::Auto); + let openrouter_key = env("OPENROUTER_API_KEY"); + let openrouter_model = env("OPENROUTER_MODEL"); + let openrouter_base = env_or("OPENROUTER_BASE_URL", "https://openrouter.ai/api/v1"); + let primary_endpoint = Endpoint { + provider, + api_key: api_key.clone(), + model: model.clone(), + base_url: base_url.clone(), + openai_api, + }; + let fallback = derive_fallback_chain( + env("BUZZ_AGENT_FALLBACK_PROVIDERS").as_deref(), + &primary_endpoint, + &[ + RawSlot { + id: "openai", + provider: Provider::OpenAi, + api_key: openai_key.as_deref(), + model: openai_model.as_deref(), + base_url: &openai_base, + openai_api: openai_slot_api, + }, + RawSlot { + id: "openrouter", + provider: Provider::OpenRouter, + api_key: openrouter_key.as_deref(), + model: openrouter_model.as_deref(), + base_url: &openrouter_base, + openai_api: OpenAiApi::Chat, + }, + RawSlot { + id: "anthropic", + provider: Provider::Anthropic, + api_key: anthropic_key.as_deref(), + model: anthropic_model.as_deref(), + base_url: &anthropic_base, + openai_api: OpenAiApi::Auto, + }, + ], + )?; + let system_prompt = match (env("BUZZ_AGENT_SYSTEM_PROMPT"), env("BUZZ_AGENT_SYSTEM_PROMPT_FILE")) { (Some(_), Some(_)) => return Err( "config: BUZZ_AGENT_SYSTEM_PROMPT and BUZZ_AGENT_SYSTEM_PROMPT_FILE are mutually exclusive".into()), @@ -835,6 +1051,7 @@ impl Config { api_key, model, base_url, + fallback, anthropic_api_version: env_or("ANTHROPIC_API_VERSION", "2023-06-01"), openai_api, prefer_mesh_for_auto: parse_env("BUZZ_AGENT_PREFER_MESH_FOR_AUTO", 0u8)? != 0, @@ -882,6 +1099,9 @@ impl Config { provider, api_key, base_url, + // Catalog discovery targets one explicit endpoint; there is + // nothing to fail over to. + fallback: Vec::new(), model: String::new(), system_prompt: String::new(), anthropic_api_version: "2023-06-01".into(), @@ -912,6 +1132,37 @@ impl Config { } } + /// This config's own provider slot as an [`Endpoint`] — the first link in + /// the failover chain. + pub fn primary_endpoint(&self) -> Endpoint { + Endpoint { + provider: self.provider, + api_key: self.api_key.clone(), + model: self.model.clone(), + base_url: self.base_url.clone(), + openai_api: self.openai_api, + } + } + + /// This config aimed at `endpoint` instead of its own provider slot. + /// + /// Everything that is not a provider coordinate — prompts, limits, + /// timeouts, effort — is preserved, so a failover retry differs from the + /// original attempt only in where it is sent. + pub fn with_endpoint(&self, endpoint: &Endpoint) -> Self { + Self { + provider: endpoint.provider, + api_key: endpoint.api_key.clone(), + model: endpoint.model.clone(), + base_url: endpoint.base_url.clone(), + openai_api: endpoint.openai_api, + // The chain is walked one level up. Clearing it here keeps an + // attempt from recursively failing over inside itself. + fallback: Vec::new(), + ..self.clone() + } + } + fn validate(&self) -> Result<(), String> { const MIN_HISTORY_BYTES: usize = 4096; const MIN_LINE_BYTES: usize = 1024; @@ -2763,4 +3014,207 @@ mod tests { let err = resolve_provider(Some("openrouter"), None, None, None).unwrap_err(); assert!(err.contains("OPENROUTER_API_KEY")); } + + // ---- failover chain derivation ---------------------------------------- + // + // Modelled on the real estate deployment that motivated failover: z.ai's + // GLM through the `anthropic` slot as primary, with Ollama Cloud and + // OpenRouter available as alternates. + + const OLLAMA: &str = "https://ollama.com/v1"; + const OPENROUTER: &str = "https://openrouter.ai/api/v1"; + const ZAI: &str = "https://api.z.ai/api/anthropic"; + + fn slots() -> Vec> { + vec![ + RawSlot { + id: "openai", + provider: Provider::OpenAi, + api_key: Some("ollama-key"), + model: Some("qwen3.5:397b"), + base_url: OLLAMA, + openai_api: OpenAiApi::Chat, + }, + RawSlot { + id: "openrouter", + provider: Provider::OpenRouter, + api_key: Some("openrouter-key"), + model: Some("nvidia/nemotron:free"), + base_url: OPENROUTER, + openai_api: OpenAiApi::Chat, + }, + RawSlot { + id: "anthropic", + provider: Provider::Anthropic, + api_key: Some("zai-key"), + model: Some("glm-5.2"), + base_url: ZAI, + openai_api: OpenAiApi::Auto, + }, + ] + } + + fn endpoint(provider: Provider, base_url: &str, model: &str) -> Endpoint { + Endpoint { + provider, + api_key: "key".into(), + model: model.into(), + base_url: base_url.into(), + openai_api: OpenAiApi::Auto, + } + } + + fn zai_primary() -> Endpoint { + endpoint(Provider::Anthropic, ZAI, "glm-5.2") + } + + fn ids(chain: &[Endpoint]) -> Vec { + chain.iter().map(|e| e.base_url.clone()).collect() + } + + #[test] + fn fallback_chain_auto_derives_ollama_then_openrouter() { + let chain = derive_fallback_chain(None, &zai_primary(), &slots()).unwrap(); + assert_eq!(ids(&chain), vec![OLLAMA, OPENROUTER]); + } + + #[test] + fn fallback_chain_none_disables_failover() { + for raw in ["none", "NONE", " none "] { + let chain = derive_fallback_chain(Some(raw), &zai_primary(), &slots()).unwrap(); + assert!(chain.is_empty(), "{raw:?} must disable failover"); + } + } + + #[test] + fn fallback_chain_honors_an_explicit_order() { + let chain = + derive_fallback_chain(Some("openrouter, openai"), &zai_primary(), &slots()).unwrap(); + assert_eq!(ids(&chain), vec![OPENROUTER, OLLAMA]); + } + + #[test] + fn fallback_chain_accepts_the_openai_compat_spelling() { + let chain = derive_fallback_chain(Some("openai-compat"), &zai_primary(), &slots()).unwrap(); + assert_eq!(ids(&chain), vec![OLLAMA]); + } + + /// The trap the estate config hits: the `anthropic` slot IS the primary + /// (z.ai), so naming it as a fallback buys nothing — the same key, the + /// same URL, the same exhausted quota. + #[test] + fn fallback_chain_skips_the_primarys_own_upstream() { + let chain = derive_fallback_chain(Some("anthropic"), &zai_primary(), &slots()).unwrap(); + assert!( + chain.is_empty(), + "an endpoint identical to the primary cannot stand in for it" + ); + } + + #[test] + fn fallback_chain_skips_the_primary_during_auto_derivation() { + // Primary is Ollama itself — auto-derivation must not list it twice. + let primary = endpoint(Provider::OpenAi, OLLAMA, "qwen3.5:397b"); + let chain = derive_fallback_chain(None, &primary, &slots()).unwrap(); + assert_eq!(ids(&chain), vec![OPENROUTER]); + } + + #[test] + fn fallback_chain_matching_ignores_a_trailing_slash() { + let primary = endpoint(Provider::OpenAi, &format!("{OLLAMA}/"), "qwen3.5:397b"); + let chain = derive_fallback_chain(None, &primary, &slots()).unwrap(); + assert_eq!( + ids(&chain), + vec![OPENROUTER], + "a trailing slash is the same upstream, not a second one" + ); + } + + #[test] + fn fallback_chain_carries_each_slots_own_model() { + let chain = derive_fallback_chain(None, &zai_primary(), &slots()).unwrap(); + assert_eq!( + chain[0].model, "qwen3.5:397b", + "a fallback must answer with its own model — the primary's \ + `glm-5.2` means nothing to Ollama" + ); + assert_eq!(chain[1].model, "nvidia/nemotron:free"); + } + + #[test] + fn fallback_chain_skips_a_slot_with_no_key_or_model() { + let mut incomplete = slots(); + incomplete[0].api_key = None; + incomplete[1].model = Some(" "); + let chain = derive_fallback_chain(None, &zai_primary(), &incomplete).unwrap(); + assert!( + chain.is_empty(), + "a slot with nothing to call must not enter the chain" + ); + } + + #[test] + fn fallback_chain_dedupes_repeated_entries() { + let chain = + derive_fallback_chain(Some("openai,openai,openrouter"), &zai_primary(), &slots()) + .unwrap(); + assert_eq!(ids(&chain), vec![OLLAMA, OPENROUTER]); + } + + #[test] + fn fallback_chain_rejects_an_unknown_provider_id() { + let err = + derive_fallback_chain(Some("openrouter,typo"), &zai_primary(), &slots()).unwrap_err(); + assert!(err.contains("typo"), "{err}"); + assert!(err.contains("BUZZ_AGENT_FALLBACK_PROVIDERS"), "{err}"); + } + + /// Databricks resolves its bearer through host-specific OAuth PKCE, which + /// the env surface cannot express twice. Saying so beats silently dropping + /// it and leaving the operator believing failover is armed. + #[test] + fn fallback_chain_rejects_databricks_with_a_clear_message() { + let err = derive_fallback_chain(Some("databricks"), &zai_primary(), &slots()).unwrap_err(); + assert!(err.contains("not supported as a fallback"), "{err}"); + } + + #[test] + fn fallback_chain_is_empty_when_no_other_slot_is_configured() { + let only_primary = vec![slots()[2]]; + let chain = derive_fallback_chain(None, &zai_primary(), &only_primary).unwrap(); + assert!(chain.is_empty()); + } + + #[test] + fn circuit_key_is_stable_across_trailing_slashes_and_models() { + let a = endpoint(Provider::OpenAi, OLLAMA, "model-a"); + let b = endpoint(Provider::OpenAi, &format!("{OLLAMA}/"), "model-b"); + assert_eq!( + a.circuit_key(), + b.circuit_key(), + "one upstream is one circuit, whatever model is asked of it" + ); + assert_ne!( + a.circuit_key(), + endpoint(Provider::OpenRouter, OPENROUTER, "model-a").circuit_key() + ); + } + + #[test] + fn with_endpoint_swaps_the_provider_and_keeps_everything_else() { + let mut cfg = Config::for_discovery(Provider::Anthropic, "zai-key".into(), ZAI.into()); + cfg.max_output_tokens = 4096; + cfg.system_prompt = "keep me".into(); + + let swapped = cfg.with_endpoint(&endpoint(Provider::OpenAi, OLLAMA, "qwen3.5:397b")); + assert_eq!(swapped.provider, Provider::OpenAi); + assert_eq!(swapped.base_url, OLLAMA); + assert_eq!(swapped.model, "qwen3.5:397b"); + assert_eq!(swapped.system_prompt, "keep me"); + assert_eq!(swapped.max_output_tokens, 4096); + assert!( + swapped.fallback.is_empty(), + "an attempt must not recursively fail over inside itself" + ); + } } diff --git a/crates/buzz-agent/src/health.rs b/crates/buzz-agent/src/health.rs new file mode 100644 index 0000000000..794353eab8 --- /dev/null +++ b/crates/buzz-agent/src/health.rs @@ -0,0 +1,281 @@ +//! In-process circuit breaker for the provider failover chain. +//! +//! Without it, every turn re-discovers an outage the hard way: the primary is +//! tried, waits out its retries, and only then cuts over. A quota wall lasts +//! days, so that cost is paid on every turn for as long as it lasts. The +//! breaker remembers, and sends the next turn straight to a provider that +//! works. +//! +//! State machine per endpoint: +//! +//! ```text +//! closed --2 consecutive cutover failures--> open (cooldown starts) +//! open, now < open_until blocked: skipped by the chain +//! open, now >= open_until half-open: one probe allowed +//! probe fails cooldown doubles, stays open +//! probe succeeds closed, counters reset +//! ``` +//! +//! Cooldown starts at 60s and doubles per failed probe to a 15min cap. +//! Quota and auth failures start at 5min instead: a weekly quota window or a +//! revoked key does not heal inside a minute, so probing that fast is waste. +//! +//! Deliberately in-process and not persisted. The state is a latency +//! optimization, not a source of truth — a restarted agent re-learning an +//! outage costs one slow turn, which is not worth a file to corrupt. It is +//! also reactive by construction: a circuit opens only after real traffic has +//! failed, so it cannot predict an exhausted quota before anything tries. +//! +//! Fail-soft throughout: the breaker never reports an endpoint as down unless +//! it has seen it fail, and the chain walk ignores it entirely when every +//! candidate is blocked. It must never be the reason a turn has nowhere to go. + +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use crate::types::CutoverKind; + +/// Consecutive cutover-class failures before an endpoint is taken out of +/// rotation. Two, not one: a single 503 is noise, and opening on it would +/// bounce a healthy provider out over a blip. +const FAILURE_THRESHOLD: u32 = 2; + +/// Opening cooldown for failures that usually clear quickly (5xx, transport). +const BASE_COOLDOWN: Duration = Duration::from_secs(60); + +/// Opening cooldown for failures that do not (quota windows, bad credentials). +const SLOW_HEAL_COOLDOWN: Duration = Duration::from_secs(5 * 60); + +/// Ceiling on the doubling, so a long outage still gets probed periodically. +const MAX_COOLDOWN: Duration = Duration::from_secs(15 * 60); + +#[derive(Debug)] +struct Circuit { + consecutive_failures: u32, + /// `None` while closed. `Some(t)` once opened — the circuit is blocking + /// until `t`, and half-open (one probe allowed) from `t` onward. + open_until: Option, + cooldown: Duration, +} + +impl Circuit { + fn closed() -> Self { + Self { + consecutive_failures: 0, + open_until: None, + cooldown: BASE_COOLDOWN, + } + } +} + +/// Per-endpoint availability memory shared across the turns of one agent +/// process. Keyed by [`Endpoint::circuit_key`](crate::config::Endpoint::circuit_key). +#[derive(Debug, Default)] +pub struct Breaker { + circuits: Mutex>, +} + +impl Breaker { + pub fn new() -> Self { + Self::default() + } + + /// Recover the guard rather than propagating a poisoned lock: a panic in + /// some other turn's bookkeeping must not take the failover path down with + /// it. Worst case the map holds slightly stale counters. + fn circuits(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.circuits.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// Whether `key` is currently blocked. False for an endpoint that has + /// never failed, and false once the cooldown has elapsed (the half-open + /// probe). + pub fn is_open(&self, key: &str) -> bool { + self.is_open_at(key, Instant::now()) + } + + fn is_open_at(&self, key: &str, now: Instant) -> bool { + self.circuits() + .get(key) + .and_then(|c| c.open_until) + .is_some_and(|until| now < until) + } + + /// Record a cutover-class failure. Never call this for a stop-class error + /// (a 400): a malformed request says nothing about whether the endpoint is + /// up, and would take a healthy provider out of rotation. + pub fn record_failure(&self, key: &str, kind: CutoverKind) { + self.record_failure_at(key, kind, Instant::now()); + } + + fn record_failure_at(&self, key: &str, kind: CutoverKind, now: Instant) { + let mut circuits = self.circuits(); + let circuit = circuits + .entry(key.to_owned()) + .or_insert_with(Circuit::closed); + + if circuit.open_until.is_some() { + // A failed probe, or an attempt forced through because every + // candidate was blocked. Back off further; already open, so this + // is not a transition. + circuit.cooldown = (circuit.cooldown * 2).min(MAX_COOLDOWN); + circuit.open_until = Some(now + circuit.cooldown); + return; + } + + circuit.consecutive_failures += 1; + if circuit.consecutive_failures < FAILURE_THRESHOLD { + return; + } + circuit.cooldown = if kind.is_slow_to_heal() { + SLOW_HEAL_COOLDOWN + } else { + BASE_COOLDOWN + }; + circuit.open_until = Some(now + circuit.cooldown); + tracing::warn!( + endpoint = key, + kind = kind.as_str(), + cooldown_secs = circuit.cooldown.as_secs(), + failures = circuit.consecutive_failures, + "llm: endpoint taken out of rotation" + ); + } + + /// Record a success, closing the circuit and clearing the backoff. + pub fn record_success(&self, key: &str) { + let mut circuits = self.circuits(); + if let Some(circuit) = circuits.get(key) { + if circuit.open_until.is_some() { + tracing::info!(endpoint = key, "llm: endpoint recovered"); + } + circuits.remove(key); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const KEY: &str = "anthropic|https://api.z.ai/api/anthropic"; + + #[test] + fn unknown_endpoint_is_never_blocked() { + let breaker = Breaker::new(); + assert!(!breaker.is_open(KEY)); + } + + #[test] + fn one_failure_does_not_open_the_circuit() { + let breaker = Breaker::new(); + breaker.record_failure(KEY, CutoverKind::Server); + assert!( + !breaker.is_open(KEY), + "a single blip must not take an endpoint out of rotation" + ); + } + + #[test] + fn threshold_failures_open_the_circuit() { + let breaker = Breaker::new(); + breaker.record_failure(KEY, CutoverKind::Server); + breaker.record_failure(KEY, CutoverKind::Server); + assert!(breaker.is_open(KEY)); + } + + #[test] + fn quota_opens_with_the_slow_heal_cooldown() { + let breaker = Breaker::new(); + let now = Instant::now(); + breaker.record_failure_at(KEY, CutoverKind::Quota, now); + breaker.record_failure_at(KEY, CutoverKind::Quota, now); + + // Still blocked well past the 60s a transport failure would have used. + assert!(breaker.is_open_at(KEY, now + Duration::from_secs(61))); + assert!(breaker.is_open_at(KEY, now + SLOW_HEAL_COOLDOWN - Duration::from_secs(1))); + // Half-open once the 5min window elapses. + assert!(!breaker.is_open_at(KEY, now + SLOW_HEAL_COOLDOWN)); + } + + #[test] + fn server_failures_open_with_the_short_cooldown() { + let breaker = Breaker::new(); + let now = Instant::now(); + breaker.record_failure_at(KEY, CutoverKind::Server, now); + breaker.record_failure_at(KEY, CutoverKind::Server, now); + + assert!(breaker.is_open_at(KEY, now + Duration::from_secs(59))); + assert!(!breaker.is_open_at(KEY, now + BASE_COOLDOWN)); + } + + #[test] + fn a_failed_probe_doubles_the_cooldown() { + let breaker = Breaker::new(); + let now = Instant::now(); + breaker.record_failure_at(KEY, CutoverKind::Server, now); + breaker.record_failure_at(KEY, CutoverKind::Server, now); + + // Cooldown elapses, the probe is allowed, and it fails too. + let probe_at = now + BASE_COOLDOWN; + assert!(!breaker.is_open_at(KEY, probe_at)); + breaker.record_failure_at(KEY, CutoverKind::Server, probe_at); + + // Next window is 120s, not another 60s. + assert!(breaker.is_open_at(KEY, probe_at + Duration::from_secs(119))); + assert!(!breaker.is_open_at(KEY, probe_at + Duration::from_secs(120))); + } + + #[test] + fn cooldown_doubling_is_capped() { + let breaker = Breaker::new(); + let mut now = Instant::now(); + breaker.record_failure_at(KEY, CutoverKind::Server, now); + breaker.record_failure_at(KEY, CutoverKind::Server, now); + + // Fail probe after probe; the window must never exceed the cap. + for _ in 0..12 { + now += MAX_COOLDOWN; + breaker.record_failure_at(KEY, CutoverKind::Server, now); + } + assert!(!breaker.is_open_at(KEY, now + MAX_COOLDOWN)); + } + + #[test] + fn success_closes_an_open_circuit() { + let breaker = Breaker::new(); + breaker.record_failure(KEY, CutoverKind::Quota); + breaker.record_failure(KEY, CutoverKind::Quota); + assert!(breaker.is_open(KEY)); + + breaker.record_success(KEY); + assert!(!breaker.is_open(KEY)); + } + + #[test] + fn success_resets_the_failure_count() { + let breaker = Breaker::new(); + breaker.record_failure(KEY, CutoverKind::Server); + breaker.record_success(KEY); + // The earlier failure must not combine with this one to hit the + // threshold — otherwise a slow trickle of unrelated blips eventually + // opens the circuit on a healthy endpoint. + breaker.record_failure(KEY, CutoverKind::Server); + assert!(!breaker.is_open(KEY)); + } + + #[test] + fn circuits_are_independent_per_endpoint() { + let breaker = Breaker::new(); + let other = "openrouter|https://openrouter.ai/api/v1"; + breaker.record_failure(KEY, CutoverKind::Quota); + breaker.record_failure(KEY, CutoverKind::Quota); + + assert!(breaker.is_open(KEY)); + assert!( + !breaker.is_open(other), + "one provider's quota wall must not bench the rest of the chain" + ); + } +} diff --git a/crates/buzz-agent/src/lib.rs b/crates/buzz-agent/src/lib.rs index 9a45bf4c98..3f092b3fb7 100644 --- a/crates/buzz-agent/src/lib.rs +++ b/crates/buzz-agent/src/lib.rs @@ -5,6 +5,7 @@ mod builtin; pub mod catalog; pub mod config; mod handoff; +mod health; mod hints; mod llm; mod mcp; diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs index 73c7e1faf2..c20090416e 100644 --- a/crates/buzz-agent/src/llm.rs +++ b/crates/buzz-agent/src/llm.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; @@ -10,10 +10,12 @@ use tokio::time::Instant; use crate::auth::{PkceOAuthConfig, PkceOAuthTokenSource, StaticTokenSource, TokenSource}; use crate::config::{ is_openai_host, normalize_effort_for_anthropic_route, normalize_effort_for_openai_route, - Config, OpenAiApi, Provider, ThinkingEffort, + Config, Endpoint, OpenAiApi, Provider, ThinkingEffort, }; +use crate::health::Breaker; use crate::types::{ - AgentError, HistoryItem, LlmResponse, ProviderStop, ToolCall, ToolDef, ToolResultContent, + AgentError, CutoverKind, HistoryItem, LlmResponse, ProviderStop, ToolCall, ToolDef, + ToolResultContent, }; /// Databricks OAuth client_id — the public Databricks-published CLI client. @@ -85,6 +87,59 @@ enum DatabricksV2Route { MlflowChatCompletions, } +/// One endpoint to try, fully resolved: the config aimed at it, the model to +/// ask it for, and the circuit key its outcome is recorded against. +struct Attempt { + key: String, + provider: Provider, + cfg: Config, + model: String, +} + +impl Attempt { + /// Record that a fallback rescued a call the primary could not serve. + /// `index` is the position in the chain, so 0 (the primary) says nothing — + /// the ordinary case should not produce log noise. + fn note_rescue(&self, index: usize, what: &str) { + if index > 0 { + tracing::info!( + provider = ?self.provider, + model = %self.model, + "llm: {what} served by a fallback provider" + ); + } + } + + fn note_failure(&self, kind: CutoverKind, error: &AgentError) { + tracing::warn!( + provider = ?self.provider, + model = %self.model, + kind = kind.as_str(), + error = %error, + "llm: endpoint failed" + ); + } +} + +/// The error to surface once every endpoint in the chain has failed. +/// +/// Reworded only when more than one endpoint was in play, so a +/// single-provider deployment's errors read exactly as they did before +/// failover existed. +fn chain_exhausted(last_error: Option, tried: usize) -> AgentError { + match last_error { + Some(AgentError::LlmUnavailable { kind, detail }) if tried > 1 => { + AgentError::LlmUnavailable { + kind, + detail: format!("all {tried} providers failed; last: {detail}"), + } + } + Some(error) => error, + // Unreachable — the chain always holds at least the primary. + None => AgentError::Llm("no provider endpoint configured".into()), + } +} + pub struct Llm { http: Client, /// One-shot sticky flag: set when a Chat Completions request comes @@ -101,7 +156,23 @@ pub struct Llm { /// (the `DATABRICKS_TOKEN` env var); a refreshable PKCE engine for /// Databricks otherwise. Anthropic doesn't use this — it always /// reads `cfg.api_key` directly because the API expects `x-api-key`. + /// + /// This covers the PRIMARY endpoint only. See [`Llm::auth_for`]. auth: Arc, + /// Circuit key of the endpoint `auth` belongs to, so a request can tell + /// whether it is the primary before reaching for the fallback cache. + primary_key: String, + /// Token sources for fallback endpoints, built on first use and cached. + /// + /// A cutover cannot reuse `auth`: it holds the *primary's* credential, and + /// sending that to a different provider's base URL is an instant 401. They + /// are cached rather than rebuilt per turn because a sustained outage + /// means every turn cuts over, and rebuilding would repeat the work on + /// each one. + fallback_auth: std::sync::Mutex>>, + /// Which endpoints are known to be down, so a turn skips them instead of + /// re-discovering the outage. See [`crate::health`]. + breaker: Breaker, } impl Llm { @@ -117,9 +188,48 @@ impl Llm { auto_upgraded: AtomicBool::new(false), mesh_auto_state: Mutex::new(MeshAutoState::default()), auth, + primary_key: cfg.primary_endpoint().circuit_key(), + fallback_auth: std::sync::Mutex::new(HashMap::new()), + breaker: Breaker::new(), }) } + /// The token source for whichever endpoint `cfg` points at. + /// + /// Returns the primary's source unchanged in the common case. On a + /// cutover, `cfg` is a [`Config::with_endpoint`] clone aimed elsewhere, and + /// this mints (once) and caches that endpoint's own source. Getting this + /// wrong is silent and total: the request goes to the fallback's URL + /// carrying the primary's key, and every cutover 401s. + fn auth_for(&self, cfg: &Config) -> Result, AgentError> { + let key = cfg.primary_endpoint().circuit_key(); + if key == self.primary_key { + return Ok(Arc::clone(&self.auth)); + } + let mut cache = self + .fallback_auth + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(existing) = cache.get(&key) { + return Ok(Arc::clone(existing)); + } + let source = build_token_source(cfg)?; + cache.insert(key, Arc::clone(&source)); + Ok(source) + } + + /// Run one turn, failing over to the next configured provider when the + /// current one cannot serve it. + /// + /// The chain is the primary followed by [`Config::fallback`]. Endpoints the + /// circuit breaker has benched are skipped — unless that would leave + /// nothing to try, in which case the whole chain is attempted anyway. Stale + /// breaker state must never be the reason a turn has nowhere to go. + /// + /// A stop-class error (malformed request, unknown model) returns + /// immediately. Those fail identically on every provider, so walking the + /// chain would only reach the same answer more slowly while burning quota + /// on the way. pub async fn complete( &self, cfg: &Config, @@ -127,6 +237,75 @@ impl Llm { history: &[HistoryItem], tools: &[ToolDef], effective_model: &str, + ) -> Result { + let chain = self.attempt_chain(cfg, effective_model); + let total = chain.len(); + let mut last_error: Option = None; + for (index, attempt) in chain.into_iter().enumerate() { + match self + .complete_once(&attempt.cfg, system_prompt, history, tools, &attempt.model) + .await + { + Ok(response) => { + self.breaker.record_success(&attempt.key); + attempt.note_rescue(index, "turn"); + return Ok(response); + } + Err(error) => { + let Some(kind) = cutover_kind(&error) else { + // Stop-class: the same request fails the same way on + // every provider. Surface it rather than burn the chain. + return Err(error); + }; + self.breaker.record_failure(&attempt.key, kind); + attempt.note_failure(kind, &error); + last_error = Some(error); + } + } + } + Err(chain_exhausted(last_error, total)) + } + + /// The endpoints to try for this call, in order, each paired with the + /// config aimed at it. + /// + /// The primary honors the caller's effective model — a `session/set_model` + /// override arrives that way. Each fallback uses its own slot's model + /// instead: the primary's model id means nothing to a different provider. + /// + /// Endpoints the breaker has benched are dropped, unless that would leave + /// nothing to try — then the whole chain comes back. Stale breaker state + /// must never be the reason a call has nowhere to go. + fn attempt_chain(&self, cfg: &Config, effective_model: &str) -> Vec { + let mut candidates: Vec<(Endpoint, &str)> = Vec::with_capacity(1 + cfg.fallback.len()); + candidates.push((cfg.primary_endpoint(), effective_model)); + for endpoint in &cfg.fallback { + candidates.push((endpoint.clone(), endpoint.model.as_str())); + } + let any_live = candidates + .iter() + .any(|(endpoint, _)| !self.breaker.is_open(&endpoint.circuit_key())); + candidates + .into_iter() + .filter(|(endpoint, _)| !any_live || !self.breaker.is_open(&endpoint.circuit_key())) + .map(|(endpoint, model)| Attempt { + key: endpoint.circuit_key(), + provider: endpoint.provider, + cfg: cfg.with_endpoint(&endpoint), + model: model.to_owned(), + }) + .collect() + } + + /// One turn against exactly one endpoint. No failover: `cfg` names the + /// provider, and `effective_model` is that provider's own model. + async fn complete_once( + &self, + cfg: &Config, + system_prompt: &str, + history: &[HistoryItem], + tools: &[ToolDef], + effective_model: &str, ) -> Result { let effort = cfg.thinking_effort; let result = match cfg.provider { @@ -235,10 +414,20 @@ impl Llm { AgentError::LlmModelNotFound(s) => { AgentError::LlmModelNotFound(format!("({effective_model}) {s}")) } + AgentError::LlmUnavailable { kind, detail } => AgentError::LlmUnavailable { + kind, + detail: format!("({effective_model}) {detail}"), + }, other => other, }) } + /// Summarize, failing over exactly as [`Llm::complete`] does. + /// + /// This is not a nice-to-have. Summarization drives handoff and context + /// compaction, so a primary that cannot serve it strands a long + /// conversation at the context ceiling with no way to continue — the same + /// outage the completion path exists to survive, arriving by another door. pub async fn summarize( &self, cfg: &Config, @@ -246,6 +435,47 @@ impl Llm { user_prompt: &str, max_output_tokens: u32, effective_model: &str, + ) -> Result { + let chain = self.attempt_chain(cfg, effective_model); + let total = chain.len(); + let mut last_error: Option = None; + for (index, attempt) in chain.into_iter().enumerate() { + match self + .summarize_once( + &attempt.cfg, + system_prompt, + user_prompt, + max_output_tokens, + &attempt.model, + ) + .await + { + Ok(text) => { + self.breaker.record_success(&attempt.key); + attempt.note_rescue(index, "summary"); + return Ok(text); + } + Err(error) => { + let Some(kind) = cutover_kind(&error) else { + return Err(error); + }; + self.breaker.record_failure(&attempt.key, kind); + attempt.note_failure(kind, &error); + last_error = Some(error); + } + } + } + Err(chain_exhausted(last_error, total)) + } + + /// One summary against exactly one endpoint. No failover. + async fn summarize_once( + &self, + cfg: &Config, + system_prompt: &str, + user_prompt: &str, + max_output_tokens: u32, + effective_model: &str, ) -> Result { match cfg.provider { Provider::Anthropic => { @@ -493,8 +723,17 @@ impl Llm { async fn observe_mesh_virtual_model(&self, cfg: &Config) -> MeshCatalogObservation { let url = format!("{}/models", cfg.base_url.trim_end_matches('/')); - let bearer = match self.auth.bearer().await { - Ok(bearer) => bearer, + let bearer = match self.auth_for(cfg) { + Ok(auth) => match auth.bearer().await { + Ok(bearer) => bearer, + Err(error) => { + tracing::debug!( + %error, + "relay-mesh auto: catalog auth unavailable; preserving last confirmed route" + ); + return MeshCatalogObservation::Unknown; + } + }, Err(error) => { tracing::debug!( %error, @@ -649,7 +888,8 @@ impl Llm { // rejection can never suppress a later turn's legitimate retry. Both // statuses map to `LlmAuth` in `post`: a 403 is indistinguishable from // an expired-token 403 here, so we refresh once and let it propagate. - let mut bearer = self.auth.bearer().await.map_err(PostError::from)?; + let auth = self.auth_for(cfg).map_err(PostError::from)?; + let mut bearer = auth.bearer().await.map_err(PostError::from)?; let mut refreshed = false; loop { match post( @@ -663,11 +903,7 @@ impl Llm { { Err(PostError::Agent(AgentError::LlmAuth(_))) if !refreshed => { refreshed = true; - bearer = self - .auth - .refresh_now(&bearer) - .await - .map_err(PostError::from)?; + bearer = auth.refresh_now(&bearer).await.map_err(PostError::from)?; } result => return result, } @@ -676,13 +912,14 @@ impl Llm { async fn post_openrouter(&self, cfg: &Config, body: &Value) -> Result { let url = format!("{}/chat/completions", cfg.base_url.trim_end_matches('/')); - let mut bearer = self.auth.bearer().await?; + let auth = self.auth_for(cfg)?; + let mut bearer = auth.bearer().await?; let mut refreshed = false; loop { match openrouter_post(&self.http, &url, body, &bearer).await { Err(AgentError::LlmAuth(_)) if !refreshed => { refreshed = true; - let new_bearer = self.auth.refresh_now(&bearer).await?; + let new_bearer = auth.refresh_now(&bearer).await?; // A static key refreshes to itself — a byte-identical retry // would be a guaranteed duplicate request against a key the // server just rejected. Fail terminal immediately; the retry @@ -1706,15 +1943,25 @@ fn is_retryable_transport_error(e: &reqwest::Error) -> bool { e.is_timeout() || e.is_connect() || e.is_request() } -/// Build the terminal `AgentError::Llm` for a `post()` exit that has given up -/// retrying — persistent retryable status, transport failure, or a body-read -/// break. `detail` carries the specific cause (status/body, or the transport -/// error text); `elapsed` and `attempts` are the cumulative cost of every -/// attempt made on this call, including the retries that failed before this -/// one. When cumulative time crosses `STALL_NOTICE_THRESHOLD`, this also logs -/// a `tracing::warn!` so a slow-building outage is visible in logs even -/// before an operator reads the returned error text. -fn terminal_llm_error(elapsed: std::time::Duration, attempts: u32, detail: &str) -> AgentError { +/// Build the terminal error for a `post()` exit that has given up retrying — +/// persistent retryable status, transport failure, or a body-read break. +/// `detail` carries the specific cause (status/body, or the transport error +/// text); `elapsed` and `attempts` are the cumulative cost of every attempt +/// made on this call, including the retries that failed before this one. When +/// cumulative time crosses `STALL_NOTICE_THRESHOLD`, this also logs a +/// `tracing::warn!` so a slow-building outage is visible in logs even before +/// an operator reads the returned error text. +/// +/// Every exit here is cutover-class by construction: retrying the same +/// endpoint has already been tried and failed, so `kind` records what sort of +/// failure it was and the caller decides whether another provider can do +/// better. +fn terminal_llm_error( + kind: CutoverKind, + elapsed: std::time::Duration, + attempts: u32, + detail: &str, +) -> AgentError { if elapsed >= STALL_NOTICE_THRESHOLD { tracing::warn!( cumulative_stall = ?elapsed, @@ -1722,10 +1969,31 @@ fn terminal_llm_error(elapsed: std::time::Duration, attempts: u32, detail: &str) "llm: cumulative stall {elapsed:?} across {attempts} attempts ({detail})" ); } - AgentError::Llm(format!( - "{detail} (cumulative {elapsed:?}, {attempts} attempt{})", - if attempts == 1 { "" } else { "s" }, - )) + AgentError::LlmUnavailable { + kind, + detail: format!( + "{detail} (cumulative {elapsed:?}, {attempts} attempt{})", + if attempts == 1 { "" } else { "s" }, + ), + } +} + +/// Whether `err` is worth retrying on a *different* provider, and what sort of +/// failure it was. +/// +/// The distinction that matters: a 400 or an unknown model id fails identically +/// everywhere, so cutting over just burns the whole chain to reach the same +/// answer more slowly. Quota, credentials, upstream 5xx, and transport are the +/// failures another provider genuinely might not share. +fn cutover_kind(err: &AgentError) -> Option { + match err { + AgentError::LlmUnavailable { kind, .. } => Some(*kind), + // The token source already spent its one refresh inside the provider + // call. A credential still rejected after that is this endpoint's + // problem — another provider's key may be perfectly good. + AgentError::LlmAuth(_) => Some(CutoverKind::Auth), + _ => None, + } } /// Internal HTTP failure that preserves a mesh-specific MoA failure as a @@ -1816,6 +2084,7 @@ where continue; } return Err(PostError::Agent(terminal_llm_error( + CutoverKind::Transport, call_start.elapsed(), attempt + 1, &format!("transport: {e}"), @@ -1854,12 +2123,31 @@ where backoff_with_jitter(attempt).await; continue; } + // 429 is the quota/rate wall (z.ai's weekly `1310` lands here) and + // holds for a provider-side window; 5xx/499 is an upstream that is + // usually back shortly. The breaker benches them for different + // lengths of time. + let kind = if status == 429 { + CutoverKind::Quota + } else { + CutoverKind::Server + }; return Err(PostError::Agent(terminal_llm_error( + kind, call_start.elapsed(), attempt + 1, &format!("exhausted retries: {status}: {body}"), ))); } + // Credit/balance exhaustion. Retrying this endpoint is pointless — the + // balance will not change mid-turn — but another provider may still + // have credit, so it cuts over rather than stopping the turn. + if status == 402 { + return Err(PostError::Agent(AgentError::LlmUnavailable { + kind: CutoverKind::Quota, + detail: format!("{status}: {}", read_error_body(resp).await), + })); + } // Not a stall path: the model is misconfigured, not the transport or // upstream capacity — no retry was attempted, so cumulative duration // would be misleading. @@ -1897,6 +2185,7 @@ where Ok(None) => break, Err(e) => { return Err(PostError::Agent(terminal_llm_error( + CutoverKind::Transport, call_start.elapsed(), attempt + 1, &format!("body read: {e}"), @@ -2077,6 +2366,7 @@ async fn openrouter_post( continue; } return Err(terminal_llm_error( + CutoverKind::Transport, call_start.elapsed(), attempt + 1, &format!("transport: {e}"), @@ -2101,9 +2391,12 @@ async fn openrouter_post( ))); } if status == 402 { - return Err(AgentError::Llm( - "OpenRouter credits exhausted — check https://openrouter.ai/credits".into(), - )); + // Cutover-class, not terminal: this account is out of credit, but + // a different provider in the chain may not be. + return Err(AgentError::LlmUnavailable { + kind: CutoverKind::Quota, + detail: "OpenRouter credits exhausted — check https://openrouter.ai/credits".into(), + }); } if status == 404 { // OpenRouter overloads 404: a genuinely unknown/unavailable model id @@ -2152,6 +2445,7 @@ async fn openrouter_post( // Terminal: classify for the user return if status == 429 { Err(terminal_llm_error( + CutoverKind::Quota, call_start.elapsed(), attempt + 1, &format!("rate limited: {error_body}"), @@ -2169,6 +2463,7 @@ async fn openrouter_post( openrouter_parameter_routing_error(&error_body) } else { terminal_llm_error( + CutoverKind::Server, call_start.elapsed(), attempt + 1, &format!("exhausted retries: {status}: {error_body}"), @@ -2204,6 +2499,7 @@ async fn openrouter_post( Ok(None) => break, Err(e) => { return Err(terminal_llm_error( + CutoverKind::Transport, call_start.elapsed(), attempt + 1, &format!("body read: {e}"), @@ -2214,6 +2510,7 @@ async fn openrouter_post( return serde_json::from_slice(&buf).map_err(|e| AgentError::Llm(format!("json: {e}"))); } Err(terminal_llm_error( + CutoverKind::Server, call_start.elapsed(), MAX_RETRIES, "exhausted retries", @@ -2337,6 +2634,7 @@ mod tests { fn cfg(provider: Provider) -> Config { Config { provider, + fallback: Vec::new(), system_prompt: "system".into(), max_rounds: 10, max_output_tokens: 1024, @@ -2483,6 +2781,11 @@ mod tests { }); let status_text = match response.status { 200 => "OK", + 400 => "Bad Request", + 401 => "Unauthorized", + 402 => "Payment Required", + 404 => "Not Found", + 429 => "Too Many Requests", 500 => "Internal Server Error", 502 => "Bad Gateway", 503 => "Service Unavailable", @@ -4239,7 +4542,10 @@ mod tests { .await .unwrap_err(); match &err { - PostError::Agent(AgentError::Llm(msg)) => { + PostError::Agent(AgentError::LlmUnavailable { + kind: CutoverKind::Server, + detail: msg, + }) => { assert!( msg.contains("exhausted retries") && msg.contains("499"), "expected 'exhausted retries' + '499' in error, got: {msg}" @@ -4249,7 +4555,9 @@ mod tests { "expected cumulative duration + exact attempt count, got: {msg}" ); } - other => panic!("expected PostError::Agent(AgentError::Llm), got: {other:?}"), + other => panic!( + "a persistent 499 must be cutover-class Server so the chain can move on, got: {other:?}" + ), } assert_eq!( accepts.load(Ordering::SeqCst), @@ -4263,9 +4571,14 @@ mod tests { /// (a handful of quick retries), not an outage. #[test] fn terminal_llm_error_below_threshold_carries_detail_no_stall_claim() { - let err = terminal_llm_error(Duration::from_secs(2), 3, "exhausted retries: 499: body"); + let err = terminal_llm_error( + CutoverKind::Server, + Duration::from_secs(2), + 3, + "exhausted retries: 499: body", + ); match err { - AgentError::Llm(msg) => { + AgentError::LlmUnavailable { detail: msg, .. } => { assert!(msg.contains("cumulative 2s"), "must carry duration: {msg}"); assert!( msg.contains("3 attempts"), @@ -4286,9 +4599,14 @@ mod tests { /// Singular "1 attempt" (not "1 attempts") confirms the pluralization. #[test] fn terminal_llm_error_above_threshold_uses_singular_attempt() { - let err = terminal_llm_error(Duration::from_secs(301), 1, "transport: connection reset"); + let err = terminal_llm_error( + CutoverKind::Transport, + Duration::from_secs(301), + 1, + "transport: connection reset", + ); match err { - AgentError::Llm(msg) => { + AgentError::LlmUnavailable { detail: msg, .. } => { assert!( msg.contains("cumulative 301s"), "must carry duration: {msg}" @@ -4372,7 +4690,12 @@ mod tests { #[test] fn terminal_llm_error_below_threshold_emits_no_stall_warning() { let warnings = count_stall_warnings(|| { - let _ = terminal_llm_error(Duration::from_secs(299), 3, "exhausted retries: 499"); + let _ = terminal_llm_error( + CutoverKind::Server, + Duration::from_secs(299), + 3, + "exhausted retries: 499", + ); }); assert_eq!(warnings, 0, "no stall warning below STALL_NOTICE_THRESHOLD"); } @@ -4385,7 +4708,12 @@ mod tests { #[test] fn terminal_llm_error_at_threshold_emits_one_stall_warning() { let warnings = count_stall_warnings(|| { - let _ = terminal_llm_error(Duration::from_secs(300), 5, "transport: connection reset"); + let _ = terminal_llm_error( + CutoverKind::Transport, + Duration::from_secs(300), + 5, + "transport: connection reset", + ); }); assert_eq!( warnings, 1, @@ -4869,7 +5197,12 @@ mod tests { url } - fn llm_with(auth: Arc) -> Llm { + /// An `Llm` whose primary endpoint is `cfg`'s, wired to the supplied token + /// source. `cfg` must be fully built (including `base_url`) before this is + /// called: `auth_for` matches on the endpoint's circuit key, so an `Llm` + /// stamped with a different key would quietly mint its own source and + /// ignore the one injected here. + fn llm_with(auth: Arc, cfg: &Config) -> Llm { Llm { http: Client::builder() .timeout(Duration::from_secs(5)) @@ -4878,6 +5211,9 @@ mod tests { auto_upgraded: std::sync::atomic::AtomicBool::new(false), mesh_auto_state: Mutex::new(MeshAutoState::default()), auth, + primary_key: cfg.primary_endpoint().circuit_key(), + fallback_auth: std::sync::Mutex::new(HashMap::new()), + breaker: Breaker::new(), } } @@ -4893,9 +5229,9 @@ mod tests { let auth = Arc::new(CountingAuth { refreshes: AtomicU32::new(0), }); - let llm = llm_with(auth.clone()); let mut c = cfg(Provider::OpenAi); c.base_url = base; + let llm = llm_with(auth.clone(), &c); let out = llm .post_openai(&c, "/v1/x", &json!({}), "model") @@ -4929,9 +5265,9 @@ mod tests { let auth = Arc::new(CountingAuth { refreshes: AtomicU32::new(0), }); - let llm = llm_with(auth.clone()); let mut c = cfg(Provider::OpenAi); c.base_url = base; + let llm = llm_with(auth.clone(), &c); let err = llm .post_openai(&c, "/v1/x", &json!({}), "model") @@ -4960,9 +5296,9 @@ mod tests { let auth = Arc::new(CountingAuth { refreshes: AtomicU32::new(0), }); - let llm = llm_with(auth.clone()); let mut c = cfg(Provider::OpenAi); c.base_url = base; + let llm = llm_with(auth.clone(), &c); let err = llm .post_openai(&c, "/v1/x", &json!({}), "model") @@ -4991,9 +5327,9 @@ mod tests { let auth = Arc::new(CountingAuth { refreshes: AtomicU32::new(0), }); - let llm = llm_with(auth.clone()); let mut c = cfg(Provider::OpenAi); c.base_url = base; + let llm = llm_with(auth.clone(), &c); let out = llm .post_openai(&c, "/v1/x", &json!({}), "model") @@ -6173,8 +6509,12 @@ mod tests { .await .unwrap_err(); assert!( - matches!(&err, AgentError::Llm(s) if s.contains("credits exhausted")), - "got {err:?}" + matches!( + &err, + AgentError::LlmUnavailable { kind: CutoverKind::Quota, detail } + if detail.contains("credits exhausted") + ), + "402 must be Quota-class so the turn fails over to a provider with credit: got {err:?}" ); assert_eq!( attempts.load(std::sync::atomic::Ordering::SeqCst), @@ -6443,11 +6783,15 @@ mod tests { .await .unwrap_err(); assert!( - matches!(&err, AgentError::Llm(s) if s.contains("body read")), - "truncated body must surface as AgentError::Llm with 'body read': got {err:?}" + matches!( + &err, + AgentError::LlmUnavailable { kind: CutoverKind::Transport, detail } + if detail.contains("body read") + ), + "a truncated body is a Transport-class failure: got {err:?}" ); assert!( - matches!(&err, AgentError::Llm(s) if s.contains("cumulative")), + matches!(&err, AgentError::LlmUnavailable { detail, .. } if detail.contains("cumulative")), "body-read error must include terminal_llm_error's cumulative context: got {err:?}" ); } @@ -6503,9 +6847,9 @@ mod tests { let auth = Arc::new(StaticAuth { token: "static-key".into(), }); - let llm = llm_with(auth); let mut c = cfg(Provider::OpenRouter); c.base_url = url; + let llm = llm_with(auth, &c); let err = llm.post_openrouter(&c, &json!({})).await.unwrap_err(); assert!( @@ -6537,9 +6881,9 @@ mod tests { fresh: "fresh".into(), refreshes: std::sync::atomic::AtomicU32::new(0), }); - let llm = llm_with(auth.clone()); let mut c = cfg(Provider::OpenRouter); c.base_url = base; + let llm = llm_with(auth.clone(), &c); let result = llm.post_openrouter(&c, &json!({})).await; // `spawn_auth_stub` returns `{"ok":true}` on success. @@ -6553,4 +6897,279 @@ mod tests { "exactly one refresh" ); } + + // ---- provider failover ------------------------------------------------ + + /// A fallback endpoint pointed at `base_url`, carrying its own model. + fn fallback_endpoint(base_url: String, model: &str) -> Endpoint { + Endpoint { + provider: Provider::OpenAi, + api_key: "fallback-key".into(), + model: model.into(), + base_url, + openai_api: OpenAiApi::Chat, + } + } + + /// The model id each captured request asked for. + async fn requested_models(captured: &Arc>>) -> Vec { + captured + .lock() + .await + .iter() + .filter_map(|r| r.body.as_ref()?.get("model")?.as_str().map(str::to_owned)) + .collect() + } + + #[test] + fn cutover_kind_classifies_only_what_another_provider_could_fix() { + assert_eq!( + cutover_kind(&AgentError::LlmUnavailable { + kind: CutoverKind::Quota, + detail: String::new() + }), + Some(CutoverKind::Quota) + ); + // A rejected credential survived its one refresh — another provider's + // key may still be good. + assert_eq!( + cutover_kind(&AgentError::LlmAuth("401".into())), + Some(CutoverKind::Auth) + ); + // Stop-class: identical failure everywhere, so cutting over only + // wastes the rest of the chain. + assert_eq!( + cutover_kind(&AgentError::Llm("400 bad request".into())), + None + ); + assert_eq!( + cutover_kind(&AgentError::LlmModelNotFound("404".into())), + None + ); + assert_eq!(cutover_kind(&AgentError::Cancelled), None); + } + + /// The outage this feature exists for: the primary is out of quota, and the + /// turn is served by the next provider instead of dying. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn complete_fails_over_to_the_next_provider_on_quota() { + let (primary_url, primary_seen) = + spawn_sequence_stub(vec![StubHttpResponse::error(402, "quota exhausted")]).await; + let (fallback_url, fallback_seen) = + spawn_sequence_stub(vec![StubHttpResponse::ok(chat_response("from fallback"))]).await; + + let mut c = cfg(Provider::OpenAi); + c.base_url = primary_url; + c.fallback = vec![fallback_endpoint(fallback_url, "fallback-model")]; + let llm = Llm::new(&c).unwrap(); + + let response = complete_model(&llm, &c, "primary-model") + .await + .expect("a quota wall on the primary must not fail the turn"); + assert_eq!(response.text, "from fallback"); + + assert_eq!( + requested_models(&primary_seen).await, + vec!["primary-model"], + "the primary is asked first, with the caller's effective model" + ); + assert_eq!( + requested_models(&fallback_seen).await, + vec!["fallback-model"], + "the fallback must be asked for ITS OWN model — sending the \ + primary's model id to another provider is a guaranteed 404" + ); + } + + /// A malformed request fails the same way everywhere. Walking the chain + /// would reach the same answer more slowly while burning the fallbacks. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn complete_does_not_fail_over_on_a_bad_request() { + let (primary_url, _) = + spawn_sequence_stub(vec![StubHttpResponse::error(400, "malformed")]).await; + let (fallback_url, fallback_seen) = + spawn_sequence_stub(vec![StubHttpResponse::ok(chat_response("unreachable"))]).await; + + let mut c = cfg(Provider::OpenAi); + c.base_url = primary_url; + c.fallback = vec![fallback_endpoint(fallback_url, "fallback-model")]; + let llm = Llm::new(&c).unwrap(); + + let err = complete_model(&llm, &c, "primary-model") + .await + .expect_err("a 400 must surface, not cut over"); + assert!( + matches!(&err, AgentError::Llm(s) if s.contains("400")), + "got {err:?}" + ); + assert!( + requested_models(&fallback_seen).await.is_empty(), + "the fallback must never be contacted for a stop-class failure" + ); + } + + /// When every endpoint is down the turn still fails — but the error says + /// the whole chain was tried, not just that one provider misbehaved. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn complete_reports_the_whole_chain_when_all_endpoints_fail() { + let (primary_url, primary_seen) = + spawn_sequence_stub(vec![StubHttpResponse::error(402, "quota exhausted")]).await; + let (fallback_url, fallback_seen) = + spawn_sequence_stub(vec![StubHttpResponse::error(402, "also broke")]).await; + + let mut c = cfg(Provider::OpenAi); + c.base_url = primary_url; + c.fallback = vec![fallback_endpoint(fallback_url, "fallback-model")]; + let llm = Llm::new(&c).unwrap(); + + let err = complete_model(&llm, &c, "primary-model") + .await + .expect_err("no endpoint could serve the turn"); + assert!( + matches!( + &err, + AgentError::LlmUnavailable { kind: CutoverKind::Quota, detail } + if detail.contains("all 2 providers failed") + ), + "got {err:?}" + ); + assert_eq!(requested_models(&primary_seen).await.len(), 1); + assert_eq!(requested_models(&fallback_seen).await.len(), 1); + } + + /// Once the breaker has benched the primary, later turns skip it outright + /// instead of re-paying for the same wall. This is the difference between + /// failover costing one slow turn and costing every turn of the outage. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_benched_primary_is_skipped_on_later_turns() { + let (primary_url, primary_seen) = spawn_sequence_stub(vec![ + StubHttpResponse::error(402, "quota exhausted"), + StubHttpResponse::error(402, "quota exhausted"), + StubHttpResponse::error(402, "quota exhausted"), + ]) + .await; + let (fallback_url, fallback_seen) = spawn_sequence_stub(vec![ + StubHttpResponse::ok(chat_response("one")), + StubHttpResponse::ok(chat_response("two")), + StubHttpResponse::ok(chat_response("three")), + ]) + .await; + + let mut c = cfg(Provider::OpenAi); + c.base_url = primary_url; + c.fallback = vec![fallback_endpoint(fallback_url, "fallback-model")]; + let llm = Llm::new(&c).unwrap(); + + for _ in 0..3 { + complete_model(&llm, &c, "primary-model") + .await + .expect("the fallback serves every turn"); + } + + // Two failures open the circuit, so the third turn never touches it. + assert_eq!( + requested_models(&primary_seen).await.len(), + 2, + "the primary must stop being retried once its circuit opens" + ); + assert_eq!( + requested_models(&fallback_seen).await.len(), + 3, + "every turn is still served" + ); + } + + /// With no fallback configured, behavior is exactly what it was before + /// failover existed: one endpoint, and its error surfaces unwrapped. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_single_provider_deployment_is_unchanged() { + let (primary_url, primary_seen) = + spawn_sequence_stub(vec![StubHttpResponse::error(402, "quota exhausted")]).await; + + let mut c = cfg(Provider::OpenAi); + c.base_url = primary_url; + assert!(c.fallback.is_empty()); + let llm = Llm::new(&c).unwrap(); + + let err = complete_model(&llm, &c, "primary-model").await.unwrap_err(); + assert!( + matches!( + &err, + AgentError::LlmUnavailable { detail, .. } if !detail.contains("providers failed") + ), + "a single-provider error must not gain chain wording: got {err:?}" + ); + assert_eq!(requested_models(&primary_seen).await.len(), 1); + } + + /// Summarization drives handoff and context compaction. Without failover a + /// quota wall strands a long conversation at the context ceiling — the same + /// outage the completion path survives, arriving by another door. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn summarize_fails_over_to_the_next_provider() { + let (primary_url, primary_seen) = + spawn_sequence_stub(vec![StubHttpResponse::error(402, "quota exhausted")]).await; + let (fallback_url, fallback_seen) = spawn_sequence_stub(vec![StubHttpResponse::ok( + chat_response("summary from fallback"), + )]) + .await; + + let mut c = cfg(Provider::OpenAi); + c.base_url = primary_url; + c.fallback = vec![fallback_endpoint(fallback_url, "fallback-model")]; + let llm = Llm::new(&c).unwrap(); + + let text = llm + .summarize(&c, "system", "summarize this", 256, "primary-model") + .await + .expect("a quota wall on the primary must not strand compaction"); + assert_eq!(text, "summary from fallback"); + + assert_eq!(requested_models(&primary_seen).await, vec!["primary-model"]); + assert_eq!( + requested_models(&fallback_seen).await, + vec!["fallback-model"], + "the summary must be asked of the fallback's own model" + ); + } + + /// The breaker is shared across both paths. A primary benched by failed + /// turns is skipped by summarization too, rather than each path paying to + /// rediscover the same outage. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_primary_benched_by_turns_is_skipped_by_summarize() { + let (primary_url, primary_seen) = spawn_sequence_stub(vec![ + StubHttpResponse::error(402, "quota exhausted"), + StubHttpResponse::error(402, "quota exhausted"), + ]) + .await; + let (fallback_url, fallback_seen) = spawn_sequence_stub(vec![ + StubHttpResponse::ok(chat_response("one")), + StubHttpResponse::ok(chat_response("two")), + StubHttpResponse::ok(chat_response("the summary")), + ]) + .await; + + let mut c = cfg(Provider::OpenAi); + c.base_url = primary_url; + c.fallback = vec![fallback_endpoint(fallback_url, "fallback-model")]; + let llm = Llm::new(&c).unwrap(); + + // Two turns fail the primary over — that opens its circuit. + for _ in 0..2 { + complete_model(&llm, &c, "primary-model").await.unwrap(); + } + let text = llm + .summarize(&c, "system", "summarize this", 256, "primary-model") + .await + .unwrap(); + assert_eq!(text, "the summary"); + + assert_eq!( + requested_models(&primary_seen).await.len(), + 2, + "summarize must not re-probe a primary the turns already benched" + ); + assert_eq!(requested_models(&fallback_seen).await.len(), 3); + } } diff --git a/crates/buzz-agent/src/types.rs b/crates/buzz-agent/src/types.rs index 343a75bf72..fcda455c27 100644 --- a/crates/buzz-agent/src/types.rs +++ b/crates/buzz-agent/src/types.rs @@ -358,12 +358,58 @@ pub enum ContentBlock { Unsupported, } +/// Why a provider call failed in a way that a *different* provider could +/// plausibly satisfy. Distinguishes the failures worth cutting over on from +/// the ones that would fail identically everywhere (a malformed request), and +/// sets how long the circuit breaker holds the endpoint down. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CutoverKind { + /// Quota or rate limit — HTTP 429 or 402. Usually bound to a provider-side + /// window (z.ai's weekly `1310` resets days later), so retrying the same + /// endpoint soon is wasted work. + Quota, + /// Credentials rejected (401/403) after the token source already spent its + /// one refresh. A wrong or revoked key does not heal on its own. + Auth, + /// Upstream 5xx/499 — the provider is reachable but failing. Often brief. + Server, + /// Connect, timeout, or body-read failure. Often brief. + Transport, +} + +impl CutoverKind { + /// Whether this failure is unlikely to clear within a minute. Quota + /// windows and bad credentials persist; a 503 or a dropped connection + /// usually does not. + pub fn is_slow_to_heal(self) -> bool { + matches!(self, Self::Quota | Self::Auth) + } + + /// Short lowercase label for logs and error text. + pub fn as_str(self) -> &'static str { + match self { + Self::Quota => "quota", + Self::Auth => "auth", + Self::Server => "server", + Self::Transport => "transport", + } + } +} + #[derive(Debug)] pub enum AgentError { InvalidParams(String), Llm(String), LlmAuth(String), LlmModelNotFound(String), + /// The provider could not serve the request for a reason another provider + /// might not share — quota, credentials, upstream 5xx, or transport. This + /// is the class that drives failover; it surfaces to the caller only once + /// every endpoint in the chain has been tried. + LlmUnavailable { + kind: CutoverKind, + detail: String, + }, Mcp(String), Cancelled, } @@ -375,6 +421,9 @@ impl std::fmt::Display for AgentError { Self::Llm(s) => write!(f, "llm: {s}"), Self::LlmAuth(s) => write!(f, "llm auth: {s}"), Self::LlmModelNotFound(s) => write!(f, "llm model not found: {s}"), + Self::LlmUnavailable { kind, detail } => { + write!(f, "llm unavailable ({}): {detail}", kind.as_str()) + } Self::Mcp(s) => write!(f, "mcp: {s}"), Self::Cancelled => write!(f, "cancelled"), } @@ -389,6 +438,7 @@ impl AgentError { Self::InvalidParams(_) => -32602, Self::LlmAuth(_) => -32001, Self::LlmModelNotFound(_) => -32002, + Self::LlmUnavailable { .. } => -32003, _ => -32000, } } diff --git a/crates/buzz-agent/tests/fake_llm.rs b/crates/buzz-agent/tests/fake_llm.rs index f782a9d476..eb328ada97 100644 --- a/crates/buzz-agent/tests/fake_llm.rs +++ b/crates/buzz-agent/tests/fake_llm.rs @@ -155,6 +155,15 @@ impl Harness { .env("OPENAI_COMPAT_API_KEY", "test") .env("OPENAI_COMPAT_MODEL", "fake-model") .env("OPENAI_COMPAT_BASE_URL", base_url) + // Pin failover OFF. `cmd.env()` adds to the inherited environment + // rather than replacing it, so on a developer box or CI runner that + // exports a real OPENROUTER_/OLLAMA_ key AND model, the agent would + // auto-derive a fallback chain and answer these tests from a REAL + // provider the moment a canned response looked cutover-class — + // nondeterministic, and billable. These tests assert + // single-provider behavior; a test that wants failover sets its own + // chain. + .env("BUZZ_AGENT_FALLBACK_PROVIDERS", "none") .env("BUZZ_AGENT_LLM_TIMEOUT_SECS", "5") .env("BUZZ_AGENT_TOOL_TIMEOUT_SECS", "5") .env("BUZZ_AGENT_MAX_ROUNDS", "4") diff --git a/desktop/src/features/agents/lib/friendlyAgentLastError.test.mjs b/desktop/src/features/agents/lib/friendlyAgentLastError.test.mjs index 597b1b9323..c666d1b2cf 100644 --- a/desktop/src/features/agents/lib/friendlyAgentLastError.test.mjs +++ b/desktop/src/features/agents/lib/friendlyAgentLastError.test.mjs @@ -6,6 +6,7 @@ import { friendlyTurnErrorCopy, CLI_ACP_INTERNAL_ERROR_COPY, MODEL_NOT_FOUND_COPY, + PROVIDER_UNAVAILABLE_COPY, RELAY_MESH_DENIED_COPY, } from "./friendlyAgentLastError.ts"; @@ -153,11 +154,13 @@ test("friendlyTurnErrorCopy: unknown code passes raw text through", () => { // --- structured-code hardening --- test("unknown code prevents string-pattern cross-classification", () => { - // code -32003 is structured and unrecognized — must NOT fall through to + // code -32004 is structured and unrecognized — must NOT fall through to // the legacy string path that would wrongly promote this to denied. + // (-32003 was this fixture until buzz-agent claimed it for provider + // failover; an unclaimed code is the whole point of the test.) const result = friendlyAgentLastError( "llm auth: rate limiter denial", - -32003, + -32004, ); assert.deepEqual(result, { severity: "generic", @@ -165,6 +168,30 @@ test("unknown code prevents string-pattern cross-classification", () => { }); }); +test("code -32003 → provider-unavailable copy (severity: generic)", () => { + // The failover chain walked every configured provider and none recovered + // the turn (e.g. z.ai's `1310` weekly limit with no fallback left). + const result = friendlyAgentLastError( + "Agent reported error (code -32003): llm unavailable (quota): (glm-5.2) all 2 providers failed; last: exhausted retries: 429", + -32003, + ); + assert.deepEqual(result, { + severity: "generic", + copy: PROVIDER_UNAVAILABLE_COPY, + }); +}); + +test("embedded code -32003 recovered from message → provider-unavailable copy", () => { + const result = friendlyAgentLastError( + "Agent reported error (code -32003): llm unavailable (quota): exhausted retries: 429", + null, + ); + assert.deepEqual(result, { + severity: "generic", + copy: PROVIDER_UNAVAILABLE_COPY, + }); +}); + test("NaN code param treated as absent — string path applies", () => { // NaN is not finite; falls back to string matching. const result = friendlyAgentLastError("llm auth: denied", NaN); diff --git a/desktop/src/features/agents/lib/friendlyAgentLastError.ts b/desktop/src/features/agents/lib/friendlyAgentLastError.ts index 60c77bb04c..910b688010 100644 --- a/desktop/src/features/agents/lib/friendlyAgentLastError.ts +++ b/desktop/src/features/agents/lib/friendlyAgentLastError.ts @@ -4,7 +4,8 @@ * The error classification seam flows like this: * buzz-agent — classifies LLM failures into `AgentError` variants with * JSON-RPC codes (`-32001` auth, `-32002` model-not-found, - * `-32000` generic), defined in `crates/buzz-agent/src/types.rs`. + * `-32003` provider-unavailable, `-32000` generic), defined in + * `crates/buzz-agent/src/types.rs`. * buzz-acp — preserves the code structurally in * `AcpError::AgentError { code, message }`, whose Display is * `"Agent reported error (code N): message"`, and includes @@ -42,6 +43,9 @@ export const RELAY_MESH_DENIED_COPY = export const MODEL_NOT_FOUND_COPY = "The configured model is not available — open agent settings and select a different one from the dropdown."; +export const PROVIDER_UNAVAILABLE_COPY = + "The model provider is unavailable — out of quota, rate-limited, or down — and no fallback provider recovered the turn. Add another provider to the fallback chain (Ollama Cloud or OpenRouter via BUZZ_AGENT_FALLBACK_PROVIDERS) or wait for the quota to reset."; + export const CLI_ACP_INTERNAL_ERROR_COPY = "The agent's harness reported an internal error. For Codex agents this can mean the configured model isn't supported by your installed codex-acp — check the model in `~/.codex/config.toml` or upgrade the adapter (`brew upgrade codex-acp`)."; @@ -81,6 +85,12 @@ export function friendlyAgentLastError( return { severity: "denied", copy: RELAY_MESH_DENIED_COPY }; case -32002: return { severity: "denied", copy: MODEL_NOT_FOUND_COPY }; + case -32003: + // Provider unavailable (quota, credentials, upstream 5xx, transport). + // buzz-agent's failover chain walked every configured provider and + // none recovered the turn — this surfaces ONLY when the whole chain is + // down. A turn a fallback rescued returns Ok and never lands here. + return { severity: "generic", copy: PROVIDER_UNAVAILABLE_COPY }; case -32603: { // Standard JSON-RPC "Internal error" — emitted by external harnesses // (e.g. codex-acp) when the configured model is unsupported. Only