From 97d5532d15de0de4d04c0a801db3baab2b4c0bb9 Mon Sep 17 00:00:00 2001
From: Barrett Holien <17663953+BarrettHolien@users.noreply.github.com>
Date: Sun, 2 Aug 2026 23:34:28 -0500
Subject: [PATCH 1/3] feat(buzz-agent): fail over to the next provider on
 quota/auth/5xx
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A single provider's quota wall took every managed agent offline: Config
held one provider, so a 429 (z.ai's weekly `1310`) flattened to
AgentError::Llm and killed the turn with nowhere else to go.

Adds an ordered failover chain. Config gains `fallback: Vec<Endpoint>`,
auto-derived from the provider slots the environment already defines
(Ollama Cloud, then OpenRouter), overridable with
BUZZ_AGENT_FALLBACK_PROVIDERS and disabled with `none`. Llm::complete
walks that chain, retrying the SAME turn on the next endpoint.

Failures are split into two classes. Quota (429/402), auth (401/403
after the token source spent its refresh), upstream 5xx/499, and
transport are cutover-class: another provider may not share them. A 400
or an unknown model is stop-class and returns immediately — it fails
identically everywhere, so walking the chain would only reach the same
answer more slowly. The class is carried by a new
AgentError::LlmUnavailable { kind, detail } at JSON-RPC -32003.

An in-process circuit breaker (health.rs, ported from cortex-discord's
lib/health.js) benches an endpoint after two consecutive cutover
failures, for 5min on quota/auth and 60s on 5xx/transport, doubling to a
15min cap. Without it every turn of a days-long quota outage re-pays for
discovering the wall. It fails open: if every endpoint is benched the
whole chain is tried anyway.

Two things that are easy to get wrong and are covered by tests:

- Each fallback answers with its OWN slot's model. BUZZ_AGENT_MODEL
  applies to the primary only; sending `glm-5.2` to OpenRouter 404s.
- Each endpoint gets its own TokenSource. `Llm::auth` holds the
  primary's credential, so a cutover reusing it would authenticate the
  wrong provider and 401 every time.

A deployment with no fallback configured is unchanged, down to the
error text.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Barrett Holien <17663953+BarrettHolien@users.noreply.github.com>
---
 crates/buzz-agent/src/config.rs | 454 +++++++++++++++++++++++++
 crates/buzz-agent/src/health.rs | 281 ++++++++++++++++
 crates/buzz-agent/src/lib.rs    |   1 +
 crates/buzz-agent/src/llm.rs    | 566 +++++++++++++++++++++++++++++---
 crates/buzz-agent/src/types.rs  |  50 +++
 5 files changed, 1303 insertions(+), 49 deletions(-)
 create mode 100644 crates/buzz-agent/src/health.rs

diff --git a/crates/buzz-agent/src/config.rs b/crates/buzz-agent/src/config.rs
index afbda5379d..3e40112762 100644
--- a/crates/buzz-agent/src/config.rs
+++ b/crates/buzz-agent/src/config.rs
@@ -687,6 +687,161 @@ pub enum OpenAiApi {
     Auto,
 }
 
+/// One provider the agent can send a turn to — a provider slot's full
+/// coordinates. The primary lives in [`Config`] itself; [`Config::fallback`]
+/// holds the ordered alternates tried when the primary cannot serve the turn.
+///
+/// Each endpoint carries its own model deliberately: a fallback answers with
+/// the model its own slot configures, never the primary's. Sending the
+/// primary's model id to a different provider is a guaranteed 404.
+#[derive(Debug, Clone, PartialEq)]
+pub struct Endpoint {
+    pub provider: Provider,
+    pub api_key: String,
+    pub model: String,
+    pub base_url: String,
+    pub openai_api: OpenAiApi,
+}
+
+impl Endpoint {
+    /// Stable identity for circuit-breaker bookkeeping. Keyed on the upstream
+    /// (provider + base URL) rather than the model: a quota wall or a dead host
+    /// applies to every model behind it, and keying per-model would make the
+    /// breaker re-learn the same outage for each one.
+    pub fn circuit_key(&self) -> String {
+        format!(
+            "{:?}|{}",
+            self.provider,
+            self.base_url.trim_end_matches('/')
+        )
+    }
+
+    /// Trailing-slash-insensitive identity of the wire destination. Two slots
+    /// with the same provider and base URL are the same upstream, so one
+    /// cannot stand in for the other when that upstream is down.
+    fn same_upstream_as(&self, other: &Self) -> bool {
+        self.provider == other.provider
+            && self.base_url.trim_end_matches('/') == other.base_url.trim_end_matches('/')
+    }
+}
+
+/// A provider slot as read from the environment, before validation. Borrowed
+/// so [`derive_fallback_chain`] stays pure and testable without mutating the
+/// process environment (env-mutating tests race under a threaded runner).
+#[derive(Debug, Clone, Copy)]
+struct RawSlot<'a> {
+    /// Canonical id accepted in `BUZZ_AGENT_FALLBACK_PROVIDERS`.
+    id: &'a str,
+    provider: Provider,
+    api_key: Option<&'a str>,
+    model: Option<&'a str>,
+    base_url: &'a str,
+    openai_api: OpenAiApi,
+}
+
+/// Auto-derivation order when `BUZZ_AGENT_FALLBACK_PROVIDERS` is unset: the
+/// flat-rate Ollama Cloud slot first, then OpenRouter's free tier. The primary
+/// is removed from whatever this produces.
+const DEFAULT_FALLBACK_ORDER: [&str; 2] = ["openai", "openrouter"];
+
+/// Map a `BUZZ_AGENT_FALLBACK_PROVIDERS` entry to a slot id, accepting the
+/// same spellings as `BUZZ_AGENT_PROVIDER`.
+fn canonical_slot_id(raw: &str) -> Result<&'static str, String> {
+    match raw {
+        "anthropic" => Ok("anthropic"),
+        "openai" | "openai-compat" => Ok("openai"),
+        "openrouter" => Ok("openrouter"),
+        // Known providers that cannot serve as a fallback. Databricks resolves
+        // its bearer through OAuth PKCE against a host-specific discovery
+        // document; there is no way to express a second one in the current env
+        // surface. Say so rather than silently dropping what the operator asked
+        // for and leaving them believing failover is armed.
+        "databricks" | "databricks_v2" | "databricks-v2" => Err(format!(
+            "config: BUZZ_AGENT_FALLBACK_PROVIDERS={raw} is not supported as a fallback \
+             (use anthropic, openai, or openrouter)"
+        )),
+        other => Err(format!(
+            "config: BUZZ_AGENT_FALLBACK_PROVIDERS={other} not supported \
+             (use anthropic, openai, openrouter, or none)"
+        )),
+    }
+}
+
+/// Build the ordered failover chain.
+///
+/// - `Some("none")` disables failover entirely — the pre-failover behavior.
+/// - `Some(list)` takes the named slots in the given order. An unknown id is a
+///   startup error, matching how `BUZZ_AGENT_PROVIDER` treats a typo.
+/// - `None` auto-derives from [`DEFAULT_FALLBACK_ORDER`].
+///
+/// A candidate is dropped when its key or model is absent (nothing to call) or
+/// when it targets the same upstream as `primary` (it would fail the same way).
+/// Dropping is a warning, never an error: a missing fallback must not stop the
+/// agent from starting on a working primary.
+fn derive_fallback_chain(
+    requested: Option<&str>,
+    primary: &Endpoint,
+    slots: &[RawSlot<'_>],
+) -> Result<Vec<Endpoint>, String> {
+    let requested = requested.map(str::trim).filter(|s| !s.is_empty());
+    if requested.is_some_and(|r| r.eq_ignore_ascii_case("none")) {
+        return Ok(Vec::new());
+    }
+
+    let (ids, explicit) = match requested {
+        Some(list) => {
+            let mut ids = Vec::new();
+            for entry in list.split(',').map(str::trim).filter(|s| !s.is_empty()) {
+                ids.push(canonical_slot_id(&entry.to_ascii_lowercase())?);
+            }
+            (ids, true)
+        }
+        None => (DEFAULT_FALLBACK_ORDER.to_vec(), false),
+    };
+
+    let mut chain: Vec<Endpoint> = Vec::new();
+    for id in ids {
+        let Some(slot) = slots.iter().find(|s| s.id == id) else {
+            continue;
+        };
+        let (Some(api_key), Some(model)) = (
+            slot.api_key.map(str::trim).filter(|s| !s.is_empty()),
+            slot.model.map(str::trim).filter(|s| !s.is_empty()),
+        ) else {
+            // Only worth saying out loud when the operator named it — an
+            // absent slot under auto-derivation is the ordinary case.
+            if explicit {
+                tracing::warn!(
+                    provider = id,
+                    "config: fallback provider skipped — its API key or model is not set"
+                );
+            }
+            continue;
+        };
+        let candidate = Endpoint {
+            provider: slot.provider,
+            api_key: api_key.to_owned(),
+            model: model.to_owned(),
+            base_url: slot.base_url.to_owned(),
+            openai_api: slot.openai_api,
+        };
+        if candidate.same_upstream_as(primary) {
+            if explicit {
+                tracing::warn!(
+                    provider = id,
+                    "config: fallback provider skipped — same upstream as the primary"
+                );
+            }
+            continue;
+        }
+        if chain.iter().any(|e| e.same_upstream_as(&candidate)) {
+            continue;
+        }
+        chain.push(candidate);
+    }
+    Ok(chain)
+}
+
 #[derive(Debug, Clone)]
 pub struct Config {
     pub provider: Provider,
@@ -737,6 +892,11 @@ pub struct Config {
     pub api_key: String,
     pub model: String,
     pub base_url: String,
+    /// Ordered alternates tried when the primary cannot serve a turn (quota,
+    /// credentials, upstream 5xx, transport). Empty means no failover — the
+    /// single-provider behavior. Derived from the provider slots present in
+    /// the environment; override with `BUZZ_AGENT_FALLBACK_PROVIDERS`.
+    pub fallback: Vec<Endpoint>,
     pub anthropic_api_version: String,
     /// OpenAI endpoint selection. See [`OpenAiApi`].
     pub openai_api: OpenAiApi,
@@ -822,6 +982,62 @@ impl Config {
                 OpenAiApi::Chat, // OpenRouter uses Chat Completions only
             ),
         };
+        // Every provider slot the environment defines, read once so the chain
+        // derivation stays pure. Each slot keeps its OWN model var — the
+        // `BUZZ_AGENT_MODEL` override applies to the primary only, or a
+        // cutover would send the primary's model id to a provider that has
+        // never heard of it. `OPENAI_COMPAT_API` is parsed leniently here for
+        // the same reason it is read conditionally above: a stray value must
+        // not break a deployment whose primary is not OpenAI.
+        let anthropic_key = env("ANTHROPIC_API_KEY");
+        let anthropic_model = env("ANTHROPIC_MODEL");
+        let anthropic_base = env_or("ANTHROPIC_BASE_URL", "https://api.anthropic.com");
+        let openai_key = env("OPENAI_COMPAT_API_KEY");
+        let openai_model = env("OPENAI_COMPAT_MODEL");
+        let openai_base = env_or("OPENAI_COMPAT_BASE_URL", "https://api.openai.com/v1");
+        let openai_slot_api =
+            parse_openai_api(env("OPENAI_COMPAT_API").as_deref()).unwrap_or(OpenAiApi::Auto);
+        let openrouter_key = env("OPENROUTER_API_KEY");
+        let openrouter_model = env("OPENROUTER_MODEL");
+        let openrouter_base = env_or("OPENROUTER_BASE_URL", "https://openrouter.ai/api/v1");
+        let primary_endpoint = Endpoint {
+            provider,
+            api_key: api_key.clone(),
+            model: model.clone(),
+            base_url: base_url.clone(),
+            openai_api,
+        };
+        let fallback = derive_fallback_chain(
+            env("BUZZ_AGENT_FALLBACK_PROVIDERS").as_deref(),
+            &primary_endpoint,
+            &[
+                RawSlot {
+                    id: "openai",
+                    provider: Provider::OpenAi,
+                    api_key: openai_key.as_deref(),
+                    model: openai_model.as_deref(),
+                    base_url: &openai_base,
+                    openai_api: openai_slot_api,
+                },
+                RawSlot {
+                    id: "openrouter",
+                    provider: Provider::OpenRouter,
+                    api_key: openrouter_key.as_deref(),
+                    model: openrouter_model.as_deref(),
+                    base_url: &openrouter_base,
+                    openai_api: OpenAiApi::Chat,
+                },
+                RawSlot {
+                    id: "anthropic",
+                    provider: Provider::Anthropic,
+                    api_key: anthropic_key.as_deref(),
+                    model: anthropic_model.as_deref(),
+                    base_url: &anthropic_base,
+                    openai_api: OpenAiApi::Auto,
+                },
+            ],
+        )?;
+
         let system_prompt = match (env("BUZZ_AGENT_SYSTEM_PROMPT"), env("BUZZ_AGENT_SYSTEM_PROMPT_FILE")) {
             (Some(_), Some(_)) => return Err(
                 "config: BUZZ_AGENT_SYSTEM_PROMPT and BUZZ_AGENT_SYSTEM_PROMPT_FILE are mutually exclusive".into()),
@@ -835,6 +1051,7 @@ impl Config {
             api_key,
             model,
             base_url,
+            fallback,
             anthropic_api_version: env_or("ANTHROPIC_API_VERSION", "2023-06-01"),
             openai_api,
             prefer_mesh_for_auto: parse_env("BUZZ_AGENT_PREFER_MESH_FOR_AUTO", 0u8)? != 0,
@@ -882,6 +1099,9 @@ impl Config {
             provider,
             api_key,
             base_url,
+            // Catalog discovery targets one explicit endpoint; there is
+            // nothing to fail over to.
+            fallback: Vec::new(),
             model: String::new(),
             system_prompt: String::new(),
             anthropic_api_version: "2023-06-01".into(),
@@ -912,6 +1132,37 @@ impl Config {
         }
     }
 
+    /// This config's own provider slot as an [`Endpoint`] — the first link in
+    /// the failover chain.
+    pub fn primary_endpoint(&self) -> Endpoint {
+        Endpoint {
+            provider: self.provider,
+            api_key: self.api_key.clone(),
+            model: self.model.clone(),
+            base_url: self.base_url.clone(),
+            openai_api: self.openai_api,
+        }
+    }
+
+    /// This config aimed at `endpoint` instead of its own provider slot.
+    ///
+    /// Everything that is not a provider coordinate — prompts, limits,
+    /// timeouts, effort — is preserved, so a failover retry differs from the
+    /// original attempt only in where it is sent.
+    pub fn with_endpoint(&self, endpoint: &Endpoint) -> Self {
+        Self {
+            provider: endpoint.provider,
+            api_key: endpoint.api_key.clone(),
+            model: endpoint.model.clone(),
+            base_url: endpoint.base_url.clone(),
+            openai_api: endpoint.openai_api,
+            // The chain is walked one level up. Clearing it here keeps an
+            // attempt from recursively failing over inside itself.
+            fallback: Vec::new(),
+            ..self.clone()
+        }
+    }
+
     fn validate(&self) -> Result<(), String> {
         const MIN_HISTORY_BYTES: usize = 4096;
         const MIN_LINE_BYTES: usize = 1024;
@@ -2763,4 +3014,207 @@ mod tests {
         let err = resolve_provider(Some("openrouter"), None, None, None).unwrap_err();
         assert!(err.contains("OPENROUTER_API_KEY"));
     }
+
+    // ---- failover chain derivation ----------------------------------------
+    //
+    // Modelled on the real estate deployment that motivated failover: z.ai's
+    // GLM through the `anthropic` slot as primary, with Ollama Cloud and
+    // OpenRouter available as alternates.
+
+    const OLLAMA: &str = "https://ollama.com/v1";
+    const OPENROUTER: &str = "https://openrouter.ai/api/v1";
+    const ZAI: &str = "https://api.z.ai/api/anthropic";
+
+    fn slots() -> Vec<RawSlot<'static>> {
+        vec![
+            RawSlot {
+                id: "openai",
+                provider: Provider::OpenAi,
+                api_key: Some("ollama-key"),
+                model: Some("qwen3.5:397b"),
+                base_url: OLLAMA,
+                openai_api: OpenAiApi::Chat,
+            },
+            RawSlot {
+                id: "openrouter",
+                provider: Provider::OpenRouter,
+                api_key: Some("openrouter-key"),
+                model: Some("nvidia/nemotron:free"),
+                base_url: OPENROUTER,
+                openai_api: OpenAiApi::Chat,
+            },
+            RawSlot {
+                id: "anthropic",
+                provider: Provider::Anthropic,
+                api_key: Some("zai-key"),
+                model: Some("glm-5.2"),
+                base_url: ZAI,
+                openai_api: OpenAiApi::Auto,
+            },
+        ]
+    }
+
+    fn endpoint(provider: Provider, base_url: &str, model: &str) -> Endpoint {
+        Endpoint {
+            provider,
+            api_key: "key".into(),
+            model: model.into(),
+            base_url: base_url.into(),
+            openai_api: OpenAiApi::Auto,
+        }
+    }
+
+    fn zai_primary() -> Endpoint {
+        endpoint(Provider::Anthropic, ZAI, "glm-5.2")
+    }
+
+    fn ids(chain: &[Endpoint]) -> Vec<String> {
+        chain.iter().map(|e| e.base_url.clone()).collect()
+    }
+
+    #[test]
+    fn fallback_chain_auto_derives_ollama_then_openrouter() {
+        let chain = derive_fallback_chain(None, &zai_primary(), &slots()).unwrap();
+        assert_eq!(ids(&chain), vec![OLLAMA, OPENROUTER]);
+    }
+
+    #[test]
+    fn fallback_chain_none_disables_failover() {
+        for raw in ["none", "NONE", " none "] {
+            let chain = derive_fallback_chain(Some(raw), &zai_primary(), &slots()).unwrap();
+            assert!(chain.is_empty(), "{raw:?} must disable failover");
+        }
+    }
+
+    #[test]
+    fn fallback_chain_honors_an_explicit_order() {
+        let chain =
+            derive_fallback_chain(Some("openrouter, openai"), &zai_primary(), &slots()).unwrap();
+        assert_eq!(ids(&chain), vec![OPENROUTER, OLLAMA]);
+    }
+
+    #[test]
+    fn fallback_chain_accepts_the_openai_compat_spelling() {
+        let chain = derive_fallback_chain(Some("openai-compat"), &zai_primary(), &slots()).unwrap();
+        assert_eq!(ids(&chain), vec![OLLAMA]);
+    }
+
+    /// The trap the estate config hits: the `anthropic` slot IS the primary
+    /// (z.ai), so naming it as a fallback buys nothing — the same key, the
+    /// same URL, the same exhausted quota.
+    #[test]
+    fn fallback_chain_skips_the_primarys_own_upstream() {
+        let chain = derive_fallback_chain(Some("anthropic"), &zai_primary(), &slots()).unwrap();
+        assert!(
+            chain.is_empty(),
+            "an endpoint identical to the primary cannot stand in for it"
+        );
+    }
+
+    #[test]
+    fn fallback_chain_skips_the_primary_during_auto_derivation() {
+        // Primary is Ollama itself — auto-derivation must not list it twice.
+        let primary = endpoint(Provider::OpenAi, OLLAMA, "qwen3.5:397b");
+        let chain = derive_fallback_chain(None, &primary, &slots()).unwrap();
+        assert_eq!(ids(&chain), vec![OPENROUTER]);
+    }
+
+    #[test]
+    fn fallback_chain_matching_ignores_a_trailing_slash() {
+        let primary = endpoint(Provider::OpenAi, &format!("{OLLAMA}/"), "qwen3.5:397b");
+        let chain = derive_fallback_chain(None, &primary, &slots()).unwrap();
+        assert_eq!(
+            ids(&chain),
+            vec![OPENROUTER],
+            "a trailing slash is the same upstream, not a second one"
+        );
+    }
+
+    #[test]
+    fn fallback_chain_carries_each_slots_own_model() {
+        let chain = derive_fallback_chain(None, &zai_primary(), &slots()).unwrap();
+        assert_eq!(
+            chain[0].model, "qwen3.5:397b",
+            "a fallback must answer with its own model — the primary's \
+             `glm-5.2` means nothing to Ollama"
+        );
+        assert_eq!(chain[1].model, "nvidia/nemotron:free");
+    }
+
+    #[test]
+    fn fallback_chain_skips_a_slot_with_no_key_or_model() {
+        let mut incomplete = slots();
+        incomplete[0].api_key = None;
+        incomplete[1].model = Some("   ");
+        let chain = derive_fallback_chain(None, &zai_primary(), &incomplete).unwrap();
+        assert!(
+            chain.is_empty(),
+            "a slot with nothing to call must not enter the chain"
+        );
+    }
+
+    #[test]
+    fn fallback_chain_dedupes_repeated_entries() {
+        let chain =
+            derive_fallback_chain(Some("openai,openai,openrouter"), &zai_primary(), &slots())
+                .unwrap();
+        assert_eq!(ids(&chain), vec![OLLAMA, OPENROUTER]);
+    }
+
+    #[test]
+    fn fallback_chain_rejects_an_unknown_provider_id() {
+        let err =
+            derive_fallback_chain(Some("openrouter,typo"), &zai_primary(), &slots()).unwrap_err();
+        assert!(err.contains("typo"), "{err}");
+        assert!(err.contains("BUZZ_AGENT_FALLBACK_PROVIDERS"), "{err}");
+    }
+
+    /// Databricks resolves its bearer through host-specific OAuth PKCE, which
+    /// the env surface cannot express twice. Saying so beats silently dropping
+    /// it and leaving the operator believing failover is armed.
+    #[test]
+    fn fallback_chain_rejects_databricks_with_a_clear_message() {
+        let err = derive_fallback_chain(Some("databricks"), &zai_primary(), &slots()).unwrap_err();
+        assert!(err.contains("not supported as a fallback"), "{err}");
+    }
+
+    #[test]
+    fn fallback_chain_is_empty_when_no_other_slot_is_configured() {
+        let only_primary = vec![slots()[2]];
+        let chain = derive_fallback_chain(None, &zai_primary(), &only_primary).unwrap();
+        assert!(chain.is_empty());
+    }
+
+    #[test]
+    fn circuit_key_is_stable_across_trailing_slashes_and_models() {
+        let a = endpoint(Provider::OpenAi, OLLAMA, "model-a");
+        let b = endpoint(Provider::OpenAi, &format!("{OLLAMA}/"), "model-b");
+        assert_eq!(
+            a.circuit_key(),
+            b.circuit_key(),
+            "one upstream is one circuit, whatever model is asked of it"
+        );
+        assert_ne!(
+            a.circuit_key(),
+            endpoint(Provider::OpenRouter, OPENROUTER, "model-a").circuit_key()
+        );
+    }
+
+    #[test]
+    fn with_endpoint_swaps_the_provider_and_keeps_everything_else() {
+        let mut cfg = Config::for_discovery(Provider::Anthropic, "zai-key".into(), ZAI.into());
+        cfg.max_output_tokens = 4096;
+        cfg.system_prompt = "keep me".into();
+
+        let swapped = cfg.with_endpoint(&endpoint(Provider::OpenAi, OLLAMA, "qwen3.5:397b"));
+        assert_eq!(swapped.provider, Provider::OpenAi);
+        assert_eq!(swapped.base_url, OLLAMA);
+        assert_eq!(swapped.model, "qwen3.5:397b");
+        assert_eq!(swapped.system_prompt, "keep me");
+        assert_eq!(swapped.max_output_tokens, 4096);
+        assert!(
+            swapped.fallback.is_empty(),
+            "an attempt must not recursively fail over inside itself"
+        );
+    }
 }
diff --git a/crates/buzz-agent/src/health.rs b/crates/buzz-agent/src/health.rs
new file mode 100644
index 0000000000..794353eab8
--- /dev/null
+++ b/crates/buzz-agent/src/health.rs
@@ -0,0 +1,281 @@
+//! In-process circuit breaker for the provider failover chain.
+//!
+//! Without it, every turn re-discovers an outage the hard way: the primary is
+//! tried, waits out its retries, and only then cuts over. A quota wall lasts
+//! days, so that cost is paid on every turn for as long as it lasts. The
+//! breaker remembers, and sends the next turn straight to a provider that
+//! works.
+//!
+//! State machine per endpoint:
+//!
+//! ```text
+//! closed  --2 consecutive cutover failures-->  open (cooldown starts)
+//! open, now <  open_until                      blocked: skipped by the chain
+//! open, now >= open_until                      half-open: one probe allowed
+//!   probe fails                                cooldown doubles, stays open
+//!   probe succeeds                             closed, counters reset
+//! ```
+//!
+//! Cooldown starts at 60s and doubles per failed probe to a 15min cap.
+//! Quota and auth failures start at 5min instead: a weekly quota window or a
+//! revoked key does not heal inside a minute, so probing that fast is waste.
+//!
+//! Deliberately in-process and not persisted. The state is a latency
+//! optimization, not a source of truth — a restarted agent re-learning an
+//! outage costs one slow turn, which is not worth a file to corrupt. It is
+//! also reactive by construction: a circuit opens only after real traffic has
+//! failed, so it cannot predict an exhausted quota before anything tries.
+//!
+//! Fail-soft throughout: the breaker never reports an endpoint as down unless
+//! it has seen it fail, and the chain walk ignores it entirely when every
+//! candidate is blocked. It must never be the reason a turn has nowhere to go.
+
+use std::collections::HashMap;
+use std::sync::Mutex;
+use std::time::{Duration, Instant};
+
+use crate::types::CutoverKind;
+
+/// Consecutive cutover-class failures before an endpoint is taken out of
+/// rotation. Two, not one: a single 503 is noise, and opening on it would
+/// bounce a healthy provider out over a blip.
+const FAILURE_THRESHOLD: u32 = 2;
+
+/// Opening cooldown for failures that usually clear quickly (5xx, transport).
+const BASE_COOLDOWN: Duration = Duration::from_secs(60);
+
+/// Opening cooldown for failures that do not (quota windows, bad credentials).
+const SLOW_HEAL_COOLDOWN: Duration = Duration::from_secs(5 * 60);
+
+/// Ceiling on the doubling, so a long outage still gets probed periodically.
+const MAX_COOLDOWN: Duration = Duration::from_secs(15 * 60);
+
+#[derive(Debug)]
+struct Circuit {
+    consecutive_failures: u32,
+    /// `None` while closed. `Some(t)` once opened — the circuit is blocking
+    /// until `t`, and half-open (one probe allowed) from `t` onward.
+    open_until: Option<Instant>,
+    cooldown: Duration,
+}
+
+impl Circuit {
+    fn closed() -> Self {
+        Self {
+            consecutive_failures: 0,
+            open_until: None,
+            cooldown: BASE_COOLDOWN,
+        }
+    }
+}
+
+/// Per-endpoint availability memory shared across the turns of one agent
+/// process. Keyed by [`Endpoint::circuit_key`](crate::config::Endpoint::circuit_key).
+#[derive(Debug, Default)]
+pub struct Breaker {
+    circuits: Mutex<HashMap<String, Circuit>>,
+}
+
+impl Breaker {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Recover the guard rather than propagating a poisoned lock: a panic in
+    /// some other turn's bookkeeping must not take the failover path down with
+    /// it. Worst case the map holds slightly stale counters.
+    fn circuits(&self) -> std::sync::MutexGuard<'_, HashMap<String, Circuit>> {
+        self.circuits.lock().unwrap_or_else(|e| e.into_inner())
+    }
+
+    /// Whether `key` is currently blocked. False for an endpoint that has
+    /// never failed, and false once the cooldown has elapsed (the half-open
+    /// probe).
+    pub fn is_open(&self, key: &str) -> bool {
+        self.is_open_at(key, Instant::now())
+    }
+
+    fn is_open_at(&self, key: &str, now: Instant) -> bool {
+        self.circuits()
+            .get(key)
+            .and_then(|c| c.open_until)
+            .is_some_and(|until| now < until)
+    }
+
+    /// Record a cutover-class failure. Never call this for a stop-class error
+    /// (a 400): a malformed request says nothing about whether the endpoint is
+    /// up, and would take a healthy provider out of rotation.
+    pub fn record_failure(&self, key: &str, kind: CutoverKind) {
+        self.record_failure_at(key, kind, Instant::now());
+    }
+
+    fn record_failure_at(&self, key: &str, kind: CutoverKind, now: Instant) {
+        let mut circuits = self.circuits();
+        let circuit = circuits
+            .entry(key.to_owned())
+            .or_insert_with(Circuit::closed);
+
+        if circuit.open_until.is_some() {
+            // A failed probe, or an attempt forced through because every
+            // candidate was blocked. Back off further; already open, so this
+            // is not a transition.
+            circuit.cooldown = (circuit.cooldown * 2).min(MAX_COOLDOWN);
+            circuit.open_until = Some(now + circuit.cooldown);
+            return;
+        }
+
+        circuit.consecutive_failures += 1;
+        if circuit.consecutive_failures < FAILURE_THRESHOLD {
+            return;
+        }
+        circuit.cooldown = if kind.is_slow_to_heal() {
+            SLOW_HEAL_COOLDOWN
+        } else {
+            BASE_COOLDOWN
+        };
+        circuit.open_until = Some(now + circuit.cooldown);
+        tracing::warn!(
+            endpoint = key,
+            kind = kind.as_str(),
+            cooldown_secs = circuit.cooldown.as_secs(),
+            failures = circuit.consecutive_failures,
+            "llm: endpoint taken out of rotation"
+        );
+    }
+
+    /// Record a success, closing the circuit and clearing the backoff.
+    pub fn record_success(&self, key: &str) {
+        let mut circuits = self.circuits();
+        if let Some(circuit) = circuits.get(key) {
+            if circuit.open_until.is_some() {
+                tracing::info!(endpoint = key, "llm: endpoint recovered");
+            }
+            circuits.remove(key);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const KEY: &str = "anthropic|https://api.z.ai/api/anthropic";
+
+    #[test]
+    fn unknown_endpoint_is_never_blocked() {
+        let breaker = Breaker::new();
+        assert!(!breaker.is_open(KEY));
+    }
+
+    #[test]
+    fn one_failure_does_not_open_the_circuit() {
+        let breaker = Breaker::new();
+        breaker.record_failure(KEY, CutoverKind::Server);
+        assert!(
+            !breaker.is_open(KEY),
+            "a single blip must not take an endpoint out of rotation"
+        );
+    }
+
+    #[test]
+    fn threshold_failures_open_the_circuit() {
+        let breaker = Breaker::new();
+        breaker.record_failure(KEY, CutoverKind::Server);
+        breaker.record_failure(KEY, CutoverKind::Server);
+        assert!(breaker.is_open(KEY));
+    }
+
+    #[test]
+    fn quota_opens_with_the_slow_heal_cooldown() {
+        let breaker = Breaker::new();
+        let now = Instant::now();
+        breaker.record_failure_at(KEY, CutoverKind::Quota, now);
+        breaker.record_failure_at(KEY, CutoverKind::Quota, now);
+
+        // Still blocked well past the 60s a transport failure would have used.
+        assert!(breaker.is_open_at(KEY, now + Duration::from_secs(61)));
+        assert!(breaker.is_open_at(KEY, now + SLOW_HEAL_COOLDOWN - Duration::from_secs(1)));
+        // Half-open once the 5min window elapses.
+        assert!(!breaker.is_open_at(KEY, now + SLOW_HEAL_COOLDOWN));
+    }
+
+    #[test]
+    fn server_failures_open_with_the_short_cooldown() {
+        let breaker = Breaker::new();
+        let now = Instant::now();
+        breaker.record_failure_at(KEY, CutoverKind::Server, now);
+        breaker.record_failure_at(KEY, CutoverKind::Server, now);
+
+        assert!(breaker.is_open_at(KEY, now + Duration::from_secs(59)));
+        assert!(!breaker.is_open_at(KEY, now + BASE_COOLDOWN));
+    }
+
+    #[test]
+    fn a_failed_probe_doubles_the_cooldown() {
+        let breaker = Breaker::new();
+        let now = Instant::now();
+        breaker.record_failure_at(KEY, CutoverKind::Server, now);
+        breaker.record_failure_at(KEY, CutoverKind::Server, now);
+
+        // Cooldown elapses, the probe is allowed, and it fails too.
+        let probe_at = now + BASE_COOLDOWN;
+        assert!(!breaker.is_open_at(KEY, probe_at));
+        breaker.record_failure_at(KEY, CutoverKind::Server, probe_at);
+
+        // Next window is 120s, not another 60s.
+        assert!(breaker.is_open_at(KEY, probe_at + Duration::from_secs(119)));
+        assert!(!breaker.is_open_at(KEY, probe_at + Duration::from_secs(120)));
+    }
+
+    #[test]
+    fn cooldown_doubling_is_capped() {
+        let breaker = Breaker::new();
+        let mut now = Instant::now();
+        breaker.record_failure_at(KEY, CutoverKind::Server, now);
+        breaker.record_failure_at(KEY, CutoverKind::Server, now);
+
+        // Fail probe after probe; the window must never exceed the cap.
+        for _ in 0..12 {
+            now += MAX_COOLDOWN;
+            breaker.record_failure_at(KEY, CutoverKind::Server, now);
+        }
+        assert!(!breaker.is_open_at(KEY, now + MAX_COOLDOWN));
+    }
+
+    #[test]
+    fn success_closes_an_open_circuit() {
+        let breaker = Breaker::new();
+        breaker.record_failure(KEY, CutoverKind::Quota);
+        breaker.record_failure(KEY, CutoverKind::Quota);
+        assert!(breaker.is_open(KEY));
+
+        breaker.record_success(KEY);
+        assert!(!breaker.is_open(KEY));
+    }
+
+    #[test]
+    fn success_resets_the_failure_count() {
+        let breaker = Breaker::new();
+        breaker.record_failure(KEY, CutoverKind::Server);
+        breaker.record_success(KEY);
+        // The earlier failure must not combine with this one to hit the
+        // threshold — otherwise a slow trickle of unrelated blips eventually
+        // opens the circuit on a healthy endpoint.
+        breaker.record_failure(KEY, CutoverKind::Server);
+        assert!(!breaker.is_open(KEY));
+    }
+
+    #[test]
+    fn circuits_are_independent_per_endpoint() {
+        let breaker = Breaker::new();
+        let other = "openrouter|https://openrouter.ai/api/v1";
+        breaker.record_failure(KEY, CutoverKind::Quota);
+        breaker.record_failure(KEY, CutoverKind::Quota);
+
+        assert!(breaker.is_open(KEY));
+        assert!(
+            !breaker.is_open(other),
+            "one provider's quota wall must not bench the rest of the chain"
+        );
+    }
+}
diff --git a/crates/buzz-agent/src/lib.rs b/crates/buzz-agent/src/lib.rs
index 9a45bf4c98..3f092b3fb7 100644
--- a/crates/buzz-agent/src/lib.rs
+++ b/crates/buzz-agent/src/lib.rs
@@ -5,6 +5,7 @@ mod builtin;
 pub mod catalog;
 pub mod config;
 mod handoff;
+mod health;
 mod hints;
 mod llm;
 mod mcp;
diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs
index 73c7e1faf2..5ae53faa47 100644
--- a/crates/buzz-agent/src/llm.rs
+++ b/crates/buzz-agent/src/llm.rs
@@ -1,4 +1,4 @@
-use std::collections::BTreeSet;
+use std::collections::{BTreeSet, HashMap};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 
@@ -10,10 +10,12 @@ use tokio::time::Instant;
 use crate::auth::{PkceOAuthConfig, PkceOAuthTokenSource, StaticTokenSource, TokenSource};
 use crate::config::{
     is_openai_host, normalize_effort_for_anthropic_route, normalize_effort_for_openai_route,
-    Config, OpenAiApi, Provider, ThinkingEffort,
+    Config, Endpoint, OpenAiApi, Provider, ThinkingEffort,
 };
+use crate::health::Breaker;
 use crate::types::{
-    AgentError, HistoryItem, LlmResponse, ProviderStop, ToolCall, ToolDef, ToolResultContent,
+    AgentError, CutoverKind, HistoryItem, LlmResponse, ProviderStop, ToolCall, ToolDef,
+    ToolResultContent,
 };
 
 /// Databricks OAuth client_id — the public Databricks-published CLI client.
@@ -101,7 +103,23 @@ pub struct Llm {
     /// (the `DATABRICKS_TOKEN` env var); a refreshable PKCE engine for
     /// Databricks otherwise. Anthropic doesn't use this — it always
     /// reads `cfg.api_key` directly because the API expects `x-api-key`.
+    ///
+    /// This covers the PRIMARY endpoint only. See [`Llm::auth_for`].
     auth: Arc<dyn TokenSource>,
+    /// Circuit key of the endpoint `auth` belongs to, so a request can tell
+    /// whether it is the primary before reaching for the fallback cache.
+    primary_key: String,
+    /// Token sources for fallback endpoints, built on first use and cached.
+    ///
+    /// A cutover cannot reuse `auth`: it holds the *primary's* credential, and
+    /// sending that to a different provider's base URL is an instant 401. They
+    /// are cached rather than rebuilt per turn because a sustained outage
+    /// means every turn cuts over, and rebuilding would repeat the work on
+    /// each one.
+    fallback_auth: std::sync::Mutex<HashMap<String, Arc<dyn TokenSource>>>,
+    /// Which endpoints are known to be down, so a turn skips them instead of
+    /// re-discovering the outage. See [`crate::health`].
+    breaker: Breaker,
 }
 
 impl Llm {
@@ -117,9 +135,48 @@ impl Llm {
             auto_upgraded: AtomicBool::new(false),
             mesh_auto_state: Mutex::new(MeshAutoState::default()),
             auth,
+            primary_key: cfg.primary_endpoint().circuit_key(),
+            fallback_auth: std::sync::Mutex::new(HashMap::new()),
+            breaker: Breaker::new(),
         })
     }
 
+    /// The token source for whichever endpoint `cfg` points at.
+    ///
+    /// Returns the primary's source unchanged in the common case. On a
+    /// cutover, `cfg` is a [`Config::with_endpoint`] clone aimed elsewhere, and
+    /// this mints (once) and caches that endpoint's own source. Getting this
+    /// wrong is silent and total: the request goes to the fallback's URL
+    /// carrying the primary's key, and every cutover 401s.
+    fn auth_for(&self, cfg: &Config) -> Result<Arc<dyn TokenSource>, AgentError> {
+        let key = cfg.primary_endpoint().circuit_key();
+        if key == self.primary_key {
+            return Ok(Arc::clone(&self.auth));
+        }
+        let mut cache = self
+            .fallback_auth
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        if let Some(existing) = cache.get(&key) {
+            return Ok(Arc::clone(existing));
+        }
+        let source = build_token_source(cfg)?;
+        cache.insert(key, Arc::clone(&source));
+        Ok(source)
+    }
+
+    /// Run one turn, failing over to the next configured provider when the
+    /// current one cannot serve it.
+    ///
+    /// The chain is the primary followed by [`Config::fallback`]. Endpoints the
+    /// circuit breaker has benched are skipped — unless that would leave
+    /// nothing to try, in which case the whole chain is attempted anyway. Stale
+    /// breaker state must never be the reason a turn has nowhere to go.
+    ///
+    /// A stop-class error (malformed request, unknown model) returns
+    /// immediately. Those fail identically on every provider, so walking the
+    /// chain would only reach the same answer more slowly while burning quota
+    /// on the way.
     pub async fn complete(
         &self,
         cfg: &Config,
@@ -127,6 +184,95 @@ impl Llm {
         history: &[HistoryItem],
         tools: &[ToolDef],
         effective_model: &str,
+    ) -> Result<LlmResponse, AgentError> {
+        let primary = cfg.primary_endpoint();
+        // The primary honors the caller's effective model — a
+        // `session/set_model` override arrives that way. Each fallback uses its
+        // own slot's model instead: the primary's model id means nothing to a
+        // different provider.
+        let mut chain: Vec<(&Endpoint, &str)> = Vec::with_capacity(1 + cfg.fallback.len());
+        chain.push((&primary, effective_model));
+        for endpoint in &cfg.fallback {
+            chain.push((endpoint, endpoint.model.as_str()));
+        }
+
+        let any_live = chain
+            .iter()
+            .any(|(endpoint, _)| !self.breaker.is_open(&endpoint.circuit_key()));
+
+        let mut last_error: Option<AgentError> = None;
+        let mut tried = 0usize;
+        for (endpoint, model) in chain {
+            if any_live && self.breaker.is_open(&endpoint.circuit_key()) {
+                continue;
+            }
+            tried += 1;
+            let attempt_cfg;
+            let attempt = if *endpoint == primary {
+                cfg
+            } else {
+                attempt_cfg = cfg.with_endpoint(endpoint);
+                &attempt_cfg
+            };
+            let key = endpoint.circuit_key();
+            match self
+                .complete_once(attempt, system_prompt, history, tools, model)
+                .await
+            {
+                Ok(response) => {
+                    self.breaker.record_success(&key);
+                    if tried > 1 {
+                        tracing::info!(
+                            provider = ?endpoint.provider,
+                            model,
+                            "llm: turn served by a fallback provider"
+                        );
+                    }
+                    return Ok(response);
+                }
+                Err(error) => {
+                    let Some(kind) = cutover_kind(&error) else {
+                        // Stop-class: the same request fails the same way on
+                        // every provider. Surface it rather than burn the chain.
+                        return Err(error);
+                    };
+                    self.breaker.record_failure(&key, kind);
+                    tracing::warn!(
+                        provider = ?endpoint.provider,
+                        model,
+                        kind = kind.as_str(),
+                        error = %error,
+                        "llm: endpoint failed"
+                    );
+                    last_error = Some(error);
+                }
+            }
+        }
+
+        Err(match last_error {
+            // Only reworded when more than one endpoint was in play, so a
+            // single-provider deployment's errors read exactly as before.
+            Some(AgentError::LlmUnavailable { kind, detail }) if tried > 1 => {
+                AgentError::LlmUnavailable {
+                    kind,
+                    detail: format!("all {tried} providers failed; last: {detail}"),
+                }
+            }
+            Some(error) => error,
+            // Unreachable — the chain always holds at least the primary.
+            None => AgentError::Llm("no provider endpoint configured".into()),
+        })
+    }
+
+    /// One turn against exactly one endpoint. No failover: `cfg` names the
+    /// provider, and `effective_model` is that provider's own model.
+    async fn complete_once(
+        &self,
+        cfg: &Config,
+        system_prompt: &str,
+        history: &[HistoryItem],
+        tools: &[ToolDef],
+        effective_model: &str,
     ) -> Result<LlmResponse, AgentError> {
         let effort = cfg.thinking_effort;
         let result = match cfg.provider {
@@ -235,6 +381,10 @@ impl Llm {
             AgentError::LlmModelNotFound(s) => {
                 AgentError::LlmModelNotFound(format!("({effective_model}) {s}"))
             }
+            AgentError::LlmUnavailable { kind, detail } => AgentError::LlmUnavailable {
+                kind,
+                detail: format!("({effective_model}) {detail}"),
+            },
             other => other,
         })
     }
@@ -493,8 +643,17 @@ impl Llm {
 
     async fn observe_mesh_virtual_model(&self, cfg: &Config) -> MeshCatalogObservation {
         let url = format!("{}/models", cfg.base_url.trim_end_matches('/'));
-        let bearer = match self.auth.bearer().await {
-            Ok(bearer) => bearer,
+        let bearer = match self.auth_for(cfg) {
+            Ok(auth) => match auth.bearer().await {
+                Ok(bearer) => bearer,
+                Err(error) => {
+                    tracing::debug!(
+                        %error,
+                        "relay-mesh auto: catalog auth unavailable; preserving last confirmed route"
+                    );
+                    return MeshCatalogObservation::Unknown;
+                }
+            },
             Err(error) => {
                 tracing::debug!(
                     %error,
@@ -649,7 +808,8 @@ impl Llm {
         // rejection can never suppress a later turn's legitimate retry. Both
         // statuses map to `LlmAuth` in `post`: a 403 is indistinguishable from
         // an expired-token 403 here, so we refresh once and let it propagate.
-        let mut bearer = self.auth.bearer().await.map_err(PostError::from)?;
+        let auth = self.auth_for(cfg).map_err(PostError::from)?;
+        let mut bearer = auth.bearer().await.map_err(PostError::from)?;
         let mut refreshed = false;
         loop {
             match post(
@@ -663,11 +823,7 @@ impl Llm {
             {
                 Err(PostError::Agent(AgentError::LlmAuth(_))) if !refreshed => {
                     refreshed = true;
-                    bearer = self
-                        .auth
-                        .refresh_now(&bearer)
-                        .await
-                        .map_err(PostError::from)?;
+                    bearer = auth.refresh_now(&bearer).await.map_err(PostError::from)?;
                 }
                 result => return result,
             }
@@ -676,13 +832,14 @@ impl Llm {
 
     async fn post_openrouter(&self, cfg: &Config, body: &Value) -> Result<Value, AgentError> {
         let url = format!("{}/chat/completions", cfg.base_url.trim_end_matches('/'));
-        let mut bearer = self.auth.bearer().await?;
+        let auth = self.auth_for(cfg)?;
+        let mut bearer = auth.bearer().await?;
         let mut refreshed = false;
         loop {
             match openrouter_post(&self.http, &url, body, &bearer).await {
                 Err(AgentError::LlmAuth(_)) if !refreshed => {
                     refreshed = true;
-                    let new_bearer = self.auth.refresh_now(&bearer).await?;
+                    let new_bearer = auth.refresh_now(&bearer).await?;
                     // A static key refreshes to itself — a byte-identical retry
                     // would be a guaranteed duplicate request against a key the
                     // server just rejected. Fail terminal immediately; the retry
@@ -1706,15 +1863,25 @@ fn is_retryable_transport_error(e: &reqwest::Error) -> bool {
     e.is_timeout() || e.is_connect() || e.is_request()
 }
 
-/// Build the terminal `AgentError::Llm` for a `post()` exit that has given up
-/// retrying — persistent retryable status, transport failure, or a body-read
-/// break. `detail` carries the specific cause (status/body, or the transport
-/// error text); `elapsed` and `attempts` are the cumulative cost of every
-/// attempt made on this call, including the retries that failed before this
-/// one. When cumulative time crosses `STALL_NOTICE_THRESHOLD`, this also logs
-/// a `tracing::warn!` so a slow-building outage is visible in logs even
-/// before an operator reads the returned error text.
-fn terminal_llm_error(elapsed: std::time::Duration, attempts: u32, detail: &str) -> AgentError {
+/// Build the terminal error for a `post()` exit that has given up retrying —
+/// persistent retryable status, transport failure, or a body-read break.
+/// `detail` carries the specific cause (status/body, or the transport error
+/// text); `elapsed` and `attempts` are the cumulative cost of every attempt
+/// made on this call, including the retries that failed before this one. When
+/// cumulative time crosses `STALL_NOTICE_THRESHOLD`, this also logs a
+/// `tracing::warn!` so a slow-building outage is visible in logs even before
+/// an operator reads the returned error text.
+///
+/// Every exit here is cutover-class by construction: retrying the same
+/// endpoint has already been tried and failed, so `kind` records what sort of
+/// failure it was and the caller decides whether another provider can do
+/// better.
+fn terminal_llm_error(
+    kind: CutoverKind,
+    elapsed: std::time::Duration,
+    attempts: u32,
+    detail: &str,
+) -> AgentError {
     if elapsed >= STALL_NOTICE_THRESHOLD {
         tracing::warn!(
             cumulative_stall = ?elapsed,
@@ -1722,10 +1889,31 @@ fn terminal_llm_error(elapsed: std::time::Duration, attempts: u32, detail: &str)
             "llm: cumulative stall {elapsed:?} across {attempts} attempts ({detail})"
         );
     }
-    AgentError::Llm(format!(
-        "{detail} (cumulative {elapsed:?}, {attempts} attempt{})",
-        if attempts == 1 { "" } else { "s" },
-    ))
+    AgentError::LlmUnavailable {
+        kind,
+        detail: format!(
+            "{detail} (cumulative {elapsed:?}, {attempts} attempt{})",
+            if attempts == 1 { "" } else { "s" },
+        ),
+    }
+}
+
+/// Whether `err` is worth retrying on a *different* provider, and what sort of
+/// failure it was.
+///
+/// The distinction that matters: a 400 or an unknown model id fails identically
+/// everywhere, so cutting over just burns the whole chain to reach the same
+/// answer more slowly. Quota, credentials, upstream 5xx, and transport are the
+/// failures another provider genuinely might not share.
+fn cutover_kind(err: &AgentError) -> Option<CutoverKind> {
+    match err {
+        AgentError::LlmUnavailable { kind, .. } => Some(*kind),
+        // The token source already spent its one refresh inside the provider
+        // call. A credential still rejected after that is this endpoint's
+        // problem — another provider's key may be perfectly good.
+        AgentError::LlmAuth(_) => Some(CutoverKind::Auth),
+        _ => None,
+    }
 }
 
 /// Internal HTTP failure that preserves a mesh-specific MoA failure as a
@@ -1816,6 +2004,7 @@ where
                     continue;
                 }
                 return Err(PostError::Agent(terminal_llm_error(
+                    CutoverKind::Transport,
                     call_start.elapsed(),
                     attempt + 1,
                     &format!("transport: {e}"),
@@ -1854,12 +2043,31 @@ where
                 backoff_with_jitter(attempt).await;
                 continue;
             }
+            // 429 is the quota/rate wall (z.ai's weekly `1310` lands here) and
+            // holds for a provider-side window; 5xx/499 is an upstream that is
+            // usually back shortly. The breaker benches them for different
+            // lengths of time.
+            let kind = if status == 429 {
+                CutoverKind::Quota
+            } else {
+                CutoverKind::Server
+            };
             return Err(PostError::Agent(terminal_llm_error(
+                kind,
                 call_start.elapsed(),
                 attempt + 1,
                 &format!("exhausted retries: {status}: {body}"),
             )));
         }
+        // Credit/balance exhaustion. Retrying this endpoint is pointless — the
+        // balance will not change mid-turn — but another provider may still
+        // have credit, so it cuts over rather than stopping the turn.
+        if status == 402 {
+            return Err(PostError::Agent(AgentError::LlmUnavailable {
+                kind: CutoverKind::Quota,
+                detail: format!("{status}: {}", read_error_body(resp).await),
+            }));
+        }
         // Not a stall path: the model is misconfigured, not the transport or
         // upstream capacity — no retry was attempted, so cumulative duration
         // would be misleading.
@@ -1897,6 +2105,7 @@ where
                 Ok(None) => break,
                 Err(e) => {
                     return Err(PostError::Agent(terminal_llm_error(
+                        CutoverKind::Transport,
                         call_start.elapsed(),
                         attempt + 1,
                         &format!("body read: {e}"),
@@ -2077,6 +2286,7 @@ async fn openrouter_post(
                     continue;
                 }
                 return Err(terminal_llm_error(
+                    CutoverKind::Transport,
                     call_start.elapsed(),
                     attempt + 1,
                     &format!("transport: {e}"),
@@ -2101,9 +2311,12 @@ async fn openrouter_post(
             )));
         }
         if status == 402 {
-            return Err(AgentError::Llm(
-                "OpenRouter credits exhausted — check https://openrouter.ai/credits".into(),
-            ));
+            // Cutover-class, not terminal: this account is out of credit, but
+            // a different provider in the chain may not be.
+            return Err(AgentError::LlmUnavailable {
+                kind: CutoverKind::Quota,
+                detail: "OpenRouter credits exhausted — check https://openrouter.ai/credits".into(),
+            });
         }
         if status == 404 {
             // OpenRouter overloads 404: a genuinely unknown/unavailable model id
@@ -2152,6 +2365,7 @@ async fn openrouter_post(
             // Terminal: classify for the user
             return if status == 429 {
                 Err(terminal_llm_error(
+                    CutoverKind::Quota,
                     call_start.elapsed(),
                     attempt + 1,
                     &format!("rate limited: {error_body}"),
@@ -2169,6 +2383,7 @@ async fn openrouter_post(
                     openrouter_parameter_routing_error(&error_body)
                 } else {
                     terminal_llm_error(
+                        CutoverKind::Server,
                         call_start.elapsed(),
                         attempt + 1,
                         &format!("exhausted retries: {status}: {error_body}"),
@@ -2204,6 +2419,7 @@ async fn openrouter_post(
                 Ok(None) => break,
                 Err(e) => {
                     return Err(terminal_llm_error(
+                        CutoverKind::Transport,
                         call_start.elapsed(),
                         attempt + 1,
                         &format!("body read: {e}"),
@@ -2214,6 +2430,7 @@ async fn openrouter_post(
         return serde_json::from_slice(&buf).map_err(|e| AgentError::Llm(format!("json: {e}")));
     }
     Err(terminal_llm_error(
+        CutoverKind::Server,
         call_start.elapsed(),
         MAX_RETRIES,
         "exhausted retries",
@@ -2337,6 +2554,7 @@ mod tests {
     fn cfg(provider: Provider) -> Config {
         Config {
             provider,
+            fallback: Vec::new(),
             system_prompt: "system".into(),
             max_rounds: 10,
             max_output_tokens: 1024,
@@ -2483,6 +2701,11 @@ mod tests {
                 });
                 let status_text = match response.status {
                     200 => "OK",
+                    400 => "Bad Request",
+                    401 => "Unauthorized",
+                    402 => "Payment Required",
+                    404 => "Not Found",
+                    429 => "Too Many Requests",
                     500 => "Internal Server Error",
                     502 => "Bad Gateway",
                     503 => "Service Unavailable",
@@ -4239,7 +4462,10 @@ mod tests {
             .await
             .unwrap_err();
         match &err {
-            PostError::Agent(AgentError::Llm(msg)) => {
+            PostError::Agent(AgentError::LlmUnavailable {
+                kind: CutoverKind::Server,
+                detail: msg,
+            }) => {
                 assert!(
                     msg.contains("exhausted retries") && msg.contains("499"),
                     "expected 'exhausted retries' + '499' in error, got: {msg}"
@@ -4249,7 +4475,9 @@ mod tests {
                     "expected cumulative duration + exact attempt count, got: {msg}"
                 );
             }
-            other => panic!("expected PostError::Agent(AgentError::Llm), got: {other:?}"),
+            other => panic!(
+                "a persistent 499 must be cutover-class Server so the chain can move on, got: {other:?}"
+            ),
         }
         assert_eq!(
             accepts.load(Ordering::SeqCst),
@@ -4263,9 +4491,14 @@ mod tests {
     /// (a handful of quick retries), not an outage.
     #[test]
     fn terminal_llm_error_below_threshold_carries_detail_no_stall_claim() {
-        let err = terminal_llm_error(Duration::from_secs(2), 3, "exhausted retries: 499: body");
+        let err = terminal_llm_error(
+            CutoverKind::Server,
+            Duration::from_secs(2),
+            3,
+            "exhausted retries: 499: body",
+        );
         match err {
-            AgentError::Llm(msg) => {
+            AgentError::LlmUnavailable { detail: msg, .. } => {
                 assert!(msg.contains("cumulative 2s"), "must carry duration: {msg}");
                 assert!(
                     msg.contains("3 attempts"),
@@ -4286,9 +4519,14 @@ mod tests {
     /// Singular "1 attempt" (not "1 attempts") confirms the pluralization.
     #[test]
     fn terminal_llm_error_above_threshold_uses_singular_attempt() {
-        let err = terminal_llm_error(Duration::from_secs(301), 1, "transport: connection reset");
+        let err = terminal_llm_error(
+            CutoverKind::Transport,
+            Duration::from_secs(301),
+            1,
+            "transport: connection reset",
+        );
         match err {
-            AgentError::Llm(msg) => {
+            AgentError::LlmUnavailable { detail: msg, .. } => {
                 assert!(
                     msg.contains("cumulative 301s"),
                     "must carry duration: {msg}"
@@ -4372,7 +4610,12 @@ mod tests {
     #[test]
     fn terminal_llm_error_below_threshold_emits_no_stall_warning() {
         let warnings = count_stall_warnings(|| {
-            let _ = terminal_llm_error(Duration::from_secs(299), 3, "exhausted retries: 499");
+            let _ = terminal_llm_error(
+                CutoverKind::Server,
+                Duration::from_secs(299),
+                3,
+                "exhausted retries: 499",
+            );
         });
         assert_eq!(warnings, 0, "no stall warning below STALL_NOTICE_THRESHOLD");
     }
@@ -4385,7 +4628,12 @@ mod tests {
     #[test]
     fn terminal_llm_error_at_threshold_emits_one_stall_warning() {
         let warnings = count_stall_warnings(|| {
-            let _ = terminal_llm_error(Duration::from_secs(300), 5, "transport: connection reset");
+            let _ = terminal_llm_error(
+                CutoverKind::Transport,
+                Duration::from_secs(300),
+                5,
+                "transport: connection reset",
+            );
         });
         assert_eq!(
             warnings, 1,
@@ -4869,7 +5117,12 @@ mod tests {
         url
     }
 
-    fn llm_with(auth: Arc<dyn TokenSource>) -> Llm {
+    /// An `Llm` whose primary endpoint is `cfg`'s, wired to the supplied token
+    /// source. `cfg` must be fully built (including `base_url`) before this is
+    /// called: `auth_for` matches on the endpoint's circuit key, so an `Llm`
+    /// stamped with a different key would quietly mint its own source and
+    /// ignore the one injected here.
+    fn llm_with(auth: Arc<dyn TokenSource>, cfg: &Config) -> Llm {
         Llm {
             http: Client::builder()
                 .timeout(Duration::from_secs(5))
@@ -4878,6 +5131,9 @@ mod tests {
             auto_upgraded: std::sync::atomic::AtomicBool::new(false),
             mesh_auto_state: Mutex::new(MeshAutoState::default()),
             auth,
+            primary_key: cfg.primary_endpoint().circuit_key(),
+            fallback_auth: std::sync::Mutex::new(HashMap::new()),
+            breaker: Breaker::new(),
         }
     }
 
@@ -4893,9 +5149,9 @@ mod tests {
         let auth = Arc::new(CountingAuth {
             refreshes: AtomicU32::new(0),
         });
-        let llm = llm_with(auth.clone());
         let mut c = cfg(Provider::OpenAi);
         c.base_url = base;
+        let llm = llm_with(auth.clone(), &c);
 
         let out = llm
             .post_openai(&c, "/v1/x", &json!({}), "model")
@@ -4929,9 +5185,9 @@ mod tests {
         let auth = Arc::new(CountingAuth {
             refreshes: AtomicU32::new(0),
         });
-        let llm = llm_with(auth.clone());
         let mut c = cfg(Provider::OpenAi);
         c.base_url = base;
+        let llm = llm_with(auth.clone(), &c);
 
         let err = llm
             .post_openai(&c, "/v1/x", &json!({}), "model")
@@ -4960,9 +5216,9 @@ mod tests {
         let auth = Arc::new(CountingAuth {
             refreshes: AtomicU32::new(0),
         });
-        let llm = llm_with(auth.clone());
         let mut c = cfg(Provider::OpenAi);
         c.base_url = base;
+        let llm = llm_with(auth.clone(), &c);
 
         let err = llm
             .post_openai(&c, "/v1/x", &json!({}), "model")
@@ -4991,9 +5247,9 @@ mod tests {
         let auth = Arc::new(CountingAuth {
             refreshes: AtomicU32::new(0),
         });
-        let llm = llm_with(auth.clone());
         let mut c = cfg(Provider::OpenAi);
         c.base_url = base;
+        let llm = llm_with(auth.clone(), &c);
 
         let out = llm
             .post_openai(&c, "/v1/x", &json!({}), "model")
@@ -6173,8 +6429,12 @@ mod tests {
             .await
             .unwrap_err();
         assert!(
-            matches!(&err, AgentError::Llm(s) if s.contains("credits exhausted")),
-            "got {err:?}"
+            matches!(
+                &err,
+                AgentError::LlmUnavailable { kind: CutoverKind::Quota, detail }
+                    if detail.contains("credits exhausted")
+            ),
+            "402 must be Quota-class so the turn fails over to a provider with credit: got {err:?}"
         );
         assert_eq!(
             attempts.load(std::sync::atomic::Ordering::SeqCst),
@@ -6443,11 +6703,15 @@ mod tests {
             .await
             .unwrap_err();
         assert!(
-            matches!(&err, AgentError::Llm(s) if s.contains("body read")),
-            "truncated body must surface as AgentError::Llm with 'body read': got {err:?}"
+            matches!(
+                &err,
+                AgentError::LlmUnavailable { kind: CutoverKind::Transport, detail }
+                    if detail.contains("body read")
+            ),
+            "a truncated body is a Transport-class failure: got {err:?}"
         );
         assert!(
-            matches!(&err, AgentError::Llm(s) if s.contains("cumulative")),
+            matches!(&err, AgentError::LlmUnavailable { detail, .. } if detail.contains("cumulative")),
             "body-read error must include terminal_llm_error's cumulative context: got {err:?}"
         );
     }
@@ -6503,9 +6767,9 @@ mod tests {
         let auth = Arc::new(StaticAuth {
             token: "static-key".into(),
         });
-        let llm = llm_with(auth);
         let mut c = cfg(Provider::OpenRouter);
         c.base_url = url;
+        let llm = llm_with(auth, &c);
 
         let err = llm.post_openrouter(&c, &json!({})).await.unwrap_err();
         assert!(
@@ -6537,9 +6801,9 @@ mod tests {
             fresh: "fresh".into(),
             refreshes: std::sync::atomic::AtomicU32::new(0),
         });
-        let llm = llm_with(auth.clone());
         let mut c = cfg(Provider::OpenRouter);
         c.base_url = base;
+        let llm = llm_with(auth.clone(), &c);
 
         let result = llm.post_openrouter(&c, &json!({})).await;
         // `spawn_auth_stub` returns `{"ok":true}` on success.
@@ -6553,4 +6817,208 @@ mod tests {
             "exactly one refresh"
         );
     }
+
+    // ---- provider failover ------------------------------------------------
+
+    /// A fallback endpoint pointed at `base_url`, carrying its own model.
+    fn fallback_endpoint(base_url: String, model: &str) -> Endpoint {
+        Endpoint {
+            provider: Provider::OpenAi,
+            api_key: "fallback-key".into(),
+            model: model.into(),
+            base_url,
+            openai_api: OpenAiApi::Chat,
+        }
+    }
+
+    /// The model id each captured request asked for.
+    async fn requested_models(captured: &Arc<Mutex<Vec<CapturedHttpRequest>>>) -> Vec<String> {
+        captured
+            .lock()
+            .await
+            .iter()
+            .filter_map(|r| r.body.as_ref()?.get("model")?.as_str().map(str::to_owned))
+            .collect()
+    }
+
+    #[test]
+    fn cutover_kind_classifies_only_what_another_provider_could_fix() {
+        assert_eq!(
+            cutover_kind(&AgentError::LlmUnavailable {
+                kind: CutoverKind::Quota,
+                detail: String::new()
+            }),
+            Some(CutoverKind::Quota)
+        );
+        // A rejected credential survived its one refresh — another provider's
+        // key may still be good.
+        assert_eq!(
+            cutover_kind(&AgentError::LlmAuth("401".into())),
+            Some(CutoverKind::Auth)
+        );
+        // Stop-class: identical failure everywhere, so cutting over only
+        // wastes the rest of the chain.
+        assert_eq!(
+            cutover_kind(&AgentError::Llm("400 bad request".into())),
+            None
+        );
+        assert_eq!(
+            cutover_kind(&AgentError::LlmModelNotFound("404".into())),
+            None
+        );
+        assert_eq!(cutover_kind(&AgentError::Cancelled), None);
+    }
+
+    /// The outage this feature exists for: the primary is out of quota, and the
+    /// turn is served by the next provider instead of dying.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn complete_fails_over_to_the_next_provider_on_quota() {
+        let (primary_url, primary_seen) =
+            spawn_sequence_stub(vec![StubHttpResponse::error(402, "quota exhausted")]).await;
+        let (fallback_url, fallback_seen) =
+            spawn_sequence_stub(vec![StubHttpResponse::ok(chat_response("from fallback"))]).await;
+
+        let mut c = cfg(Provider::OpenAi);
+        c.base_url = primary_url;
+        c.fallback = vec![fallback_endpoint(fallback_url, "fallback-model")];
+        let llm = Llm::new(&c).unwrap();
+
+        let response = complete_model(&llm, &c, "primary-model")
+            .await
+            .expect("a quota wall on the primary must not fail the turn");
+        assert_eq!(response.text, "from fallback");
+
+        assert_eq!(
+            requested_models(&primary_seen).await,
+            vec!["primary-model"],
+            "the primary is asked first, with the caller's effective model"
+        );
+        assert_eq!(
+            requested_models(&fallback_seen).await,
+            vec!["fallback-model"],
+            "the fallback must be asked for ITS OWN model — sending the \
+             primary's model id to another provider is a guaranteed 404"
+        );
+    }
+
+    /// A malformed request fails the same way everywhere. Walking the chain
+    /// would reach the same answer more slowly while burning the fallbacks.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn complete_does_not_fail_over_on_a_bad_request() {
+        let (primary_url, _) =
+            spawn_sequence_stub(vec![StubHttpResponse::error(400, "malformed")]).await;
+        let (fallback_url, fallback_seen) =
+            spawn_sequence_stub(vec![StubHttpResponse::ok(chat_response("unreachable"))]).await;
+
+        let mut c = cfg(Provider::OpenAi);
+        c.base_url = primary_url;
+        c.fallback = vec![fallback_endpoint(fallback_url, "fallback-model")];
+        let llm = Llm::new(&c).unwrap();
+
+        let err = complete_model(&llm, &c, "primary-model")
+            .await
+            .expect_err("a 400 must surface, not cut over");
+        assert!(
+            matches!(&err, AgentError::Llm(s) if s.contains("400")),
+            "got {err:?}"
+        );
+        assert!(
+            requested_models(&fallback_seen).await.is_empty(),
+            "the fallback must never be contacted for a stop-class failure"
+        );
+    }
+
+    /// When every endpoint is down the turn still fails — but the error says
+    /// the whole chain was tried, not just that one provider misbehaved.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn complete_reports_the_whole_chain_when_all_endpoints_fail() {
+        let (primary_url, primary_seen) =
+            spawn_sequence_stub(vec![StubHttpResponse::error(402, "quota exhausted")]).await;
+        let (fallback_url, fallback_seen) =
+            spawn_sequence_stub(vec![StubHttpResponse::error(402, "also broke")]).await;
+
+        let mut c = cfg(Provider::OpenAi);
+        c.base_url = primary_url;
+        c.fallback = vec![fallback_endpoint(fallback_url, "fallback-model")];
+        let llm = Llm::new(&c).unwrap();
+
+        let err = complete_model(&llm, &c, "primary-model")
+            .await
+            .expect_err("no endpoint could serve the turn");
+        assert!(
+            matches!(
+                &err,
+                AgentError::LlmUnavailable { kind: CutoverKind::Quota, detail }
+                    if detail.contains("all 2 providers failed")
+            ),
+            "got {err:?}"
+        );
+        assert_eq!(requested_models(&primary_seen).await.len(), 1);
+        assert_eq!(requested_models(&fallback_seen).await.len(), 1);
+    }
+
+    /// Once the breaker has benched the primary, later turns skip it outright
+    /// instead of re-paying for the same wall. This is the difference between
+    /// failover costing one slow turn and costing every turn of the outage.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn a_benched_primary_is_skipped_on_later_turns() {
+        let (primary_url, primary_seen) = spawn_sequence_stub(vec![
+            StubHttpResponse::error(402, "quota exhausted"),
+            StubHttpResponse::error(402, "quota exhausted"),
+            StubHttpResponse::error(402, "quota exhausted"),
+        ])
+        .await;
+        let (fallback_url, fallback_seen) = spawn_sequence_stub(vec![
+            StubHttpResponse::ok(chat_response("one")),
+            StubHttpResponse::ok(chat_response("two")),
+            StubHttpResponse::ok(chat_response("three")),
+        ])
+        .await;
+
+        let mut c = cfg(Provider::OpenAi);
+        c.base_url = primary_url;
+        c.fallback = vec![fallback_endpoint(fallback_url, "fallback-model")];
+        let llm = Llm::new(&c).unwrap();
+
+        for _ in 0..3 {
+            complete_model(&llm, &c, "primary-model")
+                .await
+                .expect("the fallback serves every turn");
+        }
+
+        // Two failures open the circuit, so the third turn never touches it.
+        assert_eq!(
+            requested_models(&primary_seen).await.len(),
+            2,
+            "the primary must stop being retried once its circuit opens"
+        );
+        assert_eq!(
+            requested_models(&fallback_seen).await.len(),
+            3,
+            "every turn is still served"
+        );
+    }
+
+    /// With no fallback configured, behavior is exactly what it was before
+    /// failover existed: one endpoint, and its error surfaces unwrapped.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn a_single_provider_deployment_is_unchanged() {
+        let (primary_url, primary_seen) =
+            spawn_sequence_stub(vec![StubHttpResponse::error(402, "quota exhausted")]).await;
+
+        let mut c = cfg(Provider::OpenAi);
+        c.base_url = primary_url;
+        assert!(c.fallback.is_empty());
+        let llm = Llm::new(&c).unwrap();
+
+        let err = complete_model(&llm, &c, "primary-model").await.unwrap_err();
+        assert!(
+            matches!(
+                &err,
+                AgentError::LlmUnavailable { detail, .. } if !detail.contains("providers failed")
+            ),
+            "a single-provider error must not gain chain wording: got {err:?}"
+        );
+        assert_eq!(requested_models(&primary_seen).await.len(), 1);
+    }
 }
diff --git a/crates/buzz-agent/src/types.rs b/crates/buzz-agent/src/types.rs
index 343a75bf72..fcda455c27 100644
--- a/crates/buzz-agent/src/types.rs
+++ b/crates/buzz-agent/src/types.rs
@@ -358,12 +358,58 @@ pub enum ContentBlock {
     Unsupported,
 }
 
+/// Why a provider call failed in a way that a *different* provider could
+/// plausibly satisfy. Distinguishes the failures worth cutting over on from
+/// the ones that would fail identically everywhere (a malformed request), and
+/// sets how long the circuit breaker holds the endpoint down.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum CutoverKind {
+    /// Quota or rate limit — HTTP 429 or 402. Usually bound to a provider-side
+    /// window (z.ai's weekly `1310` resets days later), so retrying the same
+    /// endpoint soon is wasted work.
+    Quota,
+    /// Credentials rejected (401/403) after the token source already spent its
+    /// one refresh. A wrong or revoked key does not heal on its own.
+    Auth,
+    /// Upstream 5xx/499 — the provider is reachable but failing. Often brief.
+    Server,
+    /// Connect, timeout, or body-read failure. Often brief.
+    Transport,
+}
+
+impl CutoverKind {
+    /// Whether this failure is unlikely to clear within a minute. Quota
+    /// windows and bad credentials persist; a 503 or a dropped connection
+    /// usually does not.
+    pub fn is_slow_to_heal(self) -> bool {
+        matches!(self, Self::Quota | Self::Auth)
+    }
+
+    /// Short lowercase label for logs and error text.
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::Quota => "quota",
+            Self::Auth => "auth",
+            Self::Server => "server",
+            Self::Transport => "transport",
+        }
+    }
+}
+
 #[derive(Debug)]
 pub enum AgentError {
     InvalidParams(String),
     Llm(String),
     LlmAuth(String),
     LlmModelNotFound(String),
+    /// The provider could not serve the request for a reason another provider
+    /// might not share — quota, credentials, upstream 5xx, or transport. This
+    /// is the class that drives failover; it surfaces to the caller only once
+    /// every endpoint in the chain has been tried.
+    LlmUnavailable {
+        kind: CutoverKind,
+        detail: String,
+    },
     Mcp(String),
     Cancelled,
 }
@@ -375,6 +421,9 @@ impl std::fmt::Display for AgentError {
             Self::Llm(s) => write!(f, "llm: {s}"),
             Self::LlmAuth(s) => write!(f, "llm auth: {s}"),
             Self::LlmModelNotFound(s) => write!(f, "llm model not found: {s}"),
+            Self::LlmUnavailable { kind, detail } => {
+                write!(f, "llm unavailable ({}): {detail}", kind.as_str())
+            }
             Self::Mcp(s) => write!(f, "mcp: {s}"),
             Self::Cancelled => write!(f, "cancelled"),
         }
@@ -389,6 +438,7 @@ impl AgentError {
             Self::InvalidParams(_) => -32602,
             Self::LlmAuth(_) => -32001,
             Self::LlmModelNotFound(_) => -32002,
+            Self::LlmUnavailable { .. } => -32003,
             _ => -32000,
         }
     }

From 9355cab797ef28e252a89aaaa67d7406a19d4d2c Mon Sep 17 00:00:00 2001
From: Barrett Holien <17663953+BarrettHolien@users.noreply.github.com>
Date: Mon, 3 Aug 2026 10:48:30 -0500
Subject: [PATCH 2/3] feat(desktop): surface provider-unavailable (-32003) with
 actionable copy
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

buzz-agent's failover chain returns AgentError::LlmUnavailable at -32003
when every configured provider failed. Without a case for it the desktop
showed the raw message, which reads as an unexplained agent crash rather
than "your providers are exhausted, add another to the chain".

This surfaces ONLY when the whole chain is down. A turn a fallback
rescued returns Ok and never reaches this path.

Retargets the "unrecognized structured code" fixture from -32003 to
-32004 — that test needs a code nothing claims, and -32003 is now taken.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Barrett Holien <17663953+BarrettHolien@users.noreply.github.com>
---
 .../lib/friendlyAgentLastError.test.mjs       | 31 +++++++++++++++++--
 .../agents/lib/friendlyAgentLastError.ts      | 12 ++++++-
 2 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/desktop/src/features/agents/lib/friendlyAgentLastError.test.mjs b/desktop/src/features/agents/lib/friendlyAgentLastError.test.mjs
index 597b1b9323..c666d1b2cf 100644
--- a/desktop/src/features/agents/lib/friendlyAgentLastError.test.mjs
+++ b/desktop/src/features/agents/lib/friendlyAgentLastError.test.mjs
@@ -6,6 +6,7 @@ import {
   friendlyTurnErrorCopy,
   CLI_ACP_INTERNAL_ERROR_COPY,
   MODEL_NOT_FOUND_COPY,
+  PROVIDER_UNAVAILABLE_COPY,
   RELAY_MESH_DENIED_COPY,
 } from "./friendlyAgentLastError.ts";
 
@@ -153,11 +154,13 @@ test("friendlyTurnErrorCopy: unknown code passes raw text through", () => {
 // --- structured-code hardening ---
 
 test("unknown code prevents string-pattern cross-classification", () => {
-  // code -32003 is structured and unrecognized — must NOT fall through to
+  // code -32004 is structured and unrecognized — must NOT fall through to
   // the legacy string path that would wrongly promote this to denied.
+  // (-32003 was this fixture until buzz-agent claimed it for provider
+  // failover; an unclaimed code is the whole point of the test.)
   const result = friendlyAgentLastError(
     "llm auth: rate limiter denial",
-    -32003,
+    -32004,
   );
   assert.deepEqual(result, {
     severity: "generic",
@@ -165,6 +168,30 @@ test("unknown code prevents string-pattern cross-classification", () => {
   });
 });
 
+test("code -32003 → provider-unavailable copy (severity: generic)", () => {
+  // The failover chain walked every configured provider and none recovered
+  // the turn (e.g. z.ai's `1310` weekly limit with no fallback left).
+  const result = friendlyAgentLastError(
+    "Agent reported error (code -32003): llm unavailable (quota): (glm-5.2) all 2 providers failed; last: exhausted retries: 429",
+    -32003,
+  );
+  assert.deepEqual(result, {
+    severity: "generic",
+    copy: PROVIDER_UNAVAILABLE_COPY,
+  });
+});
+
+test("embedded code -32003 recovered from message → provider-unavailable copy", () => {
+  const result = friendlyAgentLastError(
+    "Agent reported error (code -32003): llm unavailable (quota): exhausted retries: 429",
+    null,
+  );
+  assert.deepEqual(result, {
+    severity: "generic",
+    copy: PROVIDER_UNAVAILABLE_COPY,
+  });
+});
+
 test("NaN code param treated as absent — string path applies", () => {
   // NaN is not finite; falls back to string matching.
   const result = friendlyAgentLastError("llm auth: denied", NaN);
diff --git a/desktop/src/features/agents/lib/friendlyAgentLastError.ts b/desktop/src/features/agents/lib/friendlyAgentLastError.ts
index 60c77bb04c..910b688010 100644
--- a/desktop/src/features/agents/lib/friendlyAgentLastError.ts
+++ b/desktop/src/features/agents/lib/friendlyAgentLastError.ts
@@ -4,7 +4,8 @@
  * The error classification seam flows like this:
  *   buzz-agent — classifies LLM failures into `AgentError` variants with
  *                  JSON-RPC codes (`-32001` auth, `-32002` model-not-found,
- *                  `-32000` generic), defined in `crates/buzz-agent/src/types.rs`.
+ *                  `-32003` provider-unavailable, `-32000` generic), defined in
+ *                  `crates/buzz-agent/src/types.rs`.
  *   buzz-acp   — preserves the code structurally in
  *                  `AcpError::AgentError { code, message }`, whose Display is
  *                  `"Agent reported error (code N): message"`, and includes
@@ -42,6 +43,9 @@ export const RELAY_MESH_DENIED_COPY =
 export const MODEL_NOT_FOUND_COPY =
   "The configured model is not available — open agent settings and select a different one from the dropdown.";
 
+export const PROVIDER_UNAVAILABLE_COPY =
+  "The model provider is unavailable — out of quota, rate-limited, or down — and no fallback provider recovered the turn. Add another provider to the fallback chain (Ollama Cloud or OpenRouter via BUZZ_AGENT_FALLBACK_PROVIDERS) or wait for the quota to reset.";
+
 export const CLI_ACP_INTERNAL_ERROR_COPY =
   "The agent's harness reported an internal error. For Codex agents this can mean the configured model isn't supported by your installed codex-acp — check the model in `~/.codex/config.toml` or upgrade the adapter (`brew upgrade codex-acp`).";
 
@@ -81,6 +85,12 @@ export function friendlyAgentLastError(
         return { severity: "denied", copy: RELAY_MESH_DENIED_COPY };
       case -32002:
         return { severity: "denied", copy: MODEL_NOT_FOUND_COPY };
+      case -32003:
+        // Provider unavailable (quota, credentials, upstream 5xx, transport).
+        // buzz-agent's failover chain walked every configured provider and
+        // none recovered the turn — this surfaces ONLY when the whole chain is
+        // down. A turn a fallback rescued returns Ok and never lands here.
+        return { severity: "generic", copy: PROVIDER_UNAVAILABLE_COPY };
       case -32603: {
         // Standard JSON-RPC "Internal error" — emitted by external harnesses
         // (e.g. codex-acp) when the configured model is unsupported. Only

From 7513e801bd522b115f4bf0677bbdb8d1058f7f00 Mon Sep 17 00:00:00 2001
From: Barrett Holien <17663953+BarrettHolien@users.noreply.github.com>
Date: Mon, 3 Aug 2026 11:28:35 -0500
Subject: [PATCH 3/3] feat(buzz-agent): fail over on summarize too, not just
 completion
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Summarization drives handoff and context compaction. Leaving it on the
primary alone meant a quota wall still stranded a long conversation at
the context ceiling — the same outage the completion path now survives,
arriving by another door.

Extracts the chain walk both paths now share: `attempt_chain` builds the
ordered endpoints (primary on the caller's effective model, each fallback
on its own slot's model), drops the ones the breaker has benched unless
that would leave nothing, and hands back a Config aimed at each.
`chain_exhausted` words the final error identically for both.

The breaker is shared, so a primary benched by failed turns is skipped by
summarization too rather than each path paying to rediscover the outage.

Also pins BUZZ_AGENT_FALLBACK_PROVIDERS=none in the integration harness.
`Command::env` adds to the inherited environment rather than replacing
it, so on a box exporting a real provider key AND model these tests would
auto-derive a chain and answer from a live provider the moment a canned
response looked cutover-class — nondeterministic, and billable.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Barrett Holien <17663953+BarrettHolien@users.noreply.github.com>
---
 crates/buzz-agent/src/llm.rs        | 269 ++++++++++++++++++++++------
 crates/buzz-agent/tests/fake_llm.rs |   9 +
 2 files changed, 219 insertions(+), 59 deletions(-)

diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs
index 5ae53faa47..c20090416e 100644
--- a/crates/buzz-agent/src/llm.rs
+++ b/crates/buzz-agent/src/llm.rs
@@ -87,6 +87,59 @@ enum DatabricksV2Route {
     MlflowChatCompletions,
 }
 
+/// One endpoint to try, fully resolved: the config aimed at it, the model to
+/// ask it for, and the circuit key its outcome is recorded against.
+struct Attempt {
+    key: String,
+    provider: Provider,
+    cfg: Config,
+    model: String,
+}
+
+impl Attempt {
+    /// Record that a fallback rescued a call the primary could not serve.
+    /// `index` is the position in the chain, so 0 (the primary) says nothing —
+    /// the ordinary case should not produce log noise.
+    fn note_rescue(&self, index: usize, what: &str) {
+        if index > 0 {
+            tracing::info!(
+                provider = ?self.provider,
+                model = %self.model,
+                "llm: {what} served by a fallback provider"
+            );
+        }
+    }
+
+    fn note_failure(&self, kind: CutoverKind, error: &AgentError) {
+        tracing::warn!(
+            provider = ?self.provider,
+            model = %self.model,
+            kind = kind.as_str(),
+            error = %error,
+            "llm: endpoint failed"
+        );
+    }
+}
+
+/// The error to surface once every endpoint in the chain has failed.
+///
+/// Reworded only when more than one endpoint was in play, so a
+/// single-provider deployment's errors read exactly as they did before
+/// failover existed.
+fn chain_exhausted(last_error: Option<AgentError>, tried: usize) -> AgentError {
+    match last_error {
+        Some(AgentError::LlmUnavailable { kind, detail }) if tried > 1 => {
+            AgentError::LlmUnavailable {
+                kind,
+                detail: format!("all {tried} providers failed; last: {detail}"),
+            }
+        }
+        Some(error) => error,
+        // Unreachable — the chain always holds at least the primary.
+        None => AgentError::Llm("no provider endpoint configured".into()),
+    }
+}
+
 pub struct Llm {
     http: Client,
     /// One-shot sticky flag: set when a Chat Completions request comes
@@ -185,49 +238,17 @@ impl Llm {
         tools: &[ToolDef],
         effective_model: &str,
     ) -> Result<LlmResponse, AgentError> {
-        let primary = cfg.primary_endpoint();
-        // The primary honors the caller's effective model — a
-        // `session/set_model` override arrives that way. Each fallback uses its
-        // own slot's model instead: the primary's model id means nothing to a
-        // different provider.
-        let mut chain: Vec<(&Endpoint, &str)> = Vec::with_capacity(1 + cfg.fallback.len());
-        chain.push((&primary, effective_model));
-        for endpoint in &cfg.fallback {
-            chain.push((endpoint, endpoint.model.as_str()));
-        }
-
-        let any_live = chain
-            .iter()
-            .any(|(endpoint, _)| !self.breaker.is_open(&endpoint.circuit_key()));
-
+        let chain = self.attempt_chain(cfg, effective_model);
+        let total = chain.len();
         let mut last_error: Option<AgentError> = None;
-        let mut tried = 0usize;
-        for (endpoint, model) in chain {
-            if any_live && self.breaker.is_open(&endpoint.circuit_key()) {
-                continue;
-            }
-            tried += 1;
-            let attempt_cfg;
-            let attempt = if *endpoint == primary {
-                cfg
-            } else {
-                attempt_cfg = cfg.with_endpoint(endpoint);
-                &attempt_cfg
-            };
-            let key = endpoint.circuit_key();
+        for (index, attempt) in chain.into_iter().enumerate() {
             match self
-                .complete_once(attempt, system_prompt, history, tools, model)
+                .complete_once(&attempt.cfg, system_prompt, history, tools, &attempt.model)
                 .await
             {
                 Ok(response) => {
-                    self.breaker.record_success(&key);
-                    if tried > 1 {
-                        tracing::info!(
-                            provider = ?endpoint.provider,
-                            model,
-                            "llm: turn served by a fallback provider"
-                        );
-                    }
+                    self.breaker.record_success(&attempt.key);
+                    attempt.note_rescue(index, "turn");
                     return Ok(response);
                 }
                 Err(error) => {
@@ -236,32 +257,44 @@ impl Llm {
                         // every provider. Surface it rather than burn the chain.
                         return Err(error);
                     };
-                    self.breaker.record_failure(&key, kind);
-                    tracing::warn!(
-                        provider = ?endpoint.provider,
-                        model,
-                        kind = kind.as_str(),
-                        error = %error,
-                        "llm: endpoint failed"
-                    );
+                    self.breaker.record_failure(&attempt.key, kind);
+                    attempt.note_failure(kind, &error);
                     last_error = Some(error);
                 }
             }
         }
+        Err(chain_exhausted(last_error, total))
+    }
 
-        Err(match last_error {
-            // Only reworded when more than one endpoint was in play, so a
-            // single-provider deployment's errors read exactly as before.
-            Some(AgentError::LlmUnavailable { kind, detail }) if tried > 1 => {
-                AgentError::LlmUnavailable {
-                    kind,
-                    detail: format!("all {tried} providers failed; last: {detail}"),
-                }
-            }
-            Some(error) => error,
-            // Unreachable — the chain always holds at least the primary.
-            None => AgentError::Llm("no provider endpoint configured".into()),
-        })
+    /// The endpoints to try for this call, in order, each paired with the
+    /// config aimed at it.
+    ///
+    /// The primary honors the caller's effective model — a `session/set_model`
+    /// override arrives that way. Each fallback uses its own slot's model
+    /// instead: the primary's model id means nothing to a different provider.
+    ///
+    /// Endpoints the breaker has benched are dropped, unless that would leave
+    /// nothing to try — then the whole chain comes back. Stale breaker state
+    /// must never be the reason a call has nowhere to go.
+    fn attempt_chain(&self, cfg: &Config, effective_model: &str) -> Vec<Attempt> {
+        let mut candidates: Vec<(Endpoint, &str)> = Vec::with_capacity(1 + cfg.fallback.len());
+        candidates.push((cfg.primary_endpoint(), effective_model));
+        for endpoint in &cfg.fallback {
+            candidates.push((endpoint.clone(), endpoint.model.as_str()));
+        }
+        let any_live = candidates
+            .iter()
+            .any(|(endpoint, _)| !self.breaker.is_open(&endpoint.circuit_key()));
+        candidates
+            .into_iter()
+            .filter(|(endpoint, _)| !any_live || !self.breaker.is_open(&endpoint.circuit_key()))
+            .map(|(endpoint, model)| Attempt {
+                key: endpoint.circuit_key(),
+                provider: endpoint.provider,
+                cfg: cfg.with_endpoint(&endpoint),
+                model: model.to_owned(),
+            })
+            .collect()
     }
 
     /// One turn against exactly one endpoint. No failover: `cfg` names the
@@ -389,6 +422,12 @@ impl Llm {
         })
     }
 
+    /// Summarize, failing over exactly as [`Llm::complete`] does.
+    ///
+    /// This is not a nice-to-have. Summarization drives handoff and context
+    /// compaction, so a primary that cannot serve it strands a long
+    /// conversation at the context ceiling with no way to continue — the same
+    /// outage the completion path exists to survive, arriving by another door.
     pub async fn summarize(
         &self,
         cfg: &Config,
@@ -396,6 +435,47 @@ impl Llm {
         user_prompt: &str,
         max_output_tokens: u32,
         effective_model: &str,
+    ) -> Result<String, AgentError> {
+        let chain = self.attempt_chain(cfg, effective_model);
+        let total = chain.len();
+        let mut last_error: Option<AgentError> = None;
+        for (index, attempt) in chain.into_iter().enumerate() {
+            match self
+                .summarize_once(
+                    &attempt.cfg,
+                    system_prompt,
+                    user_prompt,
+                    max_output_tokens,
+                    &attempt.model,
+                )
+                .await
+            {
+                Ok(text) => {
+                    self.breaker.record_success(&attempt.key);
+                    attempt.note_rescue(index, "summary");
+                    return Ok(text);
+                }
+                Err(error) => {
+                    let Some(kind) = cutover_kind(&error) else {
+                        return Err(error);
+                    };
+                    self.breaker.record_failure(&attempt.key, kind);
+                    attempt.note_failure(kind, &error);
+                    last_error = Some(error);
+                }
+            }
+        }
+        Err(chain_exhausted(last_error, total))
+    }
+
+    /// One summary against exactly one endpoint. No failover.
+    async fn summarize_once(
+        &self,
+        cfg: &Config,
+        system_prompt: &str,
+        user_prompt: &str,
+        max_output_tokens: u32,
+        effective_model: &str,
     ) -> Result<String, AgentError> {
         match cfg.provider {
             Provider::Anthropic => {
@@ -7021,4 +7101,75 @@ mod tests {
         );
         assert_eq!(requested_models(&primary_seen).await.len(), 1);
     }
+
+    /// Summarization drives handoff and context compaction. Without failover a
+    /// quota wall strands a long conversation at the context ceiling — the same
+    /// outage the completion path survives, arriving by another door.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn summarize_fails_over_to_the_next_provider() {
+        let (primary_url, primary_seen) =
+            spawn_sequence_stub(vec![StubHttpResponse::error(402, "quota exhausted")]).await;
+        let (fallback_url, fallback_seen) = spawn_sequence_stub(vec![StubHttpResponse::ok(
+            chat_response("summary from fallback"),
+        )])
+        .await;
+
+        let mut c = cfg(Provider::OpenAi);
+        c.base_url = primary_url;
+        c.fallback = vec![fallback_endpoint(fallback_url, "fallback-model")];
+        let llm = Llm::new(&c).unwrap();
+
+        let text = llm
+            .summarize(&c, "system", "summarize this", 256, "primary-model")
+            .await
+            .expect("a quota wall on the primary must not strand compaction");
+        assert_eq!(text, "summary from fallback");
+
+        assert_eq!(requested_models(&primary_seen).await, vec!["primary-model"]);
+        assert_eq!(
+            requested_models(&fallback_seen).await,
+            vec!["fallback-model"],
+            "the summary must be asked of the fallback's own model"
+        );
+    }
+
+    /// The breaker is shared across both paths. A primary benched by failed
+    /// turns is skipped by summarization too, rather than each path paying to
+    /// rediscover the same outage.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn a_primary_benched_by_turns_is_skipped_by_summarize() {
+        let (primary_url, primary_seen) = spawn_sequence_stub(vec![
+            StubHttpResponse::error(402, "quota exhausted"),
+            StubHttpResponse::error(402, "quota exhausted"),
+        ])
+        .await;
+        let (fallback_url, fallback_seen) = spawn_sequence_stub(vec![
+            StubHttpResponse::ok(chat_response("one")),
+            StubHttpResponse::ok(chat_response("two")),
+            StubHttpResponse::ok(chat_response("the summary")),
+        ])
+        .await;
+
+        let mut c = cfg(Provider::OpenAi);
+        c.base_url = primary_url;
+        c.fallback = vec![fallback_endpoint(fallback_url, "fallback-model")];
+        let llm = Llm::new(&c).unwrap();
+
+        // Two turns fail the primary over — that opens its circuit.
+        for _ in 0..2 {
+            complete_model(&llm, &c, "primary-model").await.unwrap();
+        }
+        let text = llm
+            .summarize(&c, "system", "summarize this", 256, "primary-model")
+            .await
+            .unwrap();
+        assert_eq!(text, "the summary");
+
+        assert_eq!(
+            requested_models(&primary_seen).await.len(),
+            2,
+            "summarize must not re-probe a primary the turns already benched"
+        );
+        assert_eq!(requested_models(&fallback_seen).await.len(), 3);
+    }
 }
diff --git a/crates/buzz-agent/tests/fake_llm.rs b/crates/buzz-agent/tests/fake_llm.rs
index f782a9d476..eb328ada97 100644
--- a/crates/buzz-agent/tests/fake_llm.rs
+++ b/crates/buzz-agent/tests/fake_llm.rs
@@ -155,6 +155,15 @@ impl Harness {
             .env("OPENAI_COMPAT_API_KEY", "test")
             .env("OPENAI_COMPAT_MODEL", "fake-model")
             .env("OPENAI_COMPAT_BASE_URL", base_url)
+            // Pin failover OFF. `cmd.env()` adds to the inherited environment
+            // rather than replacing it, so on a developer box or CI runner that
+            // exports a real OPENROUTER_/OLLAMA_ key AND model, the agent would
+            // auto-derive a fallback chain and answer these tests from a REAL
+            // provider the moment a canned response looked cutover-class —
+            // nondeterministic, and billable. These tests assert
+            // single-provider behavior; a test that wants failover sets its own
+            // chain.
+            .env("BUZZ_AGENT_FALLBACK_PROVIDERS", "none")
             .env("BUZZ_AGENT_LLM_TIMEOUT_SECS", "5")
             .env("BUZZ_AGENT_TOOL_TIMEOUT_SECS", "5")
             .env("BUZZ_AGENT_MAX_ROUNDS", "4")