diff --git a/src/loader.ts b/src/loader.ts index b8d193e..ede212a 100644 --- a/src/loader.ts +++ b/src/loader.ts @@ -97,6 +97,49 @@ function createCustomModel(provider: string, modelId: string, baseUrl: string): }; } +/** + * Resolve a "provider:model" string into a pi-ai Model, handling custom + * endpoints (`provider:model@base-url`), the GITAGENT_MODEL_BASE_URL override, + * and API-key normalization for non-built-in providers. + * + * Shared by the preferred model and every manifest fallback entry so all + * candidates resolve identically. + */ +export function resolveModel(modelStr: string): Model { + const { provider, modelId } = parseModelString(modelStr); + const envBaseUrl = process.env.GITAGENT_MODEL_BASE_URL; + + let model: Model; + if (modelId.includes("@")) { + // Custom endpoint: provider:model-id@base-url + const atIndex = modelId.indexOf("@"); + model = createCustomModel(provider, modelId.slice(0, atIndex), modelId.slice(atIndex + 1)); + } else if (envBaseUrl) { + // Environment-specified base URL overrides all providers + model = createCustomModel(provider, modelId, envBaseUrl); + } else { + // Standard registered model + model = getModel(provider as any, modelId as any); + } + + // For custom providers not in pi-ai's env key map, ensure an API key is available. + // pi-ai calls getEnvApiKey(model.provider) which only knows built-in providers. + // For unknown providers using openai-completions API, set provider to "openai" so + // pi-ai finds OPENAI_API_KEY. The actual auth happens via custom headers on the model. + const knownProviders = new Set(["openai", "anthropic", "google", "google-vertex", "groq", "cerebras", "xai", "openrouter", "mistral", "amazon-bedrock", "azure-openai-responses", "huggingface", "opencode", "kimi-coding", "github-copilot"]); + if (model.baseUrl && !knownProviders.has(provider)) { + // Use provider-specific key if available, otherwise use LYZR key or dummy + const providerKey = process.env[`${provider.toUpperCase()}_API_KEY`] || process.env.LYZR_API_KEY; + if (providerKey && !process.env.OPENAI_API_KEY) { + process.env.OPENAI_API_KEY = providerKey; + } + // Override provider to "openai" so pi-ai resolves the API key correctly + (model as any).provider = "openai"; + } + + return model; +} + async function ensureGitagentDir(agentDir: string): Promise { const gitagentDir = join(agentDir, ".gitagent"); await mkdir(gitagentDir, { recursive: true }); @@ -129,6 +172,9 @@ export interface LoadedAgent { systemPrompt: string; manifest: AgentManifest; model: Model; + /** Fallback models (resolved from manifest.model.fallback), tried in order + * when the primary fails with a retryable provider error. */ + fallbackModels: Model[]; skills: SkillMetadata[]; knowledge: LoadedKnowledge; workflows: WorkflowMetadata[]; @@ -387,41 +433,30 @@ Do NOT track trivial single-command tasks (e.g. "what time is it"). But DO check ); } - const { provider, modelId } = parseModelString(modelStr); - const envBaseUrl = process.env.GITAGENT_MODEL_BASE_URL; - - let model: Model; - if (modelId.includes("@")) { - // Custom endpoint: provider:model-id@base-url - const atIndex = modelId.indexOf("@"); - model = createCustomModel(provider, modelId.slice(0, atIndex), modelId.slice(atIndex + 1)); - } else if (envBaseUrl) { - // Environment-specified base URL overrides all providers - model = createCustomModel(provider, modelId, envBaseUrl); - } else { - // Standard registered model - model = getModel(provider as any, modelId as any); - } - - // For custom providers not in pi-ai's env key map, ensure an API key is available. - // pi-ai calls getEnvApiKey(model.provider) which only knows built-in providers. - // For unknown providers using openai-completions API, set provider to "openai" so - // pi-ai finds OPENAI_API_KEY. The actual auth happens via custom headers on the model. - const knownProviders = new Set(["openai", "anthropic", "google", "google-vertex", "groq", "cerebras", "xai", "openrouter", "mistral", "amazon-bedrock", "azure-openai-responses", "huggingface", "opencode", "kimi-coding", "github-copilot"]); - if (model.baseUrl && !knownProviders.has(provider)) { - // Use provider-specific key if available, otherwise use LYZR key or dummy - const providerKey = process.env[`${provider.toUpperCase()}_API_KEY`] || process.env.LYZR_API_KEY; - if (providerKey && !process.env.OPENAI_API_KEY) { - process.env.OPENAI_API_KEY = providerKey; + const model = resolveModel(modelStr); + + // Resolve fallback models declared in the manifest. Used at runtime when the + // primary provider fails with a retryable error (e.g. credit balance too low). + // Entries are trimmed and de-duplicated (including against the preferred + // model); blanks and unparseable entries are skipped rather than failing load. + const fallbackModels: Model[] = []; + const seenModelStrings = new Set([modelStr.trim()]); + for (const rawFb of manifest.model.fallback ?? []) { + const fb = (rawFb ?? "").trim(); + if (!fb || seenModelStrings.has(fb)) continue; + seenModelStrings.add(fb); + try { + fallbackModels.push(resolveModel(fb)); + } catch { + // Skip unparseable fallback entries — they shouldn't break startup. } - // Override provider to "openai" so pi-ai resolves the API key correctly - (model as any).provider = "openai"; } return { systemPrompt, manifest, model, + fallbackModels, skills, knowledge, workflows, diff --git a/src/model-fallback.ts b/src/model-fallback.ts new file mode 100644 index 0000000..52c6f30 --- /dev/null +++ b/src/model-fallback.ts @@ -0,0 +1,53 @@ +/** + * Model fallback support. + * + * `manifest.model.fallback` lets an agent declare alternate models to use when + * the primary provider fails. Historically this list was parsed but never used + * at runtime — a single provider outage (e.g. Anthropic "credit balance too + * low") would kill the run even when a working provider was also configured + * (issue #24). This module decides which provider errors are worth retrying on + * a different model. + */ + +// Signatures of provider-side failures where switching to a different model +// (usually a different provider/account) is likely to succeed. Deliberately +// conservative: we only retry on availability/billing/auth-class errors, never +// on errors caused by the request itself (bad input, content filter, etc.). +const RETRYABLE_PATTERNS: RegExp[] = [ + /credit balance/i, + /insufficient (?:credit|quota|funds|balance)/i, + /quota/i, + /billing/i, + /payment/i, + /rate.?limit/i, + /\b429\b/, + /overloaded/i, + /\b529\b/, + /temporarily unavailable/i, + /service unavailable/i, + /\b503\b/, + /\b502\b/, + /\b500\b/, + /internal server error/i, + /timeout/i, + /timed out/i, + /econnreset|econnrefused|etimedout|enotfound/i, + /authentication|unauthorized|invalid api key|invalid x-api-key|permission/i, + /\b401\b/, + /\b403\b/, +]; + +/** + * Whether a provider error message indicates the request should be retried on + * the next fallback model. + * + * - Empty/missing message → retry: we have no detail, so give the next model a + * chance rather than dead-ending on an opaque failure. + * - Recognized availability/billing/auth pattern → retry. + * - Any other non-empty message (e.g. a malformed-request error) → do not + * retry: switching models won't fix a bad request. + */ +export function isRetryableProviderError(message: string | undefined | null): boolean { + if (!message) return true; + return RETRYABLE_PATTERNS.some((re) => re.test(message)); +} diff --git a/src/sdk-types.ts b/src/sdk-types.ts index 20dc060..dce164f 100644 --- a/src/sdk-types.ts +++ b/src/sdk-types.ts @@ -52,7 +52,7 @@ export interface GCToolResultMessage { export interface GCSystemMessage { type: "system"; subtype: "session_start" | "session_end" | "hook_blocked" - | "compliance_warning" | "error"; + | "compliance_warning" | "error" | "fallback"; content: string; metadata?: Record; } diff --git a/src/sdk.ts b/src/sdk.ts index dfa9a80..f9d2bb5 100644 --- a/src/sdk.ts +++ b/src/sdk.ts @@ -23,6 +23,7 @@ import type { SandboxOptions, } from "./sdk-types.js"; import { CostTracker } from "./cost-tracker.js"; +import { isRetryableProviderError } from "./model-fallback.js"; import { context as otelContext } from "@opentelemetry/api"; import { wrapToolWithOtel, @@ -298,26 +299,45 @@ export function query(options: QueryOptions): Query { modelOptions.maxTurns = options.maxTurns; } - // 8. Create Agent + // 8. Fallback-aware agent runner. + // The manifest's preferred model plus any resolved fallbacks form an + // ordered candidate list. When a model fails with a retryable provider + // error before producing output (e.g. "credit balance too low"), the + // runner swaps the model on the same agent and retries (issue #24). + const candidateModels = [loaded.model, ...loaded.fallbackModels]; + + // Controller state shared across attempts. + let sessionStartEmitted = false; + interface AttemptState { + producedOutput: boolean; + error: { message?: string; provider?: string; model?: string; api?: string } | null; + } + let attempt: AttemptState = { producedOutput: false, error: null }; + + // 9. Build an Agent for a given model and wire event → GCMessage mapping. + const buildAgent = (model: typeof loaded.model): Agent => { const agent = new Agent({ initialState: { systemPrompt, - model: loaded.model, + model, tools, ...modelOptions, }, }); - // 9. Subscribe to events and map to GCMessage agent.subscribe((event: AgentEvent) => { switch (event.type) { case "agent_start": - pushMsg({ - type: "system", - subtype: "session_start", - content: `Agent ${loaded.manifest.name} started`, - metadata: { sessionId: _sessionId }, - }); + // Emit once even across fallback attempts. + if (!sessionStartEmitted) { + sessionStartEmitted = true; + pushMsg({ + type: "system", + subtype: "session_start", + content: `Agent ${loaded.manifest.name} started`, + metadata: { sessionId: _sessionId }, + }); + } break; case "message_update": { @@ -329,6 +349,7 @@ export function query(options: QueryOptions): Query { _llmCallStart = Date.now(); } if (e.type === "text_delta") { + attempt.producedOutput = true; accText += e.delta; pushMsg({ type: "delta", @@ -336,6 +357,7 @@ export function query(options: QueryOptions): Query { content: e.delta, }); } else if (e.type === "thinking_delta") { + attempt.producedOutput = true; accThinking += e.delta; pushMsg({ type: "delta", @@ -353,19 +375,22 @@ export function query(options: QueryOptions): Query { const msg = raw as AssistantMessage; - // Emit error system message if the LLM call failed + // Buffer a failed LLM call instead of emitting immediately — the + // fallback runner decides whether to retry on the next model or + // surface it. Only emitted (below, or by the runner) when terminal. if (msg.stopReason === "error") { - pushMsg({ - type: "system", - subtype: "error", - content: msg.errorMessage || "LLM request failed (unknown error)", - metadata: { - model: msg.model, - provider: msg.provider, - api: (msg as any).api, - }, - }); - // Still emit the assistant message so callers can inspect stopReason + attempt.error = { + message: msg.errorMessage || "LLM request failed (unknown error)", + model: msg.model, + provider: msg.provider, + api: (msg as any).api, + }; + // Reset accumulators + LLM-call timer (a partial stream may have + // set it) and skip cost tracking (usage is empty on error). + accText = ""; + accThinking = ""; + _llmCallStart = 0; + break; } const { text, thinking } = extractContent(msg); @@ -430,6 +455,8 @@ export function query(options: QueryOptions): Query { } case "tool_execution_start": + // A tool ran, so the model committed to output — no safe retry. + attempt.producedOutput = true; toolArgsMap.set(event.toolCallId, event.args ?? {}); pushMsg({ type: "tool_use", @@ -474,17 +501,112 @@ export function query(options: QueryOptions): Query { break; } - case "agent_end": + case "agent_end": { + // A thrown provider error (network reset, immediate reject) is + // reported by pi-agent-core via handleRunFailure, which emits + // agent_end with a trailing errored assistant message and NO + // message_end. Capture that here so fallback still triggers. + if (!attempt.error) { + const last = event.messages?.[event.messages.length - 1] as any; + if (last?.role === "assistant" && last.stopReason === "error") { + attempt.error = { + message: last.errorMessage || "LLM request failed (unknown error)", + model: last.model, + provider: last.provider, + api: last.api, + }; + } + } + // Do not finish the channel here — the fallback runner may start + // another attempt. session_end + finish happen once, after the + // runner settles (see finalizeRun()). + break; + } + } + }); + + return agent; + }; + + // Emit session_end once and close the stream. Called after all prompts + // (and any fallback retries) have settled. + let runFinalized = false; + const finalizeRun = () => { + if (runFinalized) return; + runFinalized = true; + pushMsg({ + type: "system", + subtype: "session_end", + content: `Agent ${loaded.manifest.name} finished`, + metadata: { sessionId: _sessionId }, + }); + channel.finish(); + }; + + // One persistent agent for the whole run so multi-turn conversation state + // survives across prompts. Fallback swaps the model on this same agent + // (and restores the pre-attempt transcript) rather than rebuilding it. + const agent = buildAgent(candidateModels[0]); + let activeModelIndex = 0; + + // Run one prompt, falling back through candidate models when the current + // model fails with a retryable provider error before producing any output. + const runPrompt = async (promptText: string): Promise => { + for (let i = activeModelIndex; i < candidateModels.length; i++) { + attempt = { producedOutput: false, error: null }; + if (i !== activeModelIndex) { + agent.state.model = candidateModels[i]; + activeModelIndex = i; + } + // Snapshot transcript so a failed attempt can be rolled back cleanly. + const snapshot = [...agent.state.messages]; + await otelContext.with(_session.ctx, () => agent.prompt(promptText)); + + const err = attempt.error; + if (!err) return; // success — stay on this model for later turns + + const hasNext = i < candidateModels.length - 1; + const canRetry = + hasNext && !attempt.producedOutput && isRetryableProviderError(err.message); + + if (canRetry) { + const next = candidateModels[i + 1]; + // Drop the failed user+assistant turn so the retry starts clean. + agent.state.messages = snapshot; pushMsg({ type: "system", - subtype: "session_end", - content: `Agent ${loaded.manifest.name} finished`, - metadata: { sessionId: _sessionId }, + subtype: "fallback", + content: + `Model ${err.provider ?? ""}:${err.model ?? ""} failed (${err.message}). ` + + `Falling back to ${next.provider}:${next.id}.`, + metadata: { + failedModel: `${err.provider ?? ""}:${err.model ?? ""}`, + nextModel: `${next.provider}:${next.id}`, + }, }); - channel.finish(); - break; + continue; + } + + // Terminal failure — surface the error to the caller. Emit the + // system error plus an errored assistant message so callers that + // inspect messages() for stopReason still see it (original contract). + pushMsg({ + type: "system", + subtype: "error", + content: err.message || "LLM request failed (unknown error)", + metadata: { model: err.model, provider: err.provider, api: err.api }, + }); + pushMsg({ + type: "assistant", + content: "", + model: err.model ?? "unknown", + provider: err.provider ?? "unknown", + stopReason: "error", + errorMessage: err.message, + }); + return; } - }); + }; // 10. Send prompt — run inside the session span's context so that // gen_ai.chat and gitagent.tool.execute spans become children of @@ -507,9 +629,7 @@ export function query(options: QueryOptions): Query { return; } } - await otelContext.with(_session.ctx, () => - agent.prompt(options.prompt as string), - ); + await runPrompt(options.prompt as string); } else { // Multi-turn: iterate the async iterable for await (const userMsg of options.prompt) { @@ -531,12 +651,13 @@ export function query(options: QueryOptions): Query { return; } } - await otelContext.with(_session.ctx, () => - agent.prompt(userMsg.content), - ); + await runPrompt(userMsg.content); } } + // Emit session_end and close the stream (once). + finalizeRun(); + // Finalize local session if active if (localSession) { try { localSession.finalize(); } catch { /* best-effort */ } diff --git a/test/model-fallback.test.ts b/test/model-fallback.test.ts new file mode 100644 index 0000000..51c712e --- /dev/null +++ b/test/model-fallback.test.ts @@ -0,0 +1,113 @@ +import { describe, it, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let isRetryableProviderError: typeof import("../dist/model-fallback.js").isRetryableProviderError; +let loadAgent: typeof import("../dist/exports.js").loadAgent; + +before(async () => { + ({ isRetryableProviderError } = await import("../dist/model-fallback.js")); + ({ loadAgent } = await import("../dist/exports.js")); +}); + +describe("isRetryableProviderError", () => { + it("retries on billing / credit / quota failures", () => { + for (const m of [ + "Your credit balance is too low to access the Claude API", + "insufficient_quota: You exceeded your current quota", + "billing hard limit reached", + "429 Too Many Requests", + "rate limit exceeded", + ]) { + assert.equal(isRetryableProviderError(m), true, m); + } + }); + + it("retries on availability / auth failures", () => { + for (const m of [ + "Overloaded", + "503 Service Unavailable", + "internal server error", + "401 Unauthorized: invalid api key", + "ETIMEDOUT", + ]) { + assert.equal(isRetryableProviderError(m), true, m); + } + }); + + it("retries on empty/missing errors (give the next model a chance)", () => { + assert.equal(isRetryableProviderError(undefined), true); + assert.equal(isRetryableProviderError(null), true); + assert.equal(isRetryableProviderError(""), true); + }); + + it("does NOT retry on request-shaped / unrecognized errors", () => { + // A malformed-request error won't be fixed by switching models. + assert.equal(isRetryableProviderError("max_tokens: must be less than 4096"), false); + assert.equal(isRetryableProviderError("content was blocked by safety filter"), false); + assert.equal(isRetryableProviderError("something exploded"), false); + }); +}); + +describe("loadAgent fallback resolution", () => { + let dir: string; + + before(async () => { + dir = await mkdtemp(join(tmpdir(), "gitagent-fb-")); + // Use custom-endpoint models (provider:id@url) so resolution never hits + // the pi-ai registry and stays deterministic offline. + const yaml = [ + "spec_version: '1.0'", + "name: fb-test", + "version: '1.0.0'", + "description: fallback test agent", + "model:", + " preferred: 'openai:gpt-4o-mini@https://example.com/v1'", + " fallback:", + " - 'anthropic:claude-3-5-sonnet@https://example.com/v1'", + " - ' anthropic:claude-3-5-sonnet@https://example.com/v1 '", // whitespace dup → skipped + " - 'openai:gpt-4o-mini@https://example.com/v1'", // dup of preferred → skipped + " - ''", // empty → skipped + "tools: []", + "runtime:", + " max_turns: 10", + "", + ].join("\n"); + await writeFile(join(dir, "agent.yaml"), yaml, "utf-8"); + }); + + after(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it("resolves manifest fallback models, trimming + de-duping and skipping blanks", async () => { + const loaded = await loadAgent(dir); + assert.equal(loaded.model.id, "gpt-4o-mini"); + // Only one anthropic entry survives: the whitespace variant is a dup, the + // openai entry matches preferred, and the empty string is skipped. + assert.equal(loaded.fallbackModels.length, 1); + assert.equal(loaded.fallbackModels[0].id, "claude-3-5-sonnet"); + }); + + it("returns an empty fallback list when none are configured", async () => { + const d2 = await mkdtemp(join(tmpdir(), "gitagent-fb2-")); + const yaml = [ + "spec_version: '1.0'", + "name: no-fb", + "version: '1.0.0'", + "description: no fallback", + "model:", + " preferred: 'openai:gpt-4o-mini@https://example.com/v1'", + "tools: []", + "runtime:", + " max_turns: 10", + "", + ].join("\n"); + await writeFile(join(d2, "agent.yaml"), yaml, "utf-8"); + const loaded = await loadAgent(d2); + assert.deepEqual(loaded.fallbackModels, []); + await rm(d2, { recursive: true, force: true }); + }); +});