diff --git a/packages/app/src/components/dialog-select-provider.tsx b/packages/app/src/components/dialog-select-provider.tsx index 89310286daf2..2b0fa888e4e0 100644 --- a/packages/app/src/components/dialog-select-provider.tsx +++ b/packages/app/src/components/dialog-select-provider.tsx @@ -24,6 +24,7 @@ export const DialogSelectProvider: Component = () => { if (id === "openai") return language.t("dialog.provider.openai.note") if (id.startsWith("github-copilot")) return language.t("dialog.provider.copilot.note") if (id === "opencode-go") return language.t("dialog.provider.opencodeGo.tagline") + if (id === "litellm") return language.t("dialog.provider.litellm.note") } return ( diff --git a/packages/app/src/components/settings-providers.tsx b/packages/app/src/components/settings-providers.tsx index ef8d03aedd0a..e8e62e31679d 100644 --- a/packages/app/src/components/settings-providers.tsx +++ b/packages/app/src/components/settings-providers.tsx @@ -26,6 +26,7 @@ const PROVIDER_NOTES = [ { match: (id: string) => id === "google", key: "dialog.provider.google.note" }, { match: (id: string) => id === "openrouter", key: "dialog.provider.openrouter.note" }, { match: (id: string) => id === "vercel", key: "dialog.provider.vercel.note" }, + { match: (id: string) => id === "litellm", key: "dialog.provider.litellm.note" }, ] as const export const SettingsProviders: Component = () => { diff --git a/packages/app/src/i18n/en.ts b/packages/app/src/i18n/en.ts index 76c96dcc02af..d509866f485d 100644 --- a/packages/app/src/i18n/en.ts +++ b/packages/app/src/i18n/en.ts @@ -110,6 +110,7 @@ export const dict = { "dialog.provider.google.note": "Gemini models for fast, structured responses", "dialog.provider.openrouter.note": "Access all supported models from one provider", "dialog.provider.vercel.note": "Unified access to AI models with smart routing", + "dialog.provider.litellm.note": "Unified proxy for 100+ LLMs with load balancing and spend tracking", "dialog.model.select.title": "Select model", "dialog.model.search.placeholder": "Search models", diff --git a/packages/core/src/provider.ts b/packages/core/src/provider.ts index 3f99cedc1cfb..7d95da01cb86 100644 --- a/packages/core/src/provider.ts +++ b/packages/core/src/provider.ts @@ -18,6 +18,7 @@ export const ID = Schema.String.pipe( openrouter: schema.make("openrouter"), mistral: schema.make("mistral"), gitlab: schema.make("gitlab"), + litellm: schema.make("litellm"), })), ) export type ID = typeof ID.Type diff --git a/packages/opencode/src/plugin/index.ts b/packages/opencode/src/plugin/index.ts index 478114209207..ec1b73259235 100644 --- a/packages/opencode/src/plugin/index.ts +++ b/packages/opencode/src/plugin/index.ts @@ -13,6 +13,7 @@ import { CodexAuthPlugin } from "./openai/codex" import { Session } from "@/session/session" import { NamedError } from "@opencode-ai/core/util/error" import { CopilotAuthPlugin } from "./github-copilot/copilot" +import { LiteLLMPlugin } from "./litellm/litellm" import { gitlabAuthPlugin as GitlabAuthPlugin } from "opencode-gitlab-auth" import { PoeAuthPlugin } from "opencode-poe-auth" import { CloudflareAIGatewayAuthPlugin, CloudflareWorkersAuthPlugin } from "./cloudflare" @@ -71,6 +72,7 @@ function internalPlugins(flags: RuntimeFlags.Info): PluginInstance[] { experimentalWebSockets: experimentalWebSocketsEnabled({ enabled: flags.experimentalWebSockets }), }), CopilotAuthPlugin, + LiteLLMPlugin, GitlabAuthPlugin, PoeAuthPlugin, CloudflareWorkersAuthPlugin, diff --git a/packages/opencode/src/plugin/litellm/litellm.ts b/packages/opencode/src/plugin/litellm/litellm.ts new file mode 100644 index 000000000000..56af13023ea9 --- /dev/null +++ b/packages/opencode/src/plugin/litellm/litellm.ts @@ -0,0 +1,66 @@ +import type { Hooks, PluginInput } from "@opencode-ai/plugin" +import * as Log from "@opencode-ai/core/util/log" +import { LiteLLMModels } from "./models" + +const log = Log.create({ service: "plugin.litellm" }) + +export async function LiteLLMPlugin(input: PluginInput): Promise { + return { + provider: { + id: "litellm", + async models(provider, ctx) { + const baseURL = (() => { + if (provider.options?.baseURL) return provider.options.baseURL as string + if (process.env["LITELLM_BASE_URL"]) return process.env["LITELLM_BASE_URL"] + return undefined + })() + + if (!baseURL) { + return provider.models + } + + const headers: Record = {} + if (ctx.auth?.type === "api" && ctx.auth.key) { + headers["Authorization"] = `Bearer ${ctx.auth.key}` + } else if (process.env["LITELLM_API_KEY"]) { + headers["Authorization"] = `Bearer ${process.env["LITELLM_API_KEY"]}` + } + + return LiteLLMModels.get(baseURL, headers, provider.models).catch((error) => { + log.error("failed to fetch litellm models", { error }) + return provider.models + }) + }, + }, + auth: { + provider: "litellm", + async loader(getAuth) { + const auth = await getAuth() + if (auth.type !== "api") { + // Fall back to env var + const envKey = process.env["LITELLM_API_KEY"] + if (envKey) return { apiKey: envKey } + return {} + } + + return { + apiKey: auth.key, + } + }, + methods: [ + { + label: "Enter LiteLLM API Key", + type: "api", + prompts: [ + { + type: "text", + key: "base_url", + message: "LiteLLM proxy base URL (e.g. http://localhost:4000)", + placeholder: "http://localhost:4000", + }, + ], + }, + ], + }, + } +} diff --git a/packages/opencode/src/plugin/litellm/models.ts b/packages/opencode/src/plugin/litellm/models.ts new file mode 100644 index 000000000000..24606c37bc7d --- /dev/null +++ b/packages/opencode/src/plugin/litellm/models.ts @@ -0,0 +1,133 @@ +import type { Model } from "@opencode-ai/sdk/v2" +import { InstallationVersion } from "@opencode-ai/core/installation/version" +import * as Log from "@opencode-ai/core/util/log" +import os from "os" + +const log = Log.create({ service: "plugin.litellm.models" }) + +interface LiteLLMModelEntry { + id: string + created?: number + owned_by?: string +} + +interface LiteLLMModelInfo { + max_tokens?: number + max_input_tokens?: number + max_output_tokens?: number + input_cost_per_token?: number + output_cost_per_token?: number + supports_vision?: boolean + supports_function_calling?: boolean + mode?: string +} + +async function fetchModelInfo( + baseURL: string, + headers: Record, +): Promise> { + try { + const url = `${baseURL.replace(/\/+$/, "")}/model/info` + const res = await fetch(url, { + headers, + signal: AbortSignal.timeout(5_000), + }) + if (!res.ok) return {} + const body = (await res.json()) as { data?: Array<{ model_name: string; model_info?: LiteLLMModelInfo }> } + if (!Array.isArray(body.data)) return {} + const info: Record = {} + for (const entry of body.data) { + if (entry.model_name && entry.model_info) { + info[entry.model_name] = entry.model_info + } + } + return info + } catch { + return {} + } +} + +function buildModel(id: string, baseURL: string, info?: LiteLLMModelInfo, prev?: Model): Model { + const maxContext = info?.max_tokens ?? info?.max_input_tokens ?? 128_000 + const maxOutput = info?.max_output_tokens ?? 16_384 + + const inputCost = info?.input_cost_per_token ?? 0 + const outputCost = info?.output_cost_per_token ?? 0 + + return { + id: id, + providerID: "litellm", + name: prev?.name ?? id, + api: { + id, + url: baseURL, + npm: "@ai-sdk/openai-compatible", + }, + status: "active", + headers: prev?.headers ?? {}, + options: prev?.options ?? {}, + cost: { + input: inputCost * 1_000_000, + output: outputCost * 1_000_000, + cache: { read: 0, write: 0 }, + }, + limit: { + context: maxContext, + output: maxOutput, + }, + capabilities: { + temperature: true, + reasoning: prev?.capabilities?.reasoning ?? false, + attachment: true, + toolcall: info?.supports_function_calling ?? prev?.capabilities?.toolcall ?? true, + input: { + text: true, + audio: false, + image: info?.supports_vision ?? prev?.capabilities?.input?.image ?? true, + video: false, + pdf: false, + }, + output: { text: true, audio: false, image: false, video: false, pdf: false }, + interleaved: false, + }, + release_date: prev?.release_date ?? "", + variants: prev?.variants ?? {}, + } +} + +export async function get( + baseURL: string, + headers: Record = {}, + existing: Record = {}, +): Promise> { + const modelsURL = `${baseURL.replace(/\/+$/, "")}/v1/models` + const reqHeaders: Record = { + "User-Agent": `opencode/${InstallationVersion} litellm (${os.platform()} ${os.release()}; ${os.arch()})`, + ...headers, + } + + const res = await fetch(modelsURL, { + headers: reqHeaders, + signal: AbortSignal.timeout(5_000), + }) + if (!res.ok) throw new Error(`Failed to fetch models: ${res.status}`) + + const body = (await res.json()) as { data?: LiteLLMModelEntry[] } + if (!Array.isArray(body.data)) return existing + + // Enrich with per-model metadata from /model/info + const modelInfo = await fetchModelInfo(baseURL, reqHeaders) + + const result = { ...existing } + + for (const entry of body.data) { + const id = entry.id + if (!id) continue + const info = modelInfo[id] + result[id] = buildModel(id, baseURL, info, existing[id]) + } + + return result +} + +export * as LiteLLMModels from "./models" diff --git a/packages/opencode/src/provider/provider.ts b/packages/opencode/src/provider/provider.ts index 2c50a9a60dc7..f4a70639b538 100644 --- a/packages/opencode/src/provider/provider.ts +++ b/packages/opencode/src/provider/provider.ts @@ -1127,7 +1127,7 @@ export function fromModelsDevProvider(provider: ModelsDev.Provider): Info { const base = fromModelsDevModel(provider, model) models[id] = { ...base, - id: ProviderV2.ModelID.make(id), + id: ProviderV2.ProviderV2.ModelID.make(id), name: `${model.name} ${mode[0].toUpperCase()}${mode.slice(1)}`, cost: opts.cost ? mergeDeep(base.cost, cost(opts.cost)) : base.cost, options: opts.provider?.body @@ -1257,6 +1257,17 @@ export const layer = Layer.effect( const providerID = ProviderV2.ID.make(p.id) if (disabled.has(providerID)) continue + // Seed provider entry if the plugin declares one but it's not in models.dev or config + if (!database[providerID]) { + database[providerID] = { + id: providerID, + name: providerID, + source: "env", + env: [], + options: {}, + models: {}, + } + } const provider = database[providerID] if (!provider) continue const pluginAuth = yield* auth.get(providerID).pipe(Effect.orDie) @@ -1268,12 +1279,17 @@ export const layer = Layer.effect( id, { ...model, - id: ProviderV2.ModelID.make(id), + id: ProviderV2.ProviderV2.ModelID.make(id), providerID, }, ]), ) }) + + // Ensure plugin-seeded providers appear in the state + if (!providers[providerID]) { + mergeProvider(providerID, { source: "env" }) + } } // extend database from config diff --git a/packages/ui/src/components/provider-icons/sprite.svg b/packages/ui/src/components/provider-icons/sprite.svg index 68b99ce56d4a..cc948dc66b40 100644 --- a/packages/ui/src/components/provider-icons/sprite.svg +++ b/packages/ui/src/components/provider-icons/sprite.svg @@ -1131,5 +1131,8 @@ d="M 554.228 729.711 C 659.104 725.931 747.227 807.802 751.164 912.672 C 755.1 1017.54 673.362 1105.79 568.498 1109.88 C 463.411 1113.98 374.937 1032.03 370.992 926.942 C 367.047 821.849 449.129 733.498 554.228 729.711 z" > + + 🚅 + diff --git a/packages/ui/src/components/provider-icons/types.ts b/packages/ui/src/components/provider-icons/types.ts index 1c6f5fe6d2f0..181da3b1a622 100644 --- a/packages/ui/src/components/provider-icons/types.ts +++ b/packages/ui/src/components/provider-icons/types.ts @@ -33,6 +33,7 @@ export const iconNames = [ "ovhcloud", "openrouter", "llmgateway", + "litellm", "opencode", "opencode-go", "openai",