From 902d63c373cf9557e7a935fc4e65b7db36fc3616 Mon Sep 17 00:00:00 2001 From: daewoongoh Date: Wed, 15 Jul 2026 19:42:52 +0900 Subject: [PATCH 1/2] fix(litellm): preserve reasoning_content for known reasoning model families LiteLLM's /v1/model/info never reports reasoning capability flags, so infer preserveReasoning from the model alias/routed-model name and use convertToR1Format to keep reasoning_content intact across tool-call turns. Signed-off-by: daewoongoh --- packages/types/src/providers/lite-llm.ts | 31 ++++++++++++++++++++++++ src/api/providers/fetchers/litellm.ts | 6 +++++ src/api/providers/lite-llm.ts | 17 ++++++++++--- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/packages/types/src/providers/lite-llm.ts b/packages/types/src/providers/lite-llm.ts index 14a68cfc3c..535ef77558 100644 --- a/packages/types/src/providers/lite-llm.ts +++ b/packages/types/src/providers/lite-llm.ts @@ -13,3 +13,34 @@ export const litellmDefaultModelInfo: ModelInfo = { cacheWritesPrice: 3.75, cacheReadsPrice: 0.3, } + +/** + * LiteLLM is a gateway: it fronts arbitrary underlying models and its + * `/v1/model/info` response carries no reasoning-related capability flags + * (no `preserveReasoning` equivalent). The underlying model identity is only + * visible as text in the model alias (`model_name`) or the routed target + * (`litellm_params.model`, e.g. `deepseek/deepseek-reasoner`, + * `bedrock/moonshot.kimi-k2-thinking`, `fireworks_ai/.../kimi-k2p7-code`). + * + * Each fragment below matches a model-family substring that requires + * `preserveReasoning: true` in its native provider config (see deepseek.ts, + * mimo.ts, moonshot.ts, bedrock.ts, fireworks.ts, zai.ts, minimax.ts, + * opencode-go.ts), so the same behavior can be inferred for a LiteLLM-routed + * alias of the same underlying model. This is best-effort: unrecognized + * aliases or renamed deployments will not match, and callers should treat + * it as a heuristic, not a source of truth. + */ +const LITELLM_PRESERVE_REASONING_FRAGMENTS = [ + String.raw`deepseek-v4-(flash|pro)`, // deepseek.ts + String.raw`deepseek-reasoner`, // deepseek.ts (legacy alias) + String.raw`mimo-v2\.5(-pro)?`, // mimo.ts + String.raw`kimi-k2-thinking`, // moonshot.ts, bedrock.ts, fireworks.ts + String.raw`kimi-k2p7-code`, // fireworks.ts (dash-separated id) + String.raw`kimi-k2\.7-code`, // fireworks.ts (dotted alias variant) + String.raw`minimax[.-]?m[23](\.\d+)?(-highspeed|-stable)?\b`, // bedrock.ts, minimax.ts, opencode-go.ts + String.raw`glm-4\.7(?!-flash)\b`, // zai.ts (excludes glm-4.7-flash(x)) + String.raw`glm-5(\.[12])?(-turbo)?(?!-flash)\b`, // zai.ts, opencode-go.ts + String.raw`qwen3\.[67]-(plus|max)`, // opencode-go.ts +] + +export const LITELLM_PRESERVE_REASONING_PATTERN = new RegExp(LITELLM_PRESERVE_REASONING_FRAGMENTS.join("|"), "i") diff --git a/src/api/providers/fetchers/litellm.ts b/src/api/providers/fetchers/litellm.ts index de895d01fb..896084a923 100644 --- a/src/api/providers/fetchers/litellm.ts +++ b/src/api/providers/fetchers/litellm.ts @@ -1,6 +1,7 @@ import axios from "axios" import type { ModelRecord } from "@roo-code/types" +import { LITELLM_PRESERVE_REASONING_PATTERN } from "@roo-code/types" import { DEFAULT_HEADERS } from "../constants" /** @@ -40,6 +41,10 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise if (!modelName || !modelInfo || !litellmModelName) continue + // LiteLLM's /v1/model/info never reports reasoning capability flags, so infer + // preserveReasoning from the alias/routed-model name against known model families. + const preservesReasoning = LITELLM_PRESERVE_REASONING_PATTERN.test(`${modelName} ${litellmModelName}`) + models[modelName] = { maxTokens: modelInfo.max_output_tokens || modelInfo.max_tokens || 8192, contextWindow: modelInfo.max_input_tokens || 200000, @@ -55,6 +60,7 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise cacheReadsPrice: modelInfo.cache_read_input_token_cost ? modelInfo.cache_read_input_token_cost * 1000000 : undefined, + ...(preservesReasoning && { preserveReasoning: true }), description: `${modelName} via LiteLLM proxy`, } } diff --git a/src/api/providers/lite-llm.ts b/src/api/providers/lite-llm.ts index 3f9b3732e5..7ca8497b39 100644 --- a/src/api/providers/lite-llm.ts +++ b/src/api/providers/lite-llm.ts @@ -9,6 +9,7 @@ import { ApiHandlerOptions } from "../../shared/api" import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" +import { convertToR1Format } from "../transform/r1-format" import { GEMINI_THOUGHT_SIGNATURE_BYPASS } from "../transform/gemini-format" import { sanitizeOpenAiCallId } from "../../utils/tool-id" @@ -117,9 +118,19 @@ export class LiteLLMHandler extends RouterProvider implements SingleCompletionHa ): ApiStream { const { id: modelId, info } = await this.fetchModel() - const openAiMessages = convertToOpenAiMessages(messages, { - normalizeToolCallId: sanitizeOpenAiCallId, - }) + // Models that require reasoning_content to be echoed back during tool-call + // continuations (see LITELLM_PRESERVE_REASONING_PATTERN) need convertToR1Format: + // it merges consecutive same-role messages and, via mergeToolResultText, folds + // text following tool_results into the last tool message so a user message + // never gets inserted mid-turn and causes the model to drop prior reasoning_content. + const openAiMessages = info.preserveReasoning + ? convertToR1Format(messages, { + normalizeToolCallId: sanitizeOpenAiCallId, + mergeToolResultText: true, + }) + : convertToOpenAiMessages(messages, { + normalizeToolCallId: sanitizeOpenAiCallId, + }) // Prepare messages with cache control if enabled and supported let systemMessage: OpenAI.Chat.ChatCompletionMessageParam From ce38822745b69d004d5cbc84c7138299c88e2dac Mon Sep 17 00:00:00 2001 From: daewoongoh Date: Wed, 15 Jul 2026 19:43:10 +0900 Subject: [PATCH 2/2] test(litellm): cover preserveReasoning inference and message conversion branching Add unit coverage for LITELLM_PRESERVE_REASONING_PATTERN matching per model family, getLiteLLMModels setting preserveReasoning on matched models, and LiteLLMHandler.createMessage branching between convertToR1Format and convertToOpenAiMessages based on info.preserveReasoning. Signed-off-by: daewoongoh --- packages/types/src/__tests__/lite-llm.test.ts | 70 +++++++++ src/api/providers/__tests__/lite-llm.spec.ts | 139 ++++++++++++++++++ .../fetchers/__tests__/litellm.spec.ts | 88 +++++++++++ 3 files changed, 297 insertions(+) create mode 100644 packages/types/src/__tests__/lite-llm.test.ts diff --git a/packages/types/src/__tests__/lite-llm.test.ts b/packages/types/src/__tests__/lite-llm.test.ts new file mode 100644 index 0000000000..2df8a4a6ae --- /dev/null +++ b/packages/types/src/__tests__/lite-llm.test.ts @@ -0,0 +1,70 @@ +import { LITELLM_PRESERVE_REASONING_PATTERN } from "../providers/lite-llm.js" + +describe("LITELLM_PRESERVE_REASONING_PATTERN", () => { + it("matches known DeepSeek reasoning aliases", () => { + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("deepseek-v4-flash")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("deepseek-v4-pro")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("deepseek/deepseek-reasoner")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("deepseek-v4-mini")).toBe(false) + }) + + it("matches known MiMo reasoning aliases", () => { + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("mimo-v2.5")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("mimo-v2.5-pro")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("mimo-v2.6")).toBe(false) + }) + + it("matches known Kimi K2 reasoning aliases across routed providers", () => { + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("bedrock/moonshot.kimi-k2-thinking")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("fireworks_ai/accounts/fireworks/models/kimi-k2p7-code")).toBe( + true, + ) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("kimi-k2.7-code")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("kimi-k2.6")).toBe(false) + }) + + it("matches MiniMax M2/M3 aliases but not other MiniMax generations", () => { + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("minimax-m2")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("minimax.m3")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("minimax-m2.5")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("minimax-m2-highspeed")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("minimax-m2-stable")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("minimax-m4")).toBe(false) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("minimax-m1")).toBe(false) + }) + + it("matches GLM-4.7 but excludes the flash variants", () => { + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("glm-4.7")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("glm-4.7-flash")).toBe(false) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("glm-4.7-flashx")).toBe(false) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("glm-4.8")).toBe(false) + }) + + it("matches GLM-5 variants but excludes the flash variant", () => { + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("glm-5")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("glm-5.1")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("glm-5.2")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("glm-5-turbo")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("glm-5.1-turbo")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("glm-5-flash")).toBe(false) + }) + + it("matches curated Qwen3 plus/max aliases used by opencode-go", () => { + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("qwen3.7-plus")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("qwen3.6-max")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("qwen3.5-plus")).toBe(false) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("qwen3.7-mini")).toBe(false) + }) + + it("does not match unrelated model names", () => { + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("gpt-4")).toBe(false) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("claude-3-opus")).toBe(false) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("")).toBe(false) + }) + + it("matches when the fragment appears anywhere in a combined alias/routed-model string", () => { + // Mirrors how litellm.ts calls it: `${modelName} ${litellmModelName}` + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("my-deepseek-alias deepseek/deepseek-reasoner")).toBe(true) + expect(LITELLM_PRESERVE_REASONING_PATTERN.test("my-gpt4-alias openai/gpt-4")).toBe(false) + }) +}) diff --git a/src/api/providers/__tests__/lite-llm.spec.ts b/src/api/providers/__tests__/lite-llm.spec.ts index ab2f261058..a5898513c5 100644 --- a/src/api/providers/__tests__/lite-llm.spec.ts +++ b/src/api/providers/__tests__/lite-llm.spec.ts @@ -1124,6 +1124,145 @@ describe("LiteLLMHandler", () => { }) }) + describe("preserveReasoning message conversion", () => { + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { + choices: [{ delta: { content: "ok" } }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + } + }, + } + + it("uses convertToR1Format (merging tool-result text) when the model info sets preserveReasoning", async () => { + const optionsWithReasoning: ApiHandlerOptions = { + ...mockOptions, + litellmModelId: "deepseek-reasoner-alias", + } + handler = new LiteLLMHandler(optionsWithReasoning) + + vi.spyOn(handler as any, "fetchModel").mockResolvedValue({ + id: "deepseek-reasoner-alias", + info: { ...litellmDefaultModelInfo, preserveReasoning: true }, + }) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [ + { role: "user", content: "Hello" }, + { + role: "assistant", + content: [ + { type: "text", text: "I'll help." }, + { type: "tool_use", id: "toolu_123", name: "read_file", input: { path: "test.txt" } }, + ], + // DeepSeek-style interleaved thinking: must be echoed back in the next request. + reasoning_content: "Let me check the file first.", + } as Anthropic.Messages.MessageParam & { reasoning_content: string }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "toolu_123", content: "file contents" }, + { type: "text", text: "Thanks, continue." }, + ], + }, + ] + + mockCreate.mockReturnValue({ + withResponse: vi.fn().mockResolvedValue({ data: mockStream }), + }) + + const generator = handler.createMessage(systemPrompt, messages) + for await (const _chunk of generator) { + // Consume + } + + const createCall = mockCreate.mock.calls[0][0] + + // convertToR1Format with mergeToolResultText folds the trailing text into the + // tool message instead of appending a separate user message. + const toolMessage = createCall.messages.find((msg: any) => msg.role === "tool") + expect(toolMessage).toBeDefined() + expect(toolMessage.content).toBe("file contents\n\nThanks, continue.") + + const trailingUserMessage = createCall.messages.find( + (msg: any) => msg.role === "user" && msg.content === "Thanks, continue.", + ) + expect(trailingUserMessage).toBeUndefined() + + // The whole point of routing through convertToR1Format: reasoning_content + // must survive on the assistant message so the model doesn't reject the + // follow-up request for missing prior reasoning. + const assistantMessage = createCall.messages.find( + (msg: any) => msg.role === "assistant" && msg.tool_calls?.length > 0, + ) + expect(assistantMessage).toBeDefined() + expect(assistantMessage.reasoning_content).toBe("Let me check the file first.") + }) + + it("uses convertToOpenAiMessages (no merging) when the model info does not set preserveReasoning", async () => { + vi.spyOn(handler as any, "fetchModel").mockResolvedValue({ + id: litellmDefaultModelId, + info: { ...litellmDefaultModelInfo, preserveReasoning: undefined }, + }) + + const systemPrompt = "You are a helpful assistant" + // Task.buildCleanConversationHistory() (src/core/task/Task.ts) already strips the + // reasoning content block before messages reach this handler when the model's + // preserveReasoning is not true, so no reasoning_content/reasoning block is present + // on the assistant message here — this input reflects what the handler actually + // receives in that case. + const messages: Anthropic.Messages.MessageParam[] = [ + { role: "user", content: "Hello" }, + { + role: "assistant", + content: [ + { type: "text", text: "I'll help." }, + { type: "tool_use", id: "toolu_123", name: "read_file", input: { path: "test.txt" } }, + ], + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "toolu_123", content: "file contents" }, + { type: "text", text: "Thanks, continue." }, + ], + }, + ] + + mockCreate.mockReturnValue({ + withResponse: vi.fn().mockResolvedValue({ data: mockStream }), + }) + + const generator = handler.createMessage(systemPrompt, messages) + for await (const _chunk of generator) { + // Consume + } + + const createCall = mockCreate.mock.calls[0][0] + + const toolMessage = createCall.messages.find((msg: any) => msg.role === "tool") + expect(toolMessage).toBeDefined() + expect(toolMessage.content).toBe("file contents") + + const trailingUserMessage = createCall.messages.find( + (msg: any) => + msg.role === "user" && + (msg.content === "Thanks, continue." || + (Array.isArray(msg.content) && + msg.content.some((part: any) => part.text === "Thanks, continue."))), + ) + expect(trailingUserMessage).toBeDefined() + + // No reasoning_content is sent to the API in this branch, matching what + // buildCleanConversationHistory already stripped upstream. + const assistantMessage = createCall.messages.find( + (msg: any) => msg.role === "assistant" && msg.tool_calls?.length > 0, + ) + expect(assistantMessage).toBeDefined() + expect(assistantMessage.reasoning_content).toBeUndefined() + }) + }) + describe("session ID header", () => { const mockStream = { async *[Symbol.asyncIterator]() { diff --git a/src/api/providers/fetchers/__tests__/litellm.spec.ts b/src/api/providers/fetchers/__tests__/litellm.spec.ts index c05cda8839..7b3d44c578 100644 --- a/src/api/providers/fetchers/__tests__/litellm.spec.ts +++ b/src/api/providers/fetchers/__tests__/litellm.spec.ts @@ -697,4 +697,92 @@ describe("getLiteLLMModels", () => { description: "model-with-only-max-output-tokens via LiteLLM proxy", }) }) + + describe("preserveReasoning inference", () => { + it("sets preserveReasoning: true when the routed model matches a known reasoning family", async () => { + const mockResponse = { + data: { + data: [ + { + model_name: "deepseek-reasoner-alias", + model_info: { + max_tokens: 8192, + max_input_tokens: 128000, + }, + litellm_params: { + model: "deepseek/deepseek-reasoner", + }, + }, + { + model_name: "kimi-k2-thinking", + model_info: { + max_tokens: 8192, + max_input_tokens: 128000, + }, + litellm_params: { + model: "bedrock/moonshot.kimi-k2-thinking", + }, + }, + ], + }, + } + + mockedAxios.get.mockResolvedValue(mockResponse) + + const result = await getLiteLLMModels("test-api-key", "http://localhost:4000") + + expect(result["deepseek-reasoner-alias"]).toMatchObject({ preserveReasoning: true }) + expect(result["kimi-k2-thinking"]).toMatchObject({ preserveReasoning: true }) + }) + + it("omits preserveReasoning when the routed model does not match a known reasoning family", async () => { + const mockResponse = { + data: { + data: [ + { + model_name: "gpt-4-turbo", + model_info: { + max_tokens: 8192, + max_input_tokens: 128000, + }, + litellm_params: { + model: "openai/gpt-4-turbo", + }, + }, + ], + }, + } + + mockedAxios.get.mockResolvedValue(mockResponse) + + const result = await getLiteLLMModels("test-api-key", "http://localhost:4000") + + expect(result["gpt-4-turbo"]).not.toHaveProperty("preserveReasoning") + }) + + it("matches against the model alias even when the routed model name does not match", async () => { + const mockResponse = { + data: { + data: [ + { + model_name: "glm-5.1-turbo", + model_info: { + max_tokens: 8192, + max_input_tokens: 128000, + }, + litellm_params: { + model: "zai/some-custom-deployment", + }, + }, + ], + }, + } + + mockedAxios.get.mockResolvedValue(mockResponse) + + const result = await getLiteLLMModels("test-api-key", "http://localhost:4000") + + expect(result["glm-5.1-turbo"]).toMatchObject({ preserveReasoning: true }) + }) + }) })