Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions core/config/load.ts
Original file line number Diff line number Diff line change
Expand Up @@ -464,7 +464,7 @@
}
if (name === "llm") {
const llm = models.find((model) => model.title === params?.modelTitle);
if (!llm) {

Check warning on line 467 in core/config/load.ts

View workflow job for this annotation

GitHub Actions / core-checks

Unexpected negated condition
errors.push({
fatal: false,
message: `Unknown reranking model ${params?.modelTitle}`,
Expand Down Expand Up @@ -560,7 +560,7 @@
id: `continue-mcp-server-${index + 1}`,
name: `MCP Server`,
requestOptions: mergeConfigYamlRequestOptions(
server.transport.type !== "stdio"

Check warning on line 563 in core/config/load.ts

View workflow job for this annotation

GitHub Actions / core-checks

Unexpected negated condition
? server.transport.requestOptions
: undefined,
config.requestOptions,
Expand Down Expand Up @@ -652,6 +652,8 @@
sourceFile: llm.sourceFile,
isFromAutoDetect: llm.isFromAutoDetect,
toolOverrides: llm.toolOverrides,
customReasoningFields:
llm.customReasoningFields ?? (llm as any).options?.customReasoningFields,
};
}

Expand Down
5 changes: 5 additions & 0 deletions core/config/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -572,6 +572,9 @@ declare global {

// IBM watsonx Options
deploymentId?: string;

/** Custom fields to check for reasoning/thinking content in streaming chunks */
customReasoningFields?: string[];
}

type RequireAtLeastOne<T, Keys extends keyof T = keyof T> = Pick<
Expand Down Expand Up @@ -960,6 +963,8 @@ declare global {
promptTemplates?: { [key: string]: string };
capabilities?: ModelCapability;
cacheBehavior?: CacheBehavior;
/** Custom fields to check for reasoning/thinking content in streaming chunks */
customReasoningFields?: string[];
}

export interface JSONEmbedOptions {
Expand Down
1 change: 1 addition & 0 deletions core/config/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ export function addModel(
contextLength: model.contextLength,
maxStopWords: model.maxStopWords,
defaultCompletionOptions: model.completionOptions,
customReasoningFields: model.customReasoningFields,
...(capabilities.length > 0 ? { capabilities } : {}),
};
config.models.push(desc);
Expand Down
21 changes: 21 additions & 0 deletions core/config/yaml/models.vitest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,27 @@ describe("llmsFromModelConfig requestOptions merging", () => {
expect(llm.requestOptions).toEqual(model.requestOptions);
});

it("should preserve custom reasoning fields from model config", async () => {
const model: ModelConfig = {
name: "test-openai",
provider: "openai",
model: "gpt-4",
customReasoningFields: ["my_custom_thinking_key"],
};

const result = await llmsFromModelConfig({
model,
uniqueId: "test-id",
llmLogger: mockLLMLogger,
config: mockConfig,
});

expect(result).toHaveLength(1);
expect((result[0] as any).customReasoningFields).toEqual([
"my_custom_thinking_key",
]);
});

it("should handle empty headers correctly in merge", async () => {
const model: ModelConfig = {
name: "test-openai",
Expand Down
1 change: 1 addition & 0 deletions core/control-plane/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ const modelDescriptionSchema = z.object({
stream: z.boolean().optional(),
})
.optional(),
customReasoningFields: z.array(z.string()).optional(),
systemMessage: z.string().optional(),
requestOptions: z
.object({
Expand Down
11 changes: 10 additions & 1 deletion core/index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ type RequiredLLMOptions =
| "completionOptions";

export interface ILLM
extends Omit<LLMOptions, RequiredLLMOptions>,
extends
Omit<LLMOptions, RequiredLLMOptions>,
Required<Pick<LLMOptions, RequiredLLMOptions>> {
get providerName(): string;
get underlyingProviderName(): string;
Expand Down Expand Up @@ -714,6 +715,9 @@ export interface LLMOptions {

/** Tool overrides for this model */
toolOverrides?: ToolOverride[];

/** Custom fields to check for reasoning/thinking content in streaming chunks */
customReasoningFields?: string[];
}

type RequireAtLeastOne<T, Keys extends keyof T = keyof T> = Pick<
Expand Down Expand Up @@ -1259,6 +1263,9 @@ export interface ModelDescription {

/** Tool overrides for this model */
toolOverrides?: ToolOverride[];

/** Custom fields to check for reasoning/thinking content in streaming chunks */
customReasoningFields?: string[];
}

export interface JSONEmbedOptions {
Expand Down Expand Up @@ -1742,6 +1749,8 @@ export interface JSONModelDescription {
useResponsesApi?: boolean;
deploymentId?: string;
isFromAutoDetect?: boolean;
/** Custom fields to check for reasoning/thinking content in streaming chunks */
customReasoningFields?: string[];
}

// config.json
Expand Down
24 changes: 20 additions & 4 deletions core/llm/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import { isOllamaInstalled } from "../util/ollamaHelper.js";
import { TokensBatchingService } from "../util/TokensBatchingService.js";
import { withExponentialBackoff } from "../util/withExponentialBackoff.js";

import { applyToolOverrides } from "../tools/applyToolOverrides.js";
import {
autodetectPromptTemplates,
autodetectTemplateFunction,
Expand Down Expand Up @@ -67,7 +68,6 @@ import {
toCompleteBody,
toFimBody,
} from "./openaiTypeConverters.js";
import { applyToolOverrides } from "../tools/applyToolOverrides.js";

export class LLMError extends Error {
constructor(
Expand Down Expand Up @@ -210,6 +210,10 @@ export abstract class BaseLLM implements ILLM {

protected openaiAdapter?: BaseLlmApi;

public get options(): LLMOptions {
return this._llmOptions;
}

constructor(_options: LLMOptions) {
this._llmOptions = _options;
this.lastRequestId = undefined;
Expand Down Expand Up @@ -643,7 +647,13 @@ export abstract class BaseLLM implements ILLM {
if (!this.lastRequestId && typeof (chunk as any).id === "string") {
this.lastRequestId = (chunk as any).id;
}
const result = fromChatCompletionChunk(chunk);
const result = fromChatCompletionChunk(
chunk,
this.options?.customReasoningFields,
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
);
if (result && result.role === "thinking") {
continue;
}
if (result) {
const content = renderChatMessage(result);
const formattedContent = this._formatChatMessage(result);
Expand Down Expand Up @@ -1065,7 +1075,10 @@ export abstract class BaseLLM implements ILLM {
if (!this.lastRequestId && typeof (chunk as any).id === "string") {
this.lastRequestId = (chunk as any).id;
}
const chatChunk = fromChatCompletionChunk(chunk as any);
const chatChunk = fromChatCompletionChunk(
chunk as any,
this.options?.customReasoningFields,
);
if (chatChunk) {
yield chatChunk;
}
Expand All @@ -1084,7 +1097,10 @@ export abstract class BaseLLM implements ILLM {
signal,
);
this.lastRequestId = response.id ?? this.lastRequestId;
const messages = fromChatResponse(response as any);
const messages = fromChatResponse(
response as any,
this.options?.customReasoningFields,
);
for (const msg of messages) {
yield msg;
}
Expand Down
5 changes: 4 additions & 1 deletion core/llm/llms/OpenAI.ts
Original file line number Diff line number Diff line change
Expand Up @@ -560,7 +560,10 @@ class OpenAI extends BaseLLM {
}

for await (const value of streamSse(response)) {
const chunk = fromChatCompletionChunk(value);
const chunk = fromChatCompletionChunk(
value,
this.options?.customReasoningFields,
);
if (chunk) {
yield chunk;
}
Expand Down
5 changes: 4 additions & 1 deletion core/llm/llms/WatsonX.ts
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,10 @@ class WatsonX extends BaseLLM {
let accumulatedArgs = "";

for await (const value of streamSse(response)) {
const message = fromChatCompletionChunk(value);
const message = fromChatCompletionChunk(
value,
this.options?.customReasoningFields,
);
if (!!message) {
if (
(message as AssistantChatMessage)?.toolCalls &&
Expand Down
62 changes: 61 additions & 1 deletion core/llm/openaiTypeConverters.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
import { toResponsesInput, isItemType } from "./openaiTypeConverters";
import {
fromChatCompletionChunk,
fromChatResponse,
toResponsesInput,
isItemType,
} from "./openaiTypeConverters";
import { ChatMessage } from "..";
import type {
EasyInputMessage,
Expand Down Expand Up @@ -40,6 +45,61 @@ function getMessagesByRole(items: ResponseInputItem[], role: string) {
}

describe("openaiTypeConverters", () => {
describe("custom reasoning fields", () => {
it("should convert a custom streaming delta field to a thinking message", () => {
const chunk = {
choices: [
{
delta: {
my_custom_thinking_key: "checking constraints",
content: "should not render as chat text",
},
},
],
};

const result = fromChatCompletionChunk(chunk as any, [
"my_custom_thinking_key",
]);

expect(result).toEqual({
role: "thinking",
content: "checking constraints",
signature: undefined,
reasoning_details: undefined,
});
});

it("should convert a custom non-streaming message field to a thinking message", () => {
const response = {
choices: [
{
message: {
role: "assistant",
my_custom_thinking_key: "planning answer",
content: "final answer",
},
},
],
};

const result = fromChatResponse(response as any, [
"my_custom_thinking_key",
]);

expect(result).toEqual([
{
role: "thinking",
content: "planning answer",
},
{
role: "assistant",
content: "final answer",
},
]);
});
});

describe("toResponsesInput", () => {
describe("tool calls handling - OpenAI Responses API", () => {
it("should emit function_call items when fc_ ID is in metadata", () => {
Expand Down
47 changes: 31 additions & 16 deletions core/llm/openaiTypeConverters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,10 @@ export function toFimBody(
} as any;
}

export function fromChatResponse(response: ChatCompletion): ChatMessage[] {
export function fromChatResponse(
response: ChatCompletion,
customFields?: string[],
): ChatMessage[] {
const messages: ChatMessage[] = [];
const message = response.choices[0].message as ChatCompletionMessage & {
reasoning?: string;
Expand All @@ -298,11 +301,16 @@ export function fromChatResponse(response: ChatCompletion): ChatMessage[] {
}[];
};

const customContent = customFields
?.map((f) => (message as any)?.[f])
.find((v) => typeof v === "string" && v.length > 0);

// Check for reasoning content first (similar to fromChatCompletionChunk)
if (message.reasoning_content || message.reasoning) {
if (message.reasoning_content || message.reasoning || customContent) {
const thinkingMessage: ChatMessage = {
role: "thinking",
content: (message as any).reasoning_content || (message as any).reasoning,
content:
customContent || message.reasoning_content || message.reasoning || "",
};

// Preserve reasoning_details if present
Expand Down Expand Up @@ -346,6 +354,7 @@ export function fromChatResponse(response: ChatCompletion): ChatMessage[] {

export function fromChatCompletionChunk(
chunk: ChatCompletionChunk,
customFields?: string[],
): ChatMessage | undefined {
const delta = chunk.choices?.[0]?.delta as
| (ChatCompletionChunk.Choice.Delta & {
Expand All @@ -357,7 +366,25 @@ export function fromChatCompletionChunk(
})
| undefined;

if (delta?.content) {
const customContent = customFields
?.map((f) => (delta as any)?.[f])
.find((v) => typeof v === "string" && v.length > 0);

if (
delta?.reasoning_content ||
delta?.reasoning ||
delta?.reasoning_details?.length ||
customContent
) {
const message: ThinkingChatMessage = {
role: "thinking",
content:
customContent || delta?.reasoning_content || delta?.reasoning || "",
signature: delta?.reasoning_details?.[0]?.signature,
reasoning_details: delta?.reasoning_details as any[],
};
return message;
} else if (delta?.content) {
return {
role: "assistant",
content: delta.content,
Expand All @@ -381,18 +408,6 @@ export function fromChatCompletionChunk(
toolCalls,
};
}
} else if (
delta?.reasoning_content ||
delta?.reasoning ||
delta?.reasoning_details?.length
) {
const message: ThinkingChatMessage = {
role: "thinking",
content: delta.reasoning_content || delta.reasoning || "",
signature: delta?.reasoning_details?.[0]?.signature,
reasoning_details: delta?.reasoning_details as any[],
};
return message;
}

return undefined;
Expand Down
1 change: 1 addition & 0 deletions packages/config-types/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ export const modelDescriptionSchema = z.object({
])
.optional(),
completionOptions: completionOptionsSchema.optional(),
customReasoningFields: z.array(z.string()).optional(),
systemMessage: z.string().optional(),
requestOptions: z
.object({
Expand Down
Loading
Loading