From 2cf00729ba8b7ee099e072584a4b4d59df086cad Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 12 Aug 2026 09:48:57 -0700 Subject: [PATCH 1/3] feat(xai): wire reasoning effort through the Grok adapter The catalog never declared reasoningEffort for xAI and the adapter never sent reasoning_effort, so the flag was dead for every Grok model. Values are per-model and verified against the live API rather than the docs, which are wrong in three places: grok-4.5 does accept xhigh, grok-4.3 supports the parameter at all (undocumented) including none, and grok-4.20-0309-reasoning rejects it outright despite being a reasoning model. Also corrects grok-4.5's missing cachedInput and drops an inline comment the new provider TSDoc now covers. --- .../docs/en/workflows/blocks/agent.mdx | 1 + apps/sim/providers/models.ts | 15 +- apps/sim/providers/xai/index.test.ts | 187 ++++++++++++++++++ apps/sim/providers/xai/index.ts | 20 +- 4 files changed, 221 insertions(+), 2 deletions(-) create mode 100644 apps/sim/providers/xai/index.test.ts diff --git a/apps/docs/content/docs/en/workflows/blocks/agent.mdx b/apps/docs/content/docs/en/workflows/blocks/agent.mdx index d4ce50bcacc..f83b5370bab 100644 --- a/apps/docs/content/docs/en/workflows/blocks/agent.mdx +++ b/apps/docs/content/docs/en/workflows/blocks/agent.mdx @@ -115,6 +115,7 @@ Live tool-call chips stream for **OpenAI, Anthropic, Azure Anthropic, Google, Ve | Google | Summaries only | `gemini-3.6-flash`, `gemini-3.5-flash-lite`, `gemini-3.5-flash`, `gemini-3.1-pro-preview`, `gemini-3.1-flash-lite`, `gemini-3-flash-preview`, `gemini-2.5-pro`, `gemini-2.5-flash`, `gemini-2.5-flash-lite` | | Vertex AI | Summaries only | `vertex/gemini-3.5-flash`, `vertex/gemini-3.1-pro-preview`, `vertex/gemini-3.1-flash-lite`, `vertex/gemini-3-flash-preview`, `vertex/gemini-2.5-pro`, `vertex/gemini-2.5-flash`, `vertex/gemini-2.5-flash-lite` | | DeepSeek | Full thinking deltas | `deepseek-v4-pro`, `deepseek-v4-flash`, `deepseek-reasoner` | +| xAI | Full thinking deltas | `grok-4.6`, `grok-4.5`, `grok-4.3`, `grok-4.20-multi-agent-0309` | | Groq | Full thinking deltas | `groq/openai/gpt-oss-120b`, `groq/openai/gpt-oss-20b`, `groq/openai/gpt-oss-safeguard-20b`, `groq/qwen/qwen3.6-27b` | | Meta | Not streamed | `muse-spark-1.1` | | Kimi | Full thinking deltas | `kimi-k2.6` | diff --git a/apps/sim/providers/models.ts b/apps/sim/providers/models.ts index a0953b03c15..8f4e7de5592 100644 --- a/apps/sim/providers/models.ts +++ b/apps/sim/providers/models.ts @@ -2153,6 +2153,9 @@ export const PROVIDER_DEFINITIONS: Record = { }, capabilities: { temperature: { min: 0, max: 2 }, + reasoningEffort: { + values: ['low', 'medium', 'high', 'xhigh'], + }, }, contextWindow: 500000, releaseDate: '2026-08-12', @@ -2162,11 +2165,15 @@ export const PROVIDER_DEFINITIONS: Record = { id: 'grok-4.5', pricing: { input: 2.0, + cachedInput: 0.3, output: 6.0, - updatedAt: '2026-07-08', + updatedAt: '2026-08-12', }, capabilities: { temperature: { min: 0, max: 2 }, + reasoningEffort: { + values: ['low', 'medium', 'high', 'xhigh'], + }, }, contextWindow: 500000, releaseDate: '2026-07-08', @@ -2181,6 +2188,9 @@ export const PROVIDER_DEFINITIONS: Record = { }, capabilities: { temperature: { min: 0, max: 2 }, + reasoningEffort: { + values: ['none', 'low', 'medium', 'high', 'xhigh'], + }, }, contextWindow: 1000000, releaseDate: '2026-04-30', @@ -2328,6 +2338,9 @@ export const PROVIDER_DEFINITIONS: Record = { }, capabilities: { temperature: { min: 0, max: 2 }, + reasoningEffort: { + values: ['none', 'low', 'medium', 'high', 'xhigh'], + }, }, contextWindow: 1000000, releaseDate: '2026-03-10', diff --git a/apps/sim/providers/xai/index.test.ts b/apps/sim/providers/xai/index.test.ts new file mode 100644 index 00000000000..a08272ada5a --- /dev/null +++ b/apps/sim/providers/xai/index.test.ts @@ -0,0 +1,187 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCreate, mockExecuteProviderTool } = vi.hoisted(() => ({ + mockCreate: vi.fn(), + mockExecuteProviderTool: vi.fn(), +})) + +vi.mock('openai', () => ({ + default: vi.fn().mockImplementation( + class { + chat = { completions: { create: mockCreate } } + } + ), +})) + +vi.mock('@/providers', () => ({ MAX_TOOL_ITERATIONS: 20 })) + +vi.mock('@/providers/runtime-context', () => ({ + executeProviderTool: mockExecuteProviderTool, +})) + +vi.mock('@/providers/models', () => ({ + getProviderModels: () => [], + getProviderDefaultModel: () => '', +})) + +vi.mock('@/providers/attachments', () => ({ + formatMessagesForProvider: (messages: unknown) => messages, +})) + +vi.mock('@/providers/trace-enrichment', () => ({ + enrichLastModelSegmentFromChatCompletions: vi.fn(), +})) + +vi.mock('@/providers/transport', () => ({ openAICompatTransport: () => ({}) })) + +vi.mock('@/providers/tool-schema-adapter', () => ({ + adaptOpenAIChatToolSchema: (tool: { id: string }) => ({ + type: 'function', + function: { name: tool.id, parameters: {} }, + }), +})) + +vi.mock('@/providers/openai-compat/assistant-history', () => ({ + createOpenAICompatAssistantHistory: () => ({ role: 'assistant', content: '' }), +})) + +vi.mock('@/providers/openai-compat/stream-events', () => ({ + createOpenAICompatibleAgentEventStream: () => new ReadableStream({ start: (c) => c.close() }), +})) + +vi.mock('@/providers/stream-events', () => ({ + createSettledAgentEventStream: () => new ReadableStream({ start: (c) => c.close() }), +})) + +vi.mock('@/providers/streaming-execution', () => ({ + createStreamingExecution: vi.fn(() => ({ stream: null, execution: null })), +})) + +vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, + calculateCost: vi.fn(() => ({ input: 0, output: 0, total: 0 })), + sumToolCosts: vi.fn(() => 0), + prepareToolExecution: vi.fn((_tool, toolArgs) => ({ + toolParams: toolArgs, + executionParams: toolArgs, + })), + prepareToolsWithUsageControl: vi.fn((tools) => ({ + tools, + toolChoice: 'auto', + forcedTools: [], + hasFilteredTools: false, + })), + checkForForcedToolUsageOpenAI: vi.fn(() => ({ + hasUsedForcedTool: false, + usedForcedTools: [], + })), +})) + +import { xAIProvider } from '@/providers/xai' + +interface ChatOptions { + content?: string | null + toolCalls?: Array<{ id: string; function: { name: string; arguments: string } }> +} + +function chat({ content = null, toolCalls }: ChatOptions = {}) { + return { + choices: [ + { + message: { content, tool_calls: toolCalls }, + finish_reason: toolCalls ? 'tool_calls' : 'stop', + }, + ], + usage: { prompt_tokens: 5, completion_tokens: 3, total_tokens: 8 }, + } +} + +function tool(name: string) { + return { id: name, name, description: 'd', parameters: {} } +} + +function run(request: Record) { + return xAIProvider.executeRequest!({ + model: 'grok-4.6', + apiKey: 'test-key', + messages: [{ role: 'user', content: 'Hi' }], + ...request, + } as never) as Promise +} + +const firstPayload = () => mockCreate.mock.calls[0][0] +const lastPayload = () => mockCreate.mock.calls.at(-1)![0] + +describe('xAIProvider.executeRequest', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCreate.mockResolvedValue(chat({ content: 'hello' })) + mockExecuteProviderTool.mockResolvedValue({ + rawResponse: { success: true, output: { ok: true } }, + modelResponse: { success: true, output: { ok: true } }, + }) + }) + + it('maps temperature and max_completion_tokens', async () => { + await run({ temperature: 0.5, maxTokens: 256 }) + + const payload = firstPayload() + expect(payload.model).toBe('grok-4.6') + expect(payload.temperature).toBe(0.5) + expect(payload.max_completion_tokens).toBe(256) + }) + + it('forwards reasoning_effort only when set to a non-default value', async () => { + await run({ reasoningEffort: 'xhigh' }) + expect(firstPayload().reasoning_effort).toBe('xhigh') + + mockCreate.mockClear() + await run({ reasoningEffort: 'auto' }) + expect(firstPayload().reasoning_effort).toBeUndefined() + + mockCreate.mockClear() + await run({}) + expect(firstPayload().reasoning_effort).toBeUndefined() + }) + + it('keeps reasoning_effort on every follow-up call in the tool loop', async () => { + mockCreate + .mockResolvedValueOnce( + chat({ toolCalls: [{ id: 'c1', function: { name: 'known', arguments: '{"q":1}' } }] }) + ) + .mockResolvedValueOnce(chat({ content: 'done' })) + + await run({ tools: [tool('known')], reasoningEffort: 'high' }) + + expect(mockCreate.mock.calls.length).toBeGreaterThan(1) + for (const [payload] of mockCreate.mock.calls) { + expect(payload.reasoning_effort).toBe('high') + } + }) + + it('keeps reasoning_effort on the response_format request', async () => { + await run({ + reasoningEffort: 'low', + responseFormat: { name: 'r', schema: { type: 'object', properties: {} } }, + }) + + const payload = lastPayload() + expect(payload.response_format.type).toBe('json_schema') + expect(payload.reasoning_effort).toBe('low') + }) + + it('keeps reasoning_effort on the direct streaming request', async () => { + await run({ reasoningEffort: 'medium', stream: true }) + + const payload = firstPayload() + expect(payload.stream).toBe(true) + expect(payload.reasoning_effort).toBe('medium') + }) +}) diff --git a/apps/sim/providers/xai/index.ts b/apps/sim/providers/xai/index.ts index a89643c6674..dc44b9378d6 100644 --- a/apps/sim/providers/xai/index.ts +++ b/apps/sim/providers/xai/index.ts @@ -38,6 +38,20 @@ import { const logger = createLogger('XAIProvider') +/** + * xAI's Grok models via an OpenAI-compatible chat-completions API + * (`api.x.ai/v1`), with these documented deviations: + * - `reasoning_effort` maps from `request.reasoningEffort`. Sim's `auto` + * sentinel means "let the model pick its own default" and is never + * forwarded — xAI rejects it outright with `Invalid reasoning effort`. + * - Only some Grok models accept the parameter at all; the rest reject it with + * `does not support parameter reasoningEffort`. Which values each model takes + * is declared per-model in `capabilities.reasoningEffort`, which is also what + * gates the Agent block's effort dropdown. + * - Output length is capped via `max_completion_tokens`. + * - `tools` and `response_format` cannot be sent in the same request, so tools + * run first and the schema is applied on a follow-up pass. + */ export const xAIProvider: ProviderConfig = { id: 'xai', name: 'xAI', @@ -102,6 +116,11 @@ export const xAIProvider: ProviderConfig = { if (request.temperature !== undefined) basePayload.temperature = request.temperature if (request.maxTokens != null) basePayload.max_completion_tokens = request.maxTokens + + if (request.reasoningEffort !== undefined && request.reasoningEffort !== 'auto') { + basePayload.reasoning_effort = request.reasoningEffort + } + let preparedTools: ReturnType | null = null if (tools?.length) { @@ -166,7 +185,6 @@ export const xAIProvider: ProviderConfig = { try { const initialCallTime = Date.now() - // xAI cannot use tools and response_format together in the same request const initialPayload = { ...basePayload } let originalToolChoice: any From 469b7699bfd66844d0ee9dc5079ada526ed70f9f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 12 Aug 2026 09:53:32 -0700 Subject: [PATCH 2/3] test(xai): type the provider test helper instead of casting to any --- apps/sim/providers/xai/index.test.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/apps/sim/providers/xai/index.test.ts b/apps/sim/providers/xai/index.test.ts index a08272ada5a..fca35f6c5ce 100644 --- a/apps/sim/providers/xai/index.test.ts +++ b/apps/sim/providers/xai/index.test.ts @@ -84,6 +84,8 @@ vi.mock('@/providers/utils', () => ({ })), })) +import type { StreamingExecution } from '@/executor/types' +import type { ProviderRequest, ProviderResponse, ProviderToolConfig } from '@/providers/types' import { xAIProvider } from '@/providers/xai' interface ChatOptions { @@ -103,17 +105,25 @@ function chat({ content = null, toolCalls }: ChatOptions = {}) { } } -function tool(name: string) { - return { id: name, name, description: 'd', parameters: {} } +function tool(name: string): ProviderToolConfig { + return { + id: name, + name, + description: 'd', + params: {}, + parameters: { type: 'object', properties: {}, required: [] }, + } } -function run(request: Record) { +function run( + request: Partial = {} +): Promise { return xAIProvider.executeRequest!({ model: 'grok-4.6', apiKey: 'test-key', messages: [{ role: 'user', content: 'Hi' }], ...request, - } as never) as Promise + }) } const firstPayload = () => mockCreate.mock.calls[0][0] From 0d5dd6164fd7b04af8fc74d288214c5db33879de Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 12 Aug 2026 10:56:38 -0700 Subject: [PATCH 3/3] fix(agent): correct reasoning-effort copy that still claimed GPT-5 only --- .../docs/en/workflows/blocks/agent.mdx | 2 +- apps/sim/blocks/blocks/agent.ts | 5 ++++- apps/sim/providers/index.test.ts | 20 +++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/apps/docs/content/docs/en/workflows/blocks/agent.mdx b/apps/docs/content/docs/en/workflows/blocks/agent.mdx index f83b5370bab..1a6fb899444 100644 --- a/apps/docs/content/docs/en/workflows/blocks/agent.mdx +++ b/apps/docs/content/docs/en/workflows/blocks/agent.mdx @@ -143,7 +143,7 @@ The Agent reads the message from Start with `` and returns a result { question: "What are the memory options for the Agent block?", answer: "Four modes: None (no memory, each run is independent), Conversation (full history keyed by a conversation ID), Sliding window by messages (the N most recent messages), and Sliding window by tokens (messages up to a token budget). Memory needs a conversation ID to persist across runs." }, { question: "What is the difference between the tool usage controls (Auto, Force, None)?", answer: "In Auto, the model decides when to call a tool based on context. In Force, the model must call the tool on every run. In None, the tool is hidden from the model and never sent, which disables it without removing it from the block." }, { question: "How does the Response Format work?", answer: "It enforces structured output by providing a JSON Schema. When set, the model's response is constrained to match the schema exactly, and each field is read directly by downstream blocks using . Without a response format, the agent returns its standard outputs: content, model, tokens, and toolCalls." }, - { question: "What does the Reasoning Effort / Thinking Level setting do?", answer: "They appear only for models that support extended reasoning. Reasoning Effort (OpenAI o-series and GPT-5 models) and Thinking Level (Anthropic Claude and Gemini models with thinking) control how much compute the model spends reasoning before responding. Higher levels produce more thorough answers but cost more tokens and take longer." }, + { question: "What does the Reasoning Effort / Thinking Level setting do?", answer: "They appear only for models that support extended reasoning. Reasoning Effort (OpenAI, Azure OpenAI, xAI Grok, DeepSeek, Groq, Meta, and Z.ai models that accept an effort level) and Thinking Level (Anthropic Claude and Gemini models with thinking) control how much compute the model spends reasoning before responding. Higher levels produce more thorough answers but cost more tokens and take longer." }, { question: "When should I turn on Prompt Caching?", answer: "Turn it on when the same agent runs repeatedly with a large, stable system prompt or tool set — cached input bills at a tenth of the normal input rate. Leave it off for one-off runs, because writing the cache costs 1.25x and nothing reads it back. The setting appears only for Anthropic Claude models; OpenAI and Gemini cache automatically with no setting and no write fee. Anthropic only caches a prefix of at least 1,024 tokens (2,048 on Haiku), and entries expire after five minutes of no use." }, { question: "How does max output tokens work with Anthropic models?", answer: "The Agent block uses each Anthropic model's full max output token limit by default (for example, 64,000 tokens). You can override this with the Max Output Tokens setting. For non-streaming requests that exceed the SDK's internal threshold, the provider automatically uses internal streaming to avoid timeouts." }, { question: "Can I use the Agent block with a custom or self-hosted model?", answer: "Yes. Use any Ollama or VLLM-compatible model by typing the model name directly into the model combobox, as long as it exposes a compatible API endpoint." }, diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index c200d1dbfca..60b631ba6e3 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -664,7 +664,10 @@ Return ONLY the JSON array.`, }, temperature: { type: 'number', description: 'Response randomness level' }, maxTokens: { type: 'number', description: 'Maximum number of tokens in the response' }, - reasoningEffort: { type: 'string', description: 'Reasoning effort level for GPT-5 models' }, + reasoningEffort: { + type: 'string', + description: 'Reasoning effort level for models that support it', + }, verbosity: { type: 'string', description: 'Verbosity level for GPT-5 models' }, thinkingLevel: { type: 'string', diff --git a/apps/sim/providers/index.test.ts b/apps/sim/providers/index.test.ts index 06fc67d4d03..5e1203ed642 100644 --- a/apps/sim/providers/index.test.ts +++ b/apps/sim/providers/index.test.ts @@ -1405,6 +1405,26 @@ describe('executeProviderRequest — model level normalization', () => { expect(sentRequest().verbosity).toBe('high') }) + it('keeps the reasoning effort a Grok model declares', async () => { + await executeProviderRequest('xai', { + model: 'grok-4.6', + workspaceId: 'ws-1', + reasoningEffort: 'xhigh', + }) + + expect(sentRequest().reasoningEffort).toBe('xhigh') + }) + + it('drops the reasoning effort for a Grok model that rejects the parameter', async () => { + await executeProviderRequest('xai', { + model: 'grok-4.20-0309-reasoning', + workspaceId: 'ws-1', + reasoningEffort: 'high', + }) + + expect(sentRequest().reasoningEffort).toBeUndefined() + }) + /** * Sim's per-model level lists drive the pickers and can lag a provider that has started * accepting a new level, so an unrecognized level is forwarded rather than dropped: the