From d2ca939b4d293b5f2fc6f067d9b9a7fd5fbb25f1 Mon Sep 17 00:00:00 2001 From: Kresna <13603341+slaveofcode@users.noreply.github.com> Date: Sat, 29 Aug 2026 07:11:51 +0700 Subject: [PATCH] feat(agent): native function-calling for cloud providers (reliable tool use) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add provider.chatTools using the real OpenAI tools/tool_calls and Anthropic tool_use APIs — so big/complex args (SVG, code, multi-step plans) are encoded by the provider, not hand-formatted by the model as fragile JSON-in-prose. useAgentChat now runs a NATIVE tool loop for cloud providers (proper assistant/tool messages), falling back to the prompt-based JSON loop for on-device WebLLM. Executor running (file/param collection, chaining, param prompting) extracted into a shared runExecutor used by both loops. --- src/hooks/useAgentChat.ts | 171 +++++++++++++++------------- src/services/agent/provider.test.ts | 21 ++++ src/services/agent/provider.ts | 78 +++++++++++++ 3 files changed, 192 insertions(+), 78 deletions(-) diff --git a/src/hooks/useAgentChat.ts b/src/hooks/useAgentChat.ts index 5d169f6..0fb2b62 100644 --- a/src/hooks/useAgentChat.ts +++ b/src/hooks/useAgentChat.ts @@ -6,7 +6,7 @@ import { emptySession, recordUser, applyResolution, historyForPrompt } from '@/t import { prefillUrl } from '@/tools/agent/router.lib'; import { buildToolChoicePrompt, parseToolChoice } from '@/tools/agent/select.lib'; import { getToolById } from '@/registry/tools'; -import type { AgentProvider, ChatMessage } from '@/services/agent/provider'; +import type { AgentProvider, ChatMessage, ToolSpec, ToolMsg } from '@/services/agent/provider'; export interface ChatUiTurn { role: 'user' | 'assistant'; @@ -119,113 +119,128 @@ export function useAgentChat(provider: AgentProvider | null) { // tiny on-device model gets only the keyword-scoped subset so it can't mis-pick. const capable = provider.capable === true; const offered = capable ? AGENT_EXECUTORS : intent.executors; - const loopTools: LoopTool[] = offered.map(e => ({ - name: e.toolId, - description: e.description, - args: [ - ...e.files.map(f => ({ name: f.key, type: 'file' as const, required: true })), - ...e.params.map(p => ({ name: p.key, type: p.type, required: p.default === undefined })), - ], - })); - const systemPrompt = buildSystemPrompt(loopTools) + (capable - ? '\n- You can call SEVERAL tools in sequence to fulfil one request. The output file of each tool automatically becomes the input for the next, so you can chain them (e.g. get audio from a video, then compress that audio). Plan the steps and call one tool per turn.' - : ''); - const convo: ChatMessage[] = [{ role: 'system', content: systemPrompt }, { role: 'user', content: q }]; - // Files available to this task: seeded from a prior turn on a continuation, - // and accumulated within the loop so a repeated call never re-prompts. + const offeredIds = offered.map(e => e.toolId); + // Files available to this task: seeded from a prior turn on a continuation. const loopFiles: Record = intent.continued ? { ...lastFilesRef.current } : {}; - // The most recent tool OUTPUT, offered as the input to the next tool (chaining). - let chainFile: File | null = null; - const doneKeys = new Set(); + let chainFile: File | null = null; // last tool OUTPUT, piped into the next tool let produced = false; - for (let iter = 0; iter < 8; iter++) { - const raw = await provider.chat(convo); - // Generative tools (svg/canvas) often emit the artifact itself instead of - // a JSON action — a big SVG/code blob rarely survives JSON-escaping — so - // recover it and run the right tool. - const act = parseAction(raw) ?? recoverContentAction(raw, offered.map(e => e.toolId)); - if (!act) { - // Unparseable turn (small models emit junk). If we already handed the - // user a result, just close out cleanly — never dump raw JSON to chat. - push({ role: 'assistant', text: produced ? 'Done — anything else?' : "I didn't quite catch that. Could you rephrase what you'd like to do?" }); - break; - } - if (act.action === 'final') { - const r = act.text || (produced ? 'Done.' : '(done)'); - push({ role: 'assistant', text: r }); - sessionRef.current = applyResolution(sessionRef.current, { toolId: null, params: {}, reply: r }); - break; - } + const wordCount = (s: string) => s.trim().split(/\s+/).filter(Boolean).length; - const key = act.tool + JSON.stringify(act.args); - if (doneKeys.has(key)) { - // Model re-issued a call it already completed — the result is already - // shown. Stop instead of re-running (or re-prompting for the file). - push({ role: 'assistant', text: 'Done — anything else?' }); - break; - } - - const exec = executorFor(act.tool); - if (!exec) { - convo.push({ role: 'assistant', content: raw }, { role: 'user', content: `TOOL_ERROR: unknown tool ${act.tool}` }); - continue; - } + // Run ONE executor call: collect files (chaining/upload) + required params + // (asking the user for hallucinated/empty ones), execute, show the result, + // and pipe its output. Shared by both the native and prompt loops. + const runExecutor = async (exec: (typeof AGENT_EXECUTORS)[number], argsIn: Record): Promise<{ ok: boolean; resultText: string; cancelled?: boolean }> => { push({ role: 'assistant', text: `→ ${exec.toolId}` }); - const files: Record = {}; - let cancelled = false; for (const fs of exec.files) { - // Prefer the previous tool's OUTPUT (chaining), then a file already - // uploaded this task, otherwise ask. Don't cache a chained output as the - // "original upload" — a later "make it smaller" should re-run on the source. - const piped = chainFile; + const piped = chainFile; // prefer the previous tool's output (chaining) const cached = piped ?? loopFiles[fs.key]; let f: File; if (cached) { f = cached; } else { try { f = await requestFile(fs.label); } - catch { cancelled = true; break; } + catch { updateLastText(`✗ ${exec.toolId} — cancelled`); return { ok: false, resultText: 'cancelled', cancelled: true }; } } files[fs.key] = f; - if (!piped) { loopFiles[fs.key] = f; lastFilesRef.current[fs.key] = f; } + if (!piped) { loopFiles[fs.key] = f; lastFilesRef.current[fs.key] = f; } // remember only real uploads } - if (cancelled) { updateLastText(`✗ ${exec.toolId} — cancelled`); break; } - - // Fill any required text param the model left empty — OR hallucinated — - // by asking the user. A tiny model often echoes the command itself as the - // value ("QR" → a QR of the word "QR"); treat as missing when the value is - // empty, equals the whole query, or is a short phrase that itself triggers - // this same tool (a command word, not content). - const params: Record = { ...act.args }; - const wordCount = (s: string) => s.trim().split(/\s+/).filter(Boolean).length; + const params: Record = { ...argsIn }; for (const ps of exec.params) { - if (ps.default !== undefined) continue; // optional / has a default + if (ps.default !== undefined) continue; const raw = params[ps.key] == null ? '' : String(params[ps.key]).trim(); const echoed = raw !== '' && (raw.toLowerCase() === q.trim().toLowerCase() || (wordCount(raw) <= 3 && exec.match(raw))); if (raw === '' || raw.toUpperCase() === 'UPLOAD' || echoed) { try { params[ps.key] = await requestInput(ps.label); } - catch { cancelled = true; break; } + catch { updateLastText(`✗ ${exec.toolId} — cancelled`); return { ok: false, resultText: 'cancelled', cancelled: true }; } } } - if (cancelled) { updateLastText(`✗ ${exec.toolId} — cancelled`); break; } - try { - const result = await exec.execute({ files, params }, p => { - updateLastText(`→ ${exec.toolId} — ${Math.round(p * 100)}%`); - }); + const result = await exec.execute({ files, params }, p => updateLastText(`→ ${exec.toolId} — ${Math.round(p * 100)}%`)); const blobUrl = result.blob ? URL.createObjectURL(result.blob) : undefined; push({ role: 'assistant', text: `✓ ${exec.toolId}: ${result.text ?? 'produced a file'}`, blobUrl, imgUrl: result.dataUrl, filename: result.filename }); - // Pipe this output into the next tool of the chain. if (result.blob) chainFile = new File([result.blob], result.filename ?? 'output', { type: result.blob.type }); sessionRef.current = applyResolution(sessionRef.current, { toolId: exec.toolId, params, reply: result.text ?? 'done' }); - doneKeys.add(key); produced = true; - convo.push({ role: 'assistant', content: raw }, { role: 'user', content: `TOOL_RESULT ${exec.toolId}: ${result.text ? result.text.slice(0, 400) : 'produced a file for the user'}. Respond with a "final" action now unless the user asked for more.` }); + return { ok: true, resultText: result.text ? result.text.slice(0, 400) : 'produced a file for the user' }; } catch (e) { push({ role: 'assistant', text: `✗ ${exec.toolId} error: ${(e as Error).message}` }); - convo.push({ role: 'assistant', content: raw }, { role: 'user', content: `TOOL_ERROR ${exec.toolId}: ${(e as Error).message}` }); + return { ok: false, resultText: `error: ${(e as Error).message}` }; + } + }; + + // --- Native function-calling loop (cloud): the provider's real tools API --- + const chatTools = provider.chatTools; + if (chatTools) { + const tools: ToolSpec[] = offered.map(e => ({ + name: e.toolId, + description: e.description, + parameters: { + type: 'object', + properties: { + ...Object.fromEntries(e.files.map(f => [f.key, { type: 'string', description: `${f.label} — pass the string "UPLOAD" and the app will ask the user for the file` }])), + ...Object.fromEntries(e.params.map(p => [p.key, { type: p.type === 'number' ? 'number' : 'string', description: p.label }])), + }, + required: [...e.files.map(f => f.key), ...e.params.filter(p => p.default === undefined).map(p => p.key)], + }, + })); + const sys = "You are GoodWebTools' agent. Use the tools to fulfil the user's request. You can call several tools in sequence — each tool's output file automatically becomes the next tool's input, so you can chain them. For a file argument pass the string \"UPLOAD\". When the task is done, reply with a short final message and no tool call."; + const msgs: ToolMsg[] = [{ role: 'system', content: sys }, { role: 'user', content: q }]; + const done = new Set(); + for (let iter = 0; iter < 8; iter++) { + const turn = await chatTools(msgs, tools); + if (!turn.calls.length) { + const r = turn.text || (produced ? 'Done.' : 'Okay.'); + push({ role: 'assistant', text: r }); + sessionRef.current = applyResolution(sessionRef.current, { toolId: null, params: {}, reply: r }); + break; + } + msgs.push({ role: 'assistant', content: turn.text, toolCalls: turn.calls }); + let stop = false; + for (const call of turn.calls) { + const exec = executorFor(call.name); + if (!exec) { msgs.push({ role: 'tool', toolCallId: call.id, content: `unknown tool ${call.name}` }); continue; } + const key = call.name + JSON.stringify(call.args); + if (done.has(key)) { msgs.push({ role: 'tool', toolCallId: call.id, content: 'already done — reply with a final message' }); continue; } + const res = await runExecutor(exec, call.args); + if (res.cancelled) { stop = true; break; } + done.add(key); + msgs.push({ role: 'tool', toolCallId: call.id, content: res.resultText }); + } + if (stop) break; + if (iter === 7) push({ role: 'assistant', text: produced ? 'Done — anything else?' : "I couldn't finish that." }); + } + return; + } + + // --- Prompt-based loop (on-device / no tools API): JSON action protocol --- + const loopTools: LoopTool[] = offered.map(e => ({ + name: e.toolId, + description: e.description, + args: [ + ...e.files.map(f => ({ name: f.key, type: 'file' as const, required: true })), + ...e.params.map(p => ({ name: p.key, type: p.type, required: p.default === undefined })), + ], + })); + const convo: ChatMessage[] = [{ role: 'system', content: buildSystemPrompt(loopTools) }, { role: 'user', content: q }]; + const doneKeys = new Set(); + for (let iter = 0; iter < 8; iter++) { + const raw = await provider.chat(convo); + const act = parseAction(raw) ?? recoverContentAction(raw, offeredIds); + if (!act) { push({ role: 'assistant', text: produced ? 'Done — anything else?' : "I didn't quite catch that. Could you rephrase what you'd like to do?" }); break; } + if (act.action === 'final') { + const r = act.text || (produced ? 'Done.' : '(done)'); + push({ role: 'assistant', text: r }); + sessionRef.current = applyResolution(sessionRef.current, { toolId: null, params: {}, reply: r }); + break; } - // Ran out of iterations mid-task: close out rather than leave it hanging. + const key = act.tool + JSON.stringify(act.args); + if (doneKeys.has(key)) { push({ role: 'assistant', text: 'Done — anything else?' }); break; } + const exec = executorFor(act.tool); + if (!exec) { convo.push({ role: 'assistant', content: raw }, { role: 'user', content: `TOOL_ERROR: unknown tool ${act.tool}` }); continue; } + const res = await runExecutor(exec, act.args); + if (res.cancelled) break; + doneKeys.add(key); + convo.push({ role: 'assistant', content: raw }, { role: 'user', content: `TOOL_RESULT ${exec.toolId}: ${res.resultText}. Respond with a "final" action now unless the user asked for more.` }); if (iter === 7) push({ role: 'assistant', text: produced ? 'Done — anything else?' : "I couldn't finish that — try rephrasing, or a bigger model." }); } } catch (e) { diff --git a/src/services/agent/provider.test.ts b/src/services/agent/provider.test.ts index 743739f..8b0a806 100644 --- a/src/services/agent/provider.test.ts +++ b/src/services/agent/provider.test.ts @@ -36,6 +36,27 @@ describe('createCloudProvider', () => { expect(headers.authorization).toBe('Bearer sk-2'); }); + it('parses OpenAI tool_calls into ToolCall[] and sends the tools', async () => { + const f = mockFetchOnce({ choices: [{ message: { content: '', tool_calls: [{ id: 'c1', type: 'function', function: { name: 'qr-gen', arguments: '{"text":"hi"}' } }] } }] }); + const p = createCloudProvider({ kind: 'openai', baseUrl: 'https://api.example.com/v1', model: 'm', apiKey: 'sk-1' }); + const turn = await p.chatTools!([{ role: 'user', content: 'make a qr' }], [{ name: 'qr-gen', description: 'qr', parameters: { type: 'object', properties: { text: { type: 'string' } } } }]); + expect(turn.calls).toEqual([{ id: 'c1', name: 'qr-gen', args: { text: 'hi' } }]); + const body = JSON.parse((f.mock.calls[0][1] as RequestInit).body as string); + expect(body.tools[0].function.name).toBe('qr-gen'); + expect(body.tool_choice).toBe('auto'); + }); + it('returns final text with no calls', async () => { + mockFetchOnce({ choices: [{ message: { content: 'All done!' } }] }); + const p = createCloudProvider({ kind: 'openai', baseUrl: 'x', model: 'm', apiKey: 'k' }); + expect(await p.chatTools!([{ role: 'user', content: 'hi' }], [])).toEqual({ text: 'All done!', calls: [] }); + }); + it('parses Anthropic tool_use blocks', async () => { + mockFetchOnce({ content: [{ type: 'text', text: 'sure' }, { type: 'tool_use', id: 't1', name: 'svg-viewer', input: { svg: '' } }] }); + const p = createCloudProvider({ kind: 'anthropic', baseUrl: 'https://api.anthropic.com', model: 'claude', apiKey: 'k' }); + const turn = await p.chatTools!([{ role: 'user', content: 'draw' }], [{ name: 'svg-viewer', description: 'd', parameters: {} }]); + expect(turn.text).toBe('sure'); + expect(turn.calls).toEqual([{ id: 't1', name: 'svg-viewer', args: { svg: '' } }]); + }); it('proxies the Anthropic messages endpoint with x-llm-target', async () => { const f = mockFetchOnce({ content: [{ text: 'claude reply' }] }); const p = createCloudProvider({ kind: 'anthropic', baseUrl: 'https://api.anthropic.com', model: 'claude', apiKey: 'sk-3', proxy: true }); diff --git a/src/services/agent/provider.ts b/src/services/agent/provider.ts index aed9831..baa1cec 100644 --- a/src/services/agent/provider.ts +++ b/src/services/agent/provider.ts @@ -7,11 +7,41 @@ */ export interface ChatMessage { role: 'system' | 'user' | 'assistant'; content: string } + +// --- Native function-calling --------------------------------------------------- +/** A tool offered to the model (JSON-Schema params), like MCP tools/list. */ +export interface ToolSpec { name: string; description: string; parameters: Record } +/** A tool call the model asked for. */ +export interface ToolCall { id: string; name: string; args: Record } +/** One turn of a native tool loop: assistant text (optional) + tool calls. */ +export interface ToolTurn { text: string; calls: ToolCall[] } +/** Messages for the native loop — richer than ChatMessage (carry calls/results). */ +export interface ToolMsg { + role: 'system' | 'user' | 'assistant' | 'tool'; + content: string; + toolCalls?: ToolCall[]; // on an assistant turn that called tools + toolCallId?: string; // on a tool-result turn +} + export interface AgentProvider { chat(messages: ChatMessage[]): Promise; /** Capable enough to plan over the full tool catalog and chain tools (cloud * models). Tiny on-device models are NOT — they get a keyword-scoped subset. */ capable?: boolean; + /** Native tool-calling step (present on cloud providers). Uses the provider's + * real tools API so big args (SVG/code) are encoded reliably, not model-typed. */ + chatTools?(messages: ToolMsg[], tools: ToolSpec[]): Promise; +} + +/** Coerce a provider's tool-call args object to the scalar shape executors expect. */ +function coerceArgs(input: unknown): Record { + const out: Record = {}; + if (input && typeof input === 'object') { + for (const [k, v] of Object.entries(input as Record)) { + out[k] = typeof v === 'number' ? v : typeof v === 'string' ? v : (typeof v === 'boolean' ? String(v) : JSON.stringify(v)); + } + } + return out; } /** True when the browser exposes WebGPU (required for the on-device model). */ @@ -146,5 +176,53 @@ export function createCloudProvider(cfg: CloudConfig): AgentProvider { if (!r.ok) throw new Error(j.error?.message || 'API error'); return j.choices?.[0]?.message?.content ?? ''; }, + + async chatTools(messages, tools) { + if (cfg.kind === 'anthropic') { + const system = messages.find(m => m.role === 'system')?.content ?? ''; + const conv = messages.filter(m => m.role !== 'system').map(m => { + if (m.role === 'tool') return { role: 'user', content: [{ type: 'tool_result', tool_use_id: m.toolCallId, content: m.content }] }; + if (m.role === 'assistant' && m.toolCalls?.length) { + return { role: 'assistant', content: [ + ...(m.content ? [{ type: 'text', text: m.content }] : []), + ...m.toolCalls.map(c => ({ type: 'tool_use', id: c.id, name: c.name, input: c.args })), + ] }; + } + return { role: m.role === 'assistant' ? 'assistant' : 'user', content: m.content }; + }); + const r = await providerFetch(cfg.proxy, cfg.baseUrl + '/v1/messages', { + 'content-type': 'application/json', 'x-api-key': cfg.apiKey, + 'anthropic-version': '2023-06-01', 'anthropic-dangerous-direct-browser-access': 'true', + }, { model: cfg.model, max_tokens: 1500, system, tools: tools.map(t => ({ name: t.name, description: t.description, input_schema: t.parameters })), messages: conv }); + const j = await r.json(); + if (!r.ok) throw new Error(j.error?.message || 'API error'); + let text = ''; const calls: ToolCall[] = []; + for (const b of j.content ?? []) { + if (b.type === 'text') text += b.text; + else if (b.type === 'tool_use') calls.push({ id: b.id, name: b.name, args: coerceArgs(b.input) }); + } + return { text, calls }; + } + // OpenAI-compatible (GLM, OpenAI, DeepSeek, Groq, Gemini, OpenRouter, OpenCode). + const msgs = messages.map(m => { + if (m.role === 'tool') return { role: 'tool', tool_call_id: m.toolCallId, content: m.content }; + if (m.role === 'assistant' && m.toolCalls?.length) { + return { role: 'assistant', content: m.content || null, tool_calls: m.toolCalls.map(c => ({ id: c.id, type: 'function', function: { name: c.name, arguments: JSON.stringify(c.args) } })) }; + } + return { role: m.role, content: m.content }; + }); + const r = await providerFetch(cfg.proxy, cfg.baseUrl + '/chat/completions', { + 'content-type': 'application/json', authorization: `Bearer ${cfg.apiKey}`, + }, { model: cfg.model, temperature: 0.2, max_tokens: 1500, tool_choice: 'auto', tools: tools.map(t => ({ type: 'function', function: { name: t.name, description: t.description, parameters: t.parameters } })), messages: msgs }); + const j = await r.json(); + if (!r.ok) throw new Error(j.error?.message || 'API error'); + const msg = j.choices?.[0]?.message ?? {}; + const calls: ToolCall[] = (msg.tool_calls ?? []).map((tc: { id: string; function?: { name: string; arguments?: string } }) => { + let parsed: unknown = {}; + try { parsed = JSON.parse(tc.function?.arguments || '{}'); } catch { /* keep {} */ } + return { id: tc.id, name: tc.function?.name ?? '', args: coerceArgs(parsed) }; + }); + return { text: msg.content ?? '', calls }; + }, }; }