From d7a4a014268dfc48b444bdc080e88e601d49e80c Mon Sep 17 00:00:00 2001 From: Kresna <13603341+slaveofcode@users.noreply.github.com> Date: Sat, 29 Aug 2026 00:40:05 +0700 Subject: [PATCH 1/2] fix(agent): broaden AI-identity detection to plurals + 'u'/'r' shorthand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'What models are u?' still routed to browser-info — META only matched singular 'model' and 'you'. Now handles what/which model(s)/llm(s)/ai, who/what are/r you/u/ya, and 'are u chatgpt'. 'what is my browser' still routes to browser-info. --- src/tools/agent/intent.test.ts | 10 +++++++--- src/tools/agent/intent.ts | 5 +++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/tools/agent/intent.test.ts b/src/tools/agent/intent.test.ts index 1e4e8da..638d154 100644 --- a/src/tools/agent/intent.test.ts +++ b/src/tools/agent/intent.test.ts @@ -20,9 +20,13 @@ describe('classifyIntent', () => { }); it('chats about the agent itself instead of routing to a tool', () => { - expect(classifyIntent('what model are you').mode).toBe('chat'); - expect(classifyIntent('who are you').mode).toBe('chat'); - expect(classifyIntent('what can you do').mode).toBe('chat'); + for (const q of [ + 'what model are you', 'what models are u?', 'which model r you', + 'who are you', 'who r u', 'what can you do', 'what can u do', + 'are you an ai', 'are u chatgpt', 'whats your name', + ]) { + expect(classifyIntent(q).mode, q).toBe('chat'); + } }); it('opens the image crop tool for "how to crop image" (not the media trimmer)', () => { const i = classifyIntent('how to crop image'); diff --git a/src/tools/agent/intent.ts b/src/tools/agent/intent.ts index 7dde7f7..dd8e0a0 100644 --- a/src/tools/agent/intent.ts +++ b/src/tools/agent/intent.ts @@ -19,8 +19,9 @@ export type Intent = const CONTINUATION = /\b\d+(\.\d+)?\s?(kb|mb|gb|%|px|k|m)\b|\b(make it|instead|again|smaller|bigger|larger|lower|higher|reduce|to \d)/i; // Questions about the agent itself — must chat, not route to a tool. Otherwise -// "what model are you" matches the browser-info tool (it has a "model" keyword). -const META = /\b(who are you|what are you|which model|what model are you|are you (an? )?(ai|llm|bot|gpt|model|human)|your name|what can you do|what do you do|how do you work|are you (chatgpt|claude|gpt|gemini))\b/i; +// "what model(s) are you/u" matches the browser-info tool (it has a "model" +// keyword). Handles plurals and "u"/"ya" for "you". +const META = /\b(what|which)\s+(models?|llms?|ai)\b|\b(who|what)\s+(are|r)\s+(you|u|ya)\b|\bare\s+(you|u|ya)\s+(an?\s+)?(ai|a\.?i\.?|llm|bot|gpt|model|robot|human|chatgpt|claude|gemini|deepseek)\b|\byour\s+name\b|\bwhat\s+(can|do)\s+(you|u|ya)\s+do\b|\bhow\s+do\s+(you|u)\s+work\b/i; /** * Decide how to handle a message. `activeToolId` (the tool from the previous From e3cf4492504c8053b317834a97ac28c0b03a79cc Mon Sep 17 00:00:00 2001 From: Kresna <13603341+slaveofcode@users.noreply.github.com> Date: Sat, 29 Aug 2026 00:50:41 +0700 Subject: [PATCH 2/2] =?UTF-8?q?feat(agent):=20MCP-style=20tool=20harness?= =?UTF-8?q?=20=E2=80=94=20cloud=20models=20see=20all=20tools=20&=20chain?= =?UTF-8?q?=20them?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capable providers (cloud) now get the FULL executor catalog in task mode instead of the keyword-scoped subset, so the model can plan across every tool. Tool outputs pipe into the next tool's file input, so it can chain — e.g. 'get the audio from this video, then compress it'. Tiny on-device models stay scoped (they mis-pick). Adds AgentProvider.capable (cloud=true) and a multi-step system prompt. --- src/hooks/useAgentChat.ts | 42 ++++++++++++++++++----------- src/services/agent/provider.test.ts | 4 +++ src/services/agent/provider.ts | 8 +++++- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/src/hooks/useAgentChat.ts b/src/hooks/useAgentChat.ts index c6aef81..641ac5d 100644 --- a/src/hooks/useAgentChat.ts +++ b/src/hooks/useAgentChat.ts @@ -1,6 +1,6 @@ import { useRef, useState } from 'react'; import { classifyIntent } from '@/tools/agent/intent'; -import { executorFor } from '@/tools/agent/executors'; +import { executorFor, AGENT_EXECUTORS } from '@/tools/agent/executors'; import { buildSystemPrompt, parseAction, type LoopTool } from '@/tools/agent/loop.lib'; import { emptySession, recordUser, applyResolution, historyForPrompt } from '@/tools/agent/session.lib'; import { prefillUrl } from '@/tools/agent/router.lib'; @@ -114,8 +114,12 @@ export function useAgentChat(provider: AgentProvider | null) { return; } - // TASK mode: the model may only call the scoped executors. - const loopTools: LoopTool[] = intent.executors.map(e => ({ + // TASK mode. A capable cloud model gets the FULL tool catalog so it can plan + // and CHAIN tools (the output of one feeds the next) like an MCP toolset; a + // tiny on-device model gets only the keyword-scoped subset so it can't mis-pick. + const capable = provider.capable === true; + const offered = capable ? AGENT_EXECUTORS : intent.executors; + const loopTools: LoopTool[] = offered.map(e => ({ name: e.toolId, description: e.description, args: [ @@ -123,10 +127,15 @@ export function useAgentChat(provider: AgentProvider | null) { ...e.params.map(p => ({ name: p.key, type: p.type, required: p.default === undefined })), ], })); - const convo: ChatMessage[] = [{ role: 'system', content: buildSystemPrompt(loopTools) }, { role: 'user', content: q }]; + const systemPrompt = buildSystemPrompt(loopTools) + (capable + ? '\n- You can call SEVERAL tools in sequence to fulfil one request. The output file of each tool automatically becomes the input for the next, so you can chain them (e.g. get audio from a video, then compress that audio). Plan the steps and call one tool per turn.' + : ''); + const convo: ChatMessage[] = [{ role: 'system', content: systemPrompt }, { role: 'user', content: q }]; // Files available to this task: seeded from a prior turn on a continuation, // and accumulated within the loop so a repeated call never re-prompts. const loopFiles: Record = intent.continued ? { ...lastFilesRef.current } : {}; + // The most recent tool OUTPUT, offered as the input to the next tool (chaining). + let chainFile: File | null = null; const doneKeys = new Set(); let produced = false; for (let iter = 0; iter < 8; iter++) { @@ -163,18 +172,19 @@ export function useAgentChat(provider: AgentProvider | null) { const files: Record = {}; let cancelled = false; for (const fs of exec.files) { - // Reuse a file already provided this task (or carried from the previous - // turn on a continuation) instead of prompting again. - const cached = loopFiles[fs.key]; - try { - const f = cached ?? await requestFile(fs.label); - files[fs.key] = f; - loopFiles[fs.key] = f; - lastFilesRef.current[fs.key] = f; - } catch { - cancelled = true; // user dismissed the file request - break; + // Prefer the previous tool's OUTPUT (chaining), then a file already + // uploaded this task, otherwise ask. Don't cache a chained output as the + // "original upload" — a later "make it smaller" should re-run on the source. + const piped = chainFile; + const cached = piped ?? loopFiles[fs.key]; + let f: File; + if (cached) { f = cached; } + else { + try { f = await requestFile(fs.label); } + catch { cancelled = true; break; } } + files[fs.key] = f; + if (!piped) { loopFiles[fs.key] = f; lastFilesRef.current[fs.key] = f; } } if (cancelled) { updateLastText(`✗ ${exec.toolId} — cancelled`); break; } @@ -202,6 +212,8 @@ export function useAgentChat(provider: AgentProvider | null) { }); const blobUrl = result.blob ? URL.createObjectURL(result.blob) : undefined; push({ role: 'assistant', text: `✓ ${exec.toolId}: ${result.text ?? 'produced a file'}`, blobUrl, imgUrl: result.dataUrl, filename: result.filename }); + // Pipe this output into the next tool of the chain. + if (result.blob) chainFile = new File([result.blob], result.filename ?? 'output', { type: result.blob.type }); sessionRef.current = applyResolution(sessionRef.current, { toolId: exec.toolId, params, reply: result.text ?? 'done' }); doneKeys.add(key); produced = true; diff --git a/src/services/agent/provider.test.ts b/src/services/agent/provider.test.ts index fcf1ad5..743739f 100644 --- a/src/services/agent/provider.test.ts +++ b/src/services/agent/provider.test.ts @@ -10,6 +10,10 @@ function mockFetchOnce(json: unknown, ok = true) { } describe('createCloudProvider', () => { + it('is marked capable (cloud models get the full tool catalog + chaining)', () => { + const p = createCloudProvider({ kind: 'openai', baseUrl: 'https://api.example.com/v1', model: 'm', apiKey: 'sk-1' }); + expect(p.capable).toBe(true); + }); it('calls the provider directly when not proxied (OpenAI-compatible)', async () => { const f = mockFetchOnce({ choices: [{ message: { content: 'hi there' } }] }); const p = createCloudProvider({ kind: 'openai', baseUrl: 'https://api.example.com/v1', model: 'm', apiKey: 'sk-1' }); diff --git a/src/services/agent/provider.ts b/src/services/agent/provider.ts index ded809d..aed9831 100644 --- a/src/services/agent/provider.ts +++ b/src/services/agent/provider.ts @@ -7,7 +7,12 @@ */ export interface ChatMessage { role: 'system' | 'user' | 'assistant'; content: string } -export interface AgentProvider { chat(messages: ChatMessage[]): Promise } +export interface AgentProvider { + chat(messages: ChatMessage[]): Promise; + /** Capable enough to plan over the full tool catalog and chain tools (cloud + * models). Tiny on-device models are NOT — they get a keyword-scoped subset. */ + capable?: boolean; +} /** True when the browser exposes WebGPU (required for the on-device model). */ export function hasWebGPU(): boolean { @@ -117,6 +122,7 @@ function providerFetch(proxy: boolean | undefined, target: string, headers: Reco /** A provider that calls a cloud chat API with the user's key, from the browser. */ export function createCloudProvider(cfg: CloudConfig): AgentProvider { return { + capable: true, async chat(messages) { if (cfg.kind === 'anthropic') { const system = messages.find(m => m.role === 'system')?.content ?? '';