Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 27 additions & 15 deletions src/hooks/useAgentChat.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { useRef, useState } from 'react';
import { classifyIntent } from '@/tools/agent/intent';
import { executorFor } from '@/tools/agent/executors';
import { executorFor, AGENT_EXECUTORS } from '@/tools/agent/executors';
import { buildSystemPrompt, parseAction, type LoopTool } from '@/tools/agent/loop.lib';
import { emptySession, recordUser, applyResolution, historyForPrompt } from '@/tools/agent/session.lib';
import { prefillUrl } from '@/tools/agent/router.lib';
Expand Down Expand Up @@ -114,19 +114,28 @@ export function useAgentChat(provider: AgentProvider | null) {
return;
}

// TASK mode: the model may only call the scoped executors.
const loopTools: LoopTool[] = intent.executors.map(e => ({
// TASK mode. A capable cloud model gets the FULL tool catalog so it can plan
// and CHAIN tools (the output of one feeds the next) like an MCP toolset; a
// tiny on-device model gets only the keyword-scoped subset so it can't mis-pick.
const capable = provider.capable === true;
const offered = capable ? AGENT_EXECUTORS : intent.executors;
const loopTools: LoopTool[] = offered.map(e => ({
name: e.toolId,
description: e.description,
args: [
...e.files.map(f => ({ name: f.key, type: 'file' as const, required: true })),
...e.params.map(p => ({ name: p.key, type: p.type, required: p.default === undefined })),
],
}));
const convo: ChatMessage[] = [{ role: 'system', content: buildSystemPrompt(loopTools) }, { role: 'user', content: q }];
const systemPrompt = buildSystemPrompt(loopTools) + (capable
? '\n- You can call SEVERAL tools in sequence to fulfil one request. The output file of each tool automatically becomes the input for the next, so you can chain them (e.g. get audio from a video, then compress that audio). Plan the steps and call one tool per turn.'
: '');
const convo: ChatMessage[] = [{ role: 'system', content: systemPrompt }, { role: 'user', content: q }];
// Files available to this task: seeded from a prior turn on a continuation,
// and accumulated within the loop so a repeated call never re-prompts.
const loopFiles: Record<string, File> = intent.continued ? { ...lastFilesRef.current } : {};
// The most recent tool OUTPUT, offered as the input to the next tool (chaining).
let chainFile: File | null = null;
const doneKeys = new Set<string>();
let produced = false;
for (let iter = 0; iter < 8; iter++) {
Expand Down Expand Up @@ -163,18 +172,19 @@ export function useAgentChat(provider: AgentProvider | null) {
const files: Record<string, File> = {};
let cancelled = false;
for (const fs of exec.files) {
// Reuse a file already provided this task (or carried from the previous
// turn on a continuation) instead of prompting again.
const cached = loopFiles[fs.key];
try {
const f = cached ?? await requestFile(fs.label);
files[fs.key] = f;
loopFiles[fs.key] = f;
lastFilesRef.current[fs.key] = f;
} catch {
cancelled = true; // user dismissed the file request
break;
// Prefer the previous tool's OUTPUT (chaining), then a file already
// uploaded this task, otherwise ask. Don't cache a chained output as the
// "original upload" — a later "make it smaller" should re-run on the source.
const piped = chainFile;
const cached = piped ?? loopFiles[fs.key];
let f: File;
if (cached) { f = cached; }
else {
try { f = await requestFile(fs.label); }
catch { cancelled = true; break; }
}
files[fs.key] = f;
if (!piped) { loopFiles[fs.key] = f; lastFilesRef.current[fs.key] = f; }
}
if (cancelled) { updateLastText(`✗ ${exec.toolId} — cancelled`); break; }

Expand Down Expand Up @@ -202,6 +212,8 @@ export function useAgentChat(provider: AgentProvider | null) {
});
const blobUrl = result.blob ? URL.createObjectURL(result.blob) : undefined;
push({ role: 'assistant', text: `✓ ${exec.toolId}: ${result.text ?? 'produced a file'}`, blobUrl, imgUrl: result.dataUrl, filename: result.filename });
// Pipe this output into the next tool of the chain.
if (result.blob) chainFile = new File([result.blob], result.filename ?? 'output', { type: result.blob.type });
sessionRef.current = applyResolution(sessionRef.current, { toolId: exec.toolId, params, reply: result.text ?? 'done' });
doneKeys.add(key);
produced = true;
Expand Down
4 changes: 4 additions & 0 deletions src/services/agent/provider.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ function mockFetchOnce(json: unknown, ok = true) {
}

describe('createCloudProvider', () => {
it('is marked capable (cloud models get the full tool catalog + chaining)', () => {
const p = createCloudProvider({ kind: 'openai', baseUrl: 'https://api.example.com/v1', model: 'm', apiKey: 'sk-1' });
expect(p.capable).toBe(true);
});
it('calls the provider directly when not proxied (OpenAI-compatible)', async () => {
const f = mockFetchOnce({ choices: [{ message: { content: 'hi there' } }] });
const p = createCloudProvider({ kind: 'openai', baseUrl: 'https://api.example.com/v1', model: 'm', apiKey: 'sk-1' });
Expand Down
8 changes: 7 additions & 1 deletion src/services/agent/provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,12 @@
*/

export interface ChatMessage { role: 'system' | 'user' | 'assistant'; content: string }
export interface AgentProvider { chat(messages: ChatMessage[]): Promise<string> }
export interface AgentProvider {
chat(messages: ChatMessage[]): Promise<string>;
/** Capable enough to plan over the full tool catalog and chain tools (cloud
* models). Tiny on-device models are NOT — they get a keyword-scoped subset. */
capable?: boolean;
}

/** True when the browser exposes WebGPU (required for the on-device model). */
export function hasWebGPU(): boolean {
Expand Down Expand Up @@ -117,6 +122,7 @@ function providerFetch(proxy: boolean | undefined, target: string, headers: Reco
/** A provider that calls a cloud chat API with the user's key, from the browser. */
export function createCloudProvider(cfg: CloudConfig): AgentProvider {
return {
capable: true,
async chat(messages) {
if (cfg.kind === 'anthropic') {
const system = messages.find(m => m.role === 'system')?.content ?? '';
Expand Down
10 changes: 7 additions & 3 deletions src/tools/agent/intent.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,13 @@ describe('classifyIntent', () => {
});

it('chats about the agent itself instead of routing to a tool', () => {
expect(classifyIntent('what model are you').mode).toBe('chat');
expect(classifyIntent('who are you').mode).toBe('chat');
expect(classifyIntent('what can you do').mode).toBe('chat');
for (const q of [
'what model are you', 'what models are u?', 'which model r you',
'who are you', 'who r u', 'what can you do', 'what can u do',
'are you an ai', 'are u chatgpt', 'whats your name',
]) {
expect(classifyIntent(q).mode, q).toBe('chat');
}
});
it('opens the image crop tool for "how to crop image" (not the media trimmer)', () => {
const i = classifyIntent('how to crop image');
Expand Down
5 changes: 3 additions & 2 deletions src/tools/agent/intent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ export type Intent =
const CONTINUATION = /\b\d+(\.\d+)?\s?(kb|mb|gb|%|px|k|m)\b|\b(make it|instead|again|smaller|bigger|larger|lower|higher|reduce|to \d)/i;

// Questions about the agent itself — must chat, not route to a tool. Otherwise
// "what model are you" matches the browser-info tool (it has a "model" keyword).
const META = /\b(who are you|what are you|which model|what model are you|are you (an? )?(ai|llm|bot|gpt|model|human)|your name|what can you do|what do you do|how do you work|are you (chatgpt|claude|gpt|gemini))\b/i;
// "what model(s) are you/u" matches the browser-info tool (it has a "model"
// keyword). Handles plurals and "u"/"ya" for "you".
const META = /\b(what|which)\s+(models?|llms?|ai)\b|\b(who|what)\s+(are|r)\s+(you|u|ya)\b|\bare\s+(you|u|ya)\s+(an?\s+)?(ai|a\.?i\.?|llm|bot|gpt|model|robot|human|chatgpt|claude|gemini|deepseek)\b|\byour\s+name\b|\bwhat\s+(can|do)\s+(you|u|ya)\s+do\b|\bhow\s+do\s+(you|u)\s+work\b/i;

/**
* Decide how to handle a message. `activeToolId` (the tool from the previous
Expand Down
Loading