Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 27 additions & 15 deletions src/hooks/useAgentChat.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { useRef, useState } from 'react';
import { classifyIntent } from '@/tools/agent/intent';
import { executorFor } from '@/tools/agent/executors';
import { executorFor, AGENT_EXECUTORS } from '@/tools/agent/executors';
import { buildSystemPrompt, parseAction, type LoopTool } from '@/tools/agent/loop.lib';
import { emptySession, recordUser, applyResolution, historyForPrompt } from '@/tools/agent/session.lib';
import { prefillUrl } from '@/tools/agent/router.lib';
Expand Down Expand Up @@ -114,19 +114,28 @@ export function useAgentChat(provider: AgentProvider | null) {
return;
}

// TASK mode: the model may only call the scoped executors.
const loopTools: LoopTool[] = intent.executors.map(e => ({
// TASK mode. A capable cloud model gets the FULL tool catalog so it can plan
// and CHAIN tools (the output of one feeds the next) like an MCP toolset; a
// tiny on-device model gets only the keyword-scoped subset so it can't mis-pick.
const capable = provider.capable === true;
const offered = capable ? AGENT_EXECUTORS : intent.executors;
const loopTools: LoopTool[] = offered.map(e => ({
name: e.toolId,
description: e.description,
args: [
...e.files.map(f => ({ name: f.key, type: 'file' as const, required: true })),
...e.params.map(p => ({ name: p.key, type: p.type, required: p.default === undefined })),
],
}));
const convo: ChatMessage[] = [{ role: 'system', content: buildSystemPrompt(loopTools) }, { role: 'user', content: q }];
const systemPrompt = buildSystemPrompt(loopTools) + (capable
? '\n- You can call SEVERAL tools in sequence to fulfil one request. The output file of each tool automatically becomes the input for the next, so you can chain them (e.g. get audio from a video, then compress that audio). Plan the steps and call one tool per turn.'
: '');
const convo: ChatMessage[] = [{ role: 'system', content: systemPrompt }, { role: 'user', content: q }];
// Files available to this task: seeded from a prior turn on a continuation,
// and accumulated within the loop so a repeated call never re-prompts.
const loopFiles: Record<string, File> = intent.continued ? { ...lastFilesRef.current } : {};
// The most recent tool OUTPUT, offered as the input to the next tool (chaining).
let chainFile: File | null = null;
const doneKeys = new Set<string>();
let produced = false;
for (let iter = 0; iter < 8; iter++) {
Expand Down Expand Up @@ -163,18 +172,19 @@ export function useAgentChat(provider: AgentProvider | null) {
const files: Record<string, File> = {};
let cancelled = false;
for (const fs of exec.files) {
// Reuse a file already provided this task (or carried from the previous
// turn on a continuation) instead of prompting again.
const cached = loopFiles[fs.key];
try {
const f = cached ?? await requestFile(fs.label);
files[fs.key] = f;
loopFiles[fs.key] = f;
lastFilesRef.current[fs.key] = f;
} catch {
cancelled = true; // user dismissed the file request
break;
// Prefer the previous tool's OUTPUT (chaining), then a file already
// uploaded this task, otherwise ask. Don't cache a chained output as the
// "original upload" — a later "make it smaller" should re-run on the source.
const piped = chainFile;
const cached = piped ?? loopFiles[fs.key];
let f: File;
if (cached) { f = cached; }
else {
try { f = await requestFile(fs.label); }
catch { cancelled = true; break; }
}
files[fs.key] = f;
if (!piped) { loopFiles[fs.key] = f; lastFilesRef.current[fs.key] = f; }
}
if (cancelled) { updateLastText(`✗ ${exec.toolId} — cancelled`); break; }

Expand Down Expand Up @@ -202,6 +212,8 @@ export function useAgentChat(provider: AgentProvider | null) {
});
const blobUrl = result.blob ? URL.createObjectURL(result.blob) : undefined;
push({ role: 'assistant', text: `✓ ${exec.toolId}: ${result.text ?? 'produced a file'}`, blobUrl, imgUrl: result.dataUrl, filename: result.filename });
// Pipe this output into the next tool of the chain.
if (result.blob) chainFile = new File([result.blob], result.filename ?? 'output', { type: result.blob.type });
sessionRef.current = applyResolution(sessionRef.current, { toolId: exec.toolId, params, reply: result.text ?? 'done' });
doneKeys.add(key);
produced = true;
Expand Down
4 changes: 4 additions & 0 deletions src/services/agent/provider.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ function mockFetchOnce(json: unknown, ok = true) {
}

describe('createCloudProvider', () => {
it('is marked capable (cloud models get the full tool catalog + chaining)', () => {
const p = createCloudProvider({ kind: 'openai', baseUrl: 'https://api.example.com/v1', model: 'm', apiKey: 'sk-1' });
expect(p.capable).toBe(true);
});
it('calls the provider directly when not proxied (OpenAI-compatible)', async () => {
const f = mockFetchOnce({ choices: [{ message: { content: 'hi there' } }] });
const p = createCloudProvider({ kind: 'openai', baseUrl: 'https://api.example.com/v1', model: 'm', apiKey: 'sk-1' });
Expand Down
8 changes: 7 additions & 1 deletion src/services/agent/provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,12 @@
*/

export interface ChatMessage { role: 'system' | 'user' | 'assistant'; content: string }
export interface AgentProvider { chat(messages: ChatMessage[]): Promise<string> }
export interface AgentProvider {
chat(messages: ChatMessage[]): Promise<string>;
/** Capable enough to plan over the full tool catalog and chain tools (cloud
* models). Tiny on-device models are NOT — they get a keyword-scoped subset. */
capable?: boolean;
}

/** True when the browser exposes WebGPU (required for the on-device model). */
export function hasWebGPU(): boolean {
Expand Down Expand Up @@ -117,6 +122,7 @@ function providerFetch(proxy: boolean | undefined, target: string, headers: Reco
/** A provider that calls a cloud chat API with the user's key, from the browser. */
export function createCloudProvider(cfg: CloudConfig): AgentProvider {
return {
capable: true,
async chat(messages) {
if (cfg.kind === 'anthropic') {
const system = messages.find(m => m.role === 'system')?.content ?? '';
Expand Down
Loading