diff --git a/src/tools/agent/executors.test.ts b/src/tools/agent/executors.test.ts index c5bddbb..1b4923a 100644 --- a/src/tools/agent/executors.test.ts +++ b/src/tools/agent/executors.test.ts @@ -74,6 +74,11 @@ describe('executor registry', () => { expect(scopeExecutors('draw a bar chart of my sales').map(e => e.toolId)).toContain('canvas-draw'); expect(scopeExecutors('plot these data points on a canvas').map(e => e.toolId)).toContain('canvas-draw'); }); + it('scopes the data interpreter (peek-data / run-on-data)', () => { + expect(scopeExecutors('preview this csv file').map(e => e.toolId)).toContain('peek-data'); + expect(scopeExecutors('filter rows where amount is over 100 in this csv').map(e => e.toolId)).toContain('run-on-data'); + expect(scopeExecutors('pivot this data by region').map(e => e.toolId)).toContain('run-on-data'); + }); it('does not scope any media compressor for small talk', () => { expect(scopeExecutors('hello how are you today')).toEqual([]); }); diff --git a/src/tools/agent/executors.ts b/src/tools/agent/executors.ts index fbc813b..51eb513 100644 --- a/src/tools/agent/executors.ts +++ b/src/tools/agent/executors.ts @@ -403,6 +403,34 @@ export const AGENT_EXECUTORS: AgentExecutor[] = [ return { text: `Words: ${s.words}\nCharacters: ${s.characters} (${s.charactersNoSpaces} without spaces)\nSentences: ${s.sentences}\nParagraphs: ${s.paragraphs}\nLines: ${s.lines}\nReading time: ~${s.readingMinutes} min` }; }, }, + { + // v3 data interpreter: peek the schema so the model writes a correct transform. + toolId: 'peek-data', page: 'csv-json', + description: "Preview a data file's shape (dimensions + first rows). Call this FIRST when the user wants to clean/filter/reshape a data file, so you can see the columns before writing a transform.", + match: re(/(peek|preview|inspect|look at|show me|what.?s? in).*(data|csv|file|rows?|columns?)|what columns/i), + files: [{ key: 'file', accept: '.csv,.tsv,.txt,.json,text/*', label: 'Data file' }], params: [], + execute: async ({ files }) => { + const { peekData } = await import('@/tools/documents/data-run.lib'); + return { text: peekData(await files.file.text()) }; + }, + }, + { + toolId: 'run-on-data', page: 'code-scratchpad', + description: "Transform a data file by writing JavaScript. YOU write JS in `code` that reads the file's text as `input` and returns the result (assign to `output` or return it). Helpers: parseCSV(text)->rows[][], toCSV(rows)->text, JSON. Runs in a no-network sandbox. Call peek-data first to see the columns. CSV output chains to spreadsheet-convert for Excel.", + match: re(/(clean|filter|dedup|deduplicate|sort|pivot|group|reshape|transform|process|extract|merge|remove|summari[sz]e|aggregate).*(csv|data|rows?|records?|file|columns?|json)|(csv|data|rows?|spreadsheet|json).*(clean|filter|dedup|sort|pivot|group|reshape|transform|process|extract|remove|aggregate)/i), + files: [{ key: 'file', accept: '.csv,.tsv,.txt,.json,text/*', label: 'Data file' }], + params: [{ key: 'code', type: 'string', label: 'JavaScript transform (reads `input`, returns the output text)' }], + execute: async ({ files, params }, onProgress) => { + const { runDataCode } = await import('@/tools/documents/data-run.lib'); + const { extractCode } = await import('@/tools/image/canvas-run.lib'); + const code = extractCode(String(params.code ?? '')); + if (!code) throw new Error('no transform code — write JS that reads `input` and returns the result'); + onProgress?.(0.3); + const out = await runDataCode(code, await files.file.text()); + const isJson = out.trim().startsWith('{') || out.trim().startsWith('['); + return { blob: new Blob([out], { type: isJson ? 'application/json' : 'text/csv' }), filename: isJson ? 'output.json' : 'output.csv', text: `transformed the data (${out.length} chars)` }; + }, + }, { toolId: 'hash-text', description: 'Hash text (SHA-256)', match: re(/\bhash\b|sha-?\d|md5|checksum|digest/i), files: [], params: [{ key: 'text', type: 'string', label: 'Text' }], diff --git a/src/tools/documents/data-run.lib.test.ts b/src/tools/documents/data-run.lib.test.ts new file mode 100644 index 0000000..40ea527 --- /dev/null +++ b/src/tools/documents/data-run.lib.test.ts @@ -0,0 +1,18 @@ +import { describe, it, expect } from 'vitest'; +import { peekData } from './data-run.lib'; + +describe('peekData', () => { + it('reports CSV dimensions and shows the first rows', () => { + const out = peekData('name,qty\nApple,3\nBanana,5\nCherry,2'); + expect(out).toMatch(/4 lines, ~2 columns \(CSV\)/); + expect(out).toContain('name,qty'); + expect(out).toContain('Apple,3'); + }); + it('truncates long inputs with an ellipsis', () => { + const many = Array.from({ length: 40 }, (_, i) => `row${i}`).join('\n'); + const out = peekData(many, 5); + expect(out).toContain('row0'); + expect(out).toContain('…'); + expect(out).not.toContain('row30'); + }); +}); diff --git a/src/tools/documents/data-run.lib.ts b/src/tools/documents/data-run.lib.ts new file mode 100644 index 0000000..746755d --- /dev/null +++ b/src/tools/documents/data-run.lib.ts @@ -0,0 +1,39 @@ +/** + * Host side of the data interpreter. `peekData` builds a small schema sample the + * model reads before writing a transform (pure/testable); `runDataCode` runs the + * code in the sandbox worker with a timeout (needs a real browser). + */ + +/** A compact preview of a file's shape: dimensions + the first rows. */ +export function peekData(text: string, maxLines = 15): string { + const lines = text.replace(/\r\n/g, '\n').split('\n').filter((l, i, a) => i < a.length - 1 || l.length > 0); + const head = lines.slice(0, maxLines); + const looksCsv = /,/.test(head[0] ?? ''); + const cols = looksCsv ? (head[0]?.split(',').length ?? 0) : 0; + const meta = `${lines.length} line${lines.length === 1 ? '' : 's'}` + (looksCsv ? `, ~${cols} columns (CSV)` : ''); + return `${meta}\n\nFirst ${head.length} line${head.length === 1 ? '' : 's'}:\n${head.join('\n')}${lines.length > maxLines ? '\n…' : ''}`; +} + +export interface DataRunOpts { timeoutMs?: number } + +/** Run a data-transform in the sandbox worker; resolve with the output text. */ +export async function runDataCode(code: string, input: string, opts: DataRunOpts = {}): Promise { + if (typeof Worker === 'undefined') throw new Error('this browser cannot run the data sandbox'); + const timeoutMs = opts.timeoutMs ?? 8000; + const worker = new Worker(new URL('./data-sandbox.worker.ts', import.meta.url), { type: 'module' }); + try { + return await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('the transform timed out — possible infinite loop')), timeoutMs); + worker.onmessage = (e: MessageEvent) => { + clearTimeout(timer); + const d = e.data as { ok: boolean; output?: string; error?: string }; + if (d.ok && typeof d.output === 'string') resolve(d.output); + else reject(new Error(d.error || 'transform failed')); + }; + worker.onerror = ev => { clearTimeout(timer); reject(new Error(ev.message || 'sandbox error')); }; + worker.postMessage({ code, input }); + }); + } finally { + worker.terminate(); + } +} diff --git a/src/tools/documents/data-sandbox.worker.ts b/src/tools/documents/data-sandbox.worker.ts new file mode 100644 index 0000000..d340fa9 --- /dev/null +++ b/src/tools/documents/data-sandbox.worker.ts @@ -0,0 +1,48 @@ +/** + * Sandbox for running LLM-written DATA-transform code (agent v3 "data interpreter"). + * + * Same lock-down as the canvas sandbox: a Web Worker (no DOM), with the + * exfiltration/storage channels neutered, and a wall-clock timeout enforced by + * the main thread. The code gets the file's text as `input` plus small CSV/JSON + * helpers, and returns the transformed text. + */ + +const blocked = () => { throw new Error('disabled in the data sandbox'); }; +for (const key of ['fetch', 'XMLHttpRequest', 'WebSocket', 'importScripts', 'indexedDB', 'caches', 'EventSource', 'SharedWorker', 'Worker', 'Notification']) { + try { Object.defineProperty(self, key, { value: blocked, writable: false, configurable: false }); } catch { /* non-configurable */ } +} + +/** Parse a CSV line into fields, honoring double-quoted cells. */ +function splitCsvLine(line: string): string[] { + const out: string[] = []; + let cur = '', inQ = false; + for (let i = 0; i < line.length; i++) { + const c = line[i]; + if (inQ) { + if (c === '"') { if (line[i + 1] === '"') { cur += '"'; i++; } else inQ = false; } + else cur += c; + } else if (c === '"') inQ = true; + else if (c === ',') { out.push(cur); cur = ''; } + else cur += c; + } + out.push(cur); + return out; +} +const parseCSV = (text: string): string[][] => text.replace(/\r\n/g, '\n').replace(/\n+$/, '').split('\n').map(splitCsvLine); +const toCSV = (rows: unknown[][]): string => + rows.map(r => r.map(c => { const s = String(c ?? ''); return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; }).join(',')).join('\n'); + +self.onmessage = async (e: MessageEvent<{ code: string; input: string }>) => { + const { code, input } = e.data; + const post = (msg: unknown) => (self as unknown as Worker).postMessage(msg); + try { + // The code sees `input` (file text) + helpers, and returns the output string. + const fn = new Function('input', 'parseCSV', 'toCSV', 'JSON', 'Math', 'Date', + `"use strict";\nlet output;\n${code}\n;return (typeof output !== 'undefined') ? output : undefined;`); + let out = await fn(input, parseCSV, toCSV, JSON, Math, Date); + if (out === undefined || out === null) throw new Error('the code produced no output — set `output` or return a value'); + post({ ok: true, output: typeof out === 'string' ? out : JSON.stringify(out, null, 2) }); + } catch (err) { + post({ ok: false, error: err instanceof Error ? err.message : String(err) }); + } +};