feat: expose 45 MCP tools with image, select, dialog and task controls

CHANGELOG.md

@@ -13,6 +13,9 @@ evals.
 
 ### Added
 
+- A shared 45-tool MCP catalogue for stdio and HTTP, including image content,
+  native select options, explicit JavaScript dialog handling, task pause/resume/wait,
+  tabs, network/console observation, storage and file transfers.
 - An `openai-compatible` provider: a base URL, an optional API key, and a model
   name, for any server speaking the OpenAI chat-completions dialect — LM Studio,
   vLLM, llama.cpp, LiteLLM, Together, Groq, Fireworks, or a corporate gateway.

core/src/background/agent-loop.ts

@@ -900,6 +900,8 @@ export async function runTask(task: AgentTask, deps: AgentDeps): Promise<AgentTa
       // ── Critical action confirmation ──
       if (requiresConfirm(toolCall, snapshot, settings)) {
         task.status = 'awaiting_confirm';
+        task.updatedAt = Date.now();
+        await saveTask(task);
         broadcastEvent({ kind: 'task:update', task: maskTaskForLog(task) });
         const ok = await deps.shouldConfirm(task, toolCall);
         if (!ok) {
@@ -913,6 +915,8 @@ export async function runTask(task: AgentTask, deps: AgentDeps): Promise<AgentTa
           break;
         }
         task.status = 'running';
+        task.updatedAt = Date.now();
+        await saveTask(task);
         broadcastEvent({ kind: 'task:update', task: maskTaskForLog(task) });
       }
 

core/src/background/bridge-browser.test.ts

@@ -0,0 +1,89 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+vi.mock('@/lib/messaging', () => ({ ensureContentScript: vi.fn() }));
+import { boundedNumber, evaluateInTab, runTabsRequest, waitInTab } from './bridge-browser';
+
+let event: (source: { tabId: number }, method: string, params?: object) => void;
+const debuggerApi = {
+  attach: vi.fn(), detach: vi.fn(), sendCommand: vi.fn(),
+  onEvent: { addListener: vi.fn((listener) => { event = listener; }), removeListener: vi.fn() },
+};
+const tabs = { get: vi.fn(), query: vi.fn(), create: vi.fn(), update: vi.fn(), remove: vi.fn(), sendMessage: vi.fn() };
+
+beforeEach(() => {
+  vi.resetAllMocks();
+  vi.useFakeTimers();
+  debuggerApi.attach.mockResolvedValue(undefined);
+  debuggerApi.detach.mockResolvedValue(undefined);
+  vi.stubGlobal('chrome', { debugger: debuggerApi, tabs, windows: { update: vi.fn() }, scripting: { executeScript: vi.fn().mockResolvedValue([{ frameId: 0 }]) } });
+});
+afterEach(() => { vi.useRealTimers(); vi.unstubAllGlobals(); });
+
+describe('bridge browser primitives', () => {
+  it('rejects invalid timeouts and identifiers', () => {
+    for (const value of [NaN, Infinity, -1, '10', 1.5]) expect(() => boundedNumber(value, 10, 0, 100)).toThrow();
+  });
+  it('evaluates in the main world with options and always detaches', async () => {
+    debuggerApi.sendCommand.mockResolvedValue({ result: { type: 'number', value: 2 } });
+    expect(await evaluateInTab(7, { expression: '1+1' })).toEqual({ ok: true, type: 'number', value: 2 });
+    expect(debuggerApi.sendCommand).toHaveBeenCalledWith({ tabId: 7 }, 'Runtime.evaluate', expect.objectContaining({ expression: '1+1', awaitPromise: true, returnByValue: true, timeout: 10000 }));
+    expect(debuggerApi.detach).toHaveBeenCalledWith({ tabId: 7 });
+  });
+  it('bounds unresolved promises and releases debugger access', async () => {
+    debuggerApi.sendCommand.mockImplementation(async (_target, method) => {
+      if (method === 'Runtime.evaluate') return new Promise(() => {});
+      return {};
+    });
+    const pending = evaluateInTab(7, { expression: 'new Promise(() => {})', timeoutMs: 100 });
+    await vi.advanceTimersByTimeAsync(100);
+    expect(await pending).toMatchObject({ ok: false, error: expect.stringContaining('timed out after 100ms') });
+    expect(debuggerApi.detach).toHaveBeenCalledOnce();
+  });
+  it('reports JS exceptions and debugger attachment failures', async () => {
+    debuggerApi.sendCommand.mockResolvedValue({ result: {}, exceptionDetails: { text: 'Uncaught', lineNumber: 0, columnNumber: 8, exception: { description: 'SyntaxError' } } });
+    expect(await evaluateInTab(7, { expression: '(' })).toMatchObject({ ok: false, exception: { text: 'SyntaxError', line: 1, column: 9 } });
+    debuggerApi.attach.mockRejectedValue(new Error('Another debugger is attached'));
+    debuggerApi.detach.mockClear();
+    expect(await evaluateInTab(7, { expression: '1' })).toMatchObject({ ok: false, error: expect.stringContaining('DevTools') });
+    expect(debuggerApi.detach).not.toHaveBeenCalled();
+  });
+  it('waits for actual network quiet and ignores other tabs', async () => {
+    const pending = waitInTab(7, { until: 'network_idle', timeoutMs: 2000, pollMs: 50 });
+    await vi.advanceTimersByTimeAsync(0);
+    event({ tabId: 7 }, 'Network.requestWillBeSent', { requestId: 'a' });
+    event({ tabId: 7 }, 'Network.requestWillBeSent', { requestId: 'b' });
+    await vi.advanceTimersByTimeAsync(700);
+    event({ tabId: 8 }, 'Network.loadingFinished', { requestId: 'a' });
+    await vi.advanceTimersByTimeAsync(100);
+    event({ tabId: 7 }, 'Network.loadingFinished', { requestId: 'a' });
+    await vi.advanceTimersByTimeAsync(500);
+    expect(await pending).toMatchObject({ ok: true, elapsedMs: 1300, source: 'network' });
+    expect(debuggerApi.onEvent.removeListener).toHaveBeenCalledWith(event);
+    expect(debuggerApi.detach).toHaveBeenCalledOnce();
+  });
+  it('fails on a busy network timeout instead of claiming success', async () => {
+    const pending = waitInTab(7, { until: 'network_idle', timeoutMs: 600 });
+    await vi.advanceTimersByTimeAsync(0);
+    for (const requestId of ['a', 'b']) event({ tabId: 7 }, 'Network.requestWillBeSent', { requestId });
+    await vi.advanceTimersByTimeAsync(600);
+    expect(await pending).toMatchObject({ ok: false, source: 'network' });
+  });
+  it('labels the DOM fallback and routes refs to the owning frame', async () => {
+    debuggerApi.attach.mockRejectedValue(new Error('Unavailable'));
+    tabs.sendMessage.mockResolvedValue({ kind: 'action:result', result: { ok: true, extracted: { matched: 'dom_quiet' } } });
+    expect(await waitInTab(7, { until: 'network_idle', ref: '@f3e2' })).toMatchObject({ ok: true, source: 'dom', warning: expect.any(String) });
+    expect(tabs.sendMessage).toHaveBeenCalledWith(7, expect.any(Object), { frameId: 3 });
+  });
+  it('polls URL changes and reports a failed content wait', async () => {
+    tabs.get.mockResolvedValueOnce({ url: 'https://example.com/' }).mockResolvedValue({ url: 'https://example.com/search' });
+    const pending = waitInTab(7, { until: 'url', urlPattern: '/search' });
+    await vi.advanceTimersByTimeAsync(150);
+    expect(await pending).toMatchObject({ ok: true, matched: '/search' });
+    tabs.sendMessage.mockResolvedValue({ kind: 'action:result', result: { ok: false, error: 'not found' } });
+    expect(await waitInTab(7, { until: 'text', text: 'hello' })).toMatchObject({ ok: false, error: 'not found' });
+  });
+  it('keeps per-tab close failures and closes each id once', async () => {
+    tabs.remove.mockImplementation(async (id) => { if (id === 8) throw new Error('gone'); });
+    expect(await runTabsRequest('tabs.close', { tabIds: [7, 8, 7] })).toEqual({ closed: [7], errors: [{ tabId: 8, error: 'gone' }] });
+    expect(tabs.remove).toHaveBeenCalledTimes(2);
+  });
+});

core/src/background/bridge-browser.ts

@@ -0,0 +1,167 @@
+import { acquireCdp, cdpContexts } from './cdp-session';
+import { frameIdFromRef } from '@/lib/frames';
+import type { CSResponse, ToolCall } from '@/lib/types';
+import { ensureContentScript } from '@/lib/messaging';
+
+export function boundedNumber(value: unknown, fallback: number, min: number, max: number): number {
+  if (value === undefined) return fallback;
+  if (typeof value !== 'number' || !Number.isInteger(value) || value < min || value > max) {
+    throw new Error(`Expected a number between ${min} and ${max}`);
+  }
+  return value;
+}
+
+type Context = { id: number; auxData?: { isDefault?: boolean } };
+type Evaluation = {
+  result: { type: string; value?: unknown; unserializableValue?: string; objectId?: string; description?: string };
+  exceptionDetails?: { text: string; url?: string; lineNumber: number; columnNumber: number; exception?: { description?: string } };
+};
+
+export async function evaluateInTab(tabId: number, payload: Record<string, unknown>) {
+  if (typeof payload.expression !== 'string' || !payload.expression.trim()) throw new Error('Missing expression');
+  const timeout = boundedNumber(payload.timeoutMs, 10_000, 1, 120_000);
+  const frameId = boundedNumber(payload.frameId, 0, 0, Number.MAX_SAFE_INTEGER);
+  const target = { tabId };
+  const contexts: Context[] = [];
+  const onEvent = (source: chrome.debugger.Debuggee, method: string, params?: object) => {
+    if (source.tabId === tabId && method === 'Runtime.executionContextCreated') contexts.push((params as { context: Context }).context);
+  };
+  let release: (() => Promise<void>) | undefined;
+  let marker: string | undefined;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    if (!chrome.debugger?.attach) throw new Error('Chrome debugger API is unavailable');
+    release = await acquireCdp(tabId);
+    chrome.debugger.onEvent.addListener(onEvent);
+    await chrome.debugger.sendCommand(target, 'Runtime.enable');
+    let contextId: number | undefined;
+    if (frameId !== 0) {
+      // Chrome frame ids and CDP frame ids differ; a temporary DOM marker identifies the exact document.
+      marker = `data-weboperator-eval-${crypto.randomUUID()}`;
+      await chrome.scripting.executeScript({ target: { tabId, frameIds: [frameId] }, func: (key) => document.documentElement.setAttribute(key, ''), args: [marker] });
+      for (const context of [...new Map([...cdpContexts(tabId), ...contexts].map((item) => [item.id, item])).values()].filter((item) => item.auxData?.isDefault)) {
+        try {
+          const probe = await chrome.debugger.sendCommand(target, 'Runtime.evaluate', {
+            expression: `document.documentElement.hasAttribute(${JSON.stringify(marker)})`, contextId: context.id, returnByValue: true,
+          }) as Evaluation;
+          if (probe.result.value === true) { contextId = context.id; break; }
+        } catch { /* A frame can disappear between Runtime.enable and the probe. */ }
+      }
+      if (contextId === undefined) throw new Error(`Frame ${frameId} is unavailable to this debugger session (possibly a cross-process iframe)`);
+    }
+    const response = await Promise.race([chrome.debugger.sendCommand(target, 'Runtime.evaluate', {
+      expression: payload.expression, returnByValue: payload.returnByValue !== false,
+      awaitPromise: payload.awaitPromise !== false, userGesture: true, timeout,
+      ...(contextId === undefined ? {} : { contextId }),
+    }), new Promise<never>((_, reject) => { timer = setTimeout(() => reject(new Error(`Evaluation timed out after ${timeout}ms`)), timeout); })]) as Evaluation;
+    const exception = response.exceptionDetails;
+    if (exception) return { ok: false, exception: { text: exception.exception?.description ?? exception.text, url: exception.url, line: exception.lineNumber + 1, column: exception.columnNumber + 1 } };
+    return { ok: true, ...response.result };
+  } catch (error) {
+    return { ok: false, error: `Browser evaluation failed (check DevTools/debugger access): ${error instanceof Error ? error.message : String(error)}` };
+  } finally {
+    if (timer) clearTimeout(timer);
+    if (marker) await chrome.scripting.executeScript({ target: { tabId, frameIds: [frameId] }, func: (key) => document.documentElement.removeAttribute(key), args: [marker] }).catch(() => {});
+    if (release) {
+      chrome.debugger.onEvent.removeListener(onEvent);
+      await release();
+    }
+  }
+}
+
+async function waitForNetwork(tabId: number, timeoutMs: number, pollMs: number): Promise<boolean> {
+  const target = { tabId };
+  const pending = new Set<string>();
+  let quietSince: number;
+  let release: (() => Promise<void>) | undefined;
+  const onEvent = (source: chrome.debugger.Debuggee, method: string, params?: object) => {
+    if (source.tabId !== tabId) return;
+    const id = (params as { requestId?: string } | undefined)?.requestId;
+    if (!id) return;
+    const wasBusy = pending.size >= 2;
+    if (method === 'Network.requestWillBeSent') pending.add(id);
+    if (method === 'Network.loadingFinished' || method === 'Network.loadingFailed') pending.delete(id);
+    if (pending.size >= 2 || wasBusy) quietSince = Date.now();
+  };
+  try {
+    if (!chrome.debugger?.attach) throw new Error('Debugger unavailable');
+    release = await acquireCdp(tabId);
+    chrome.debugger.onEvent.addListener(onEvent);
+    await chrome.debugger.sendCommand(target, 'Network.enable');
+    const deadline = Date.now() + timeoutMs;
+    quietSince = Date.now();
+    while (Date.now() < deadline) {
+      if (pending.size >= 2) quietSince = Date.now();
+      if (Date.now() - quietSince >= 500) return true;
+      await new Promise((resolve) => setTimeout(resolve, Math.min(pollMs, deadline - Date.now())));
+    }
+    return false;
+  } finally {
+    if (release) {
+      chrome.debugger.onEvent.removeListener(onEvent);
+      await release();
+    }
+  }
+}
+
+export async function waitInTab(tabId: number, payload: Record<string, unknown>) {
+  const started = Date.now();
+  const timeoutMs = boundedNumber(payload.timeoutMs, 15_000, 0, 120_000);
+  const pollMs = boundedNumber(payload.pollMs, 150, 10, 1000);
+  const until = String(payload.until ?? 'load');
+  const result = (ok: boolean, extra: Record<string, unknown> = {}) => ({ ok, elapsedMs: Date.now() - started, ...extra });
+  if (until === 'url') {
+    if (typeof payload.urlPattern !== 'string' || !payload.urlPattern) throw new Error('url wait requires urlPattern (URL substring)');
+    do {
+      if ((await chrome.tabs.get(tabId)).url?.includes(payload.urlPattern)) return result(true, { matched: payload.urlPattern });
+      await new Promise((resolve) => setTimeout(resolve, Math.min(pollMs, Math.max(0, timeoutMs - (Date.now() - started)))));
+    } while (Date.now() - started < timeoutMs);
+    return result(false, { error: `URL did not contain ${payload.urlPattern}` });
+  }
+  let fallback: string | undefined;
+  if (until === 'network_idle') {
+    try {
+      const ok = await waitForNetwork(tabId, timeoutMs, pollMs);
+      return result(ok, { ...(ok ? {} : { error: 'Network did not become idle within timeout' }), source: 'network' });
+    } catch { fallback = 'Debugger unavailable; observed DOM quiet instead of network traffic'; }
+  }
+  if (!['element', 'text', 'load', 'timeout', 'network_idle'].includes(until)) throw new Error(`Unknown wait condition: ${until}`);
+  const remaining = Math.max(0, timeoutMs - (Date.now() - started));
+  const action: ToolCall = { name: 'wait', arguments: { ...payload, until, ms: remaining } };
+  await ensureContentScript(tabId);
+  const response = await chrome.tabs.sendMessage(tabId, { kind: 'action:run', action }, { frameId: typeof payload.frameId === 'number' ? payload.frameId : frameIdFromRef(String(payload.ref ?? '')) }) as CSResponse;
+  if (response.kind !== 'action:result') throw new Error('Unexpected wait response');
+  return result(response.result.ok, { ...(response.result.extracted as Record<string, unknown> | undefined), ...(response.result.error ? { error: response.result.error } : {}), ...(fallback ? { source: 'dom', warning: fallback } : {}) });
+}
+
+export async function runTabsRequest(type: string, payload: Record<string, unknown>) {
+  if (type === 'tabs.list') {
+    const tabs = await chrome.tabs.query({});
+    return { tabs: await Promise.all(tabs.map(async (tab) => {
+      let frameCount: number | undefined;
+      try { frameCount = (await chrome.scripting.executeScript({ target: { tabId: tab.id!, allFrames: true }, func: () => document.readyState })).length; } catch { /* Browser-internal pages cannot be inspected. */ }
+      return { tabId: tab.id, windowId: tab.windowId, url: tab.url, title: tab.title, active: tab.active, status: tab.status, groupId: tab.groupId, frameCount };
+    })) };
+  }
+  if (type === 'tabs.open') {
+    if (typeof payload.url !== 'string' || !payload.url) throw new Error('Missing url');
+    const tab = await chrome.tabs.create({ url: payload.url, active: payload.active !== false, ...(payload.windowId === undefined ? {} : { windowId: boundedNumber(payload.windowId, 0, 1, Number.MAX_SAFE_INTEGER) }) });
+    return { tabId: tab.id };
+  }
+  if (type === 'tabs.activate') {
+    const tabId = boundedNumber(payload.tabId, 0, 1, Number.MAX_SAFE_INTEGER);
+    const tab = await chrome.tabs.update(tabId, { active: true });
+    if (!tab) throw new Error(`Tab not found: ${tabId}`);
+    await chrome.windows.update(tab.windowId, { focused: true });
+    return { ok: true };
+  }
+  if (!Array.isArray(payload.tabIds) || !payload.tabIds.length) throw new Error('tabIds must be a non-empty array');
+  const ids = [...new Set(payload.tabIds.map((id) => boundedNumber(id, 0, 1, Number.MAX_SAFE_INTEGER)))];
+  const closed: number[] = [];
+  const errors: { tabId: number; error: string }[] = [];
+  for (const tabId of ids) {
+    try { await chrome.tabs.remove(tabId); closed.push(tabId); }
+    catch (error) { errors.push({ tabId, error: error instanceof Error ? error.message : String(error) }); }
+  }
+  return { closed, errors };
+}

core/src/background/bridge-confirmation.test.ts

@@ -0,0 +1,23 @@
+import { expect, it, vi } from 'vitest';
+import { confirmBridgeTask } from './bridge-confirmation';
+
+it('resumes an approved pending task and reports its next status', () => {
+  const resolve = vi.fn();
+  expect(confirmBridgeTask({ id: 'task', approve: true }, new Map([['task', { resolve }]]))).toEqual({ ok: true, status: 'running' });
+  expect(resolve).toHaveBeenCalledWith(true);
+});
+it('rejects a pending action and supports legacy allow', () => {
+  const resolve = vi.fn();
+  const pending = new Map([['task', { resolve }]]);
+  expect(confirmBridgeTask({ id: 'task', approve: false }, pending)).toEqual({ ok: true, status: 'failed' });
+  expect(resolve).toHaveBeenCalledWith(false);
+  expect(confirmBridgeTask({ id: 'task', allow: true }, pending).status).toBe('running');
+});
+it('never resolves missing, malformed, or contradictory approvals', () => {
+  const resolve = vi.fn();
+  const pending = new Map([['task', { resolve }]]);
+  for (const payload of [{ id: 'task' }, { id: 'task', approve: 'true' }, { id: 'task', approve: true, allow: false }, { id: 'missing', approve: true }]) {
+    expect(() => confirmBridgeTask(payload, pending)).toThrow();
+  }
+  expect(resolve).not.toHaveBeenCalled();
+});

core/src/background/bridge-confirmation.ts

@@ -0,0 +1,12 @@
+/** Resolve a bridge confirmation without approving absent or malformed decisions. */
+export function confirmBridgeTask(payload: Record<string, unknown>, pendingConfirms: ReadonlyMap<string, { resolve: (approve: boolean) => void }>) {
+  const id = String(payload.id ?? '');
+  if (!id) throw new Error('Missing task id');
+  const approve = payload.approve ?? payload.allow;
+  if (typeof approve !== 'boolean') throw new Error('approve must be a boolean');
+  if (payload.approve !== undefined && payload.allow !== undefined && payload.approve !== payload.allow) throw new Error('approve and allow disagree');
+  const pending = pendingConfirms.get(id);
+  if (!pending) throw new Error(`Task ${id} is not awaiting confirmation`);
+  pending.resolve(approve);
+  return { ok: true, status: approve ? 'running' : 'failed' };
+}