diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index 1ded647cb..31db1ab1c 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -449,6 +449,8 @@ export class Agent extends LoopDetector { this._richTextToolbarGuard = new RichTextToolbarGuard(); this._richTextToolbarProbe = new RichTextToolbarProbe(this); this._uploadSelectorRecoveryRequired = new Map(); // tabId -> prior ambiguous match count; cleared only by inspection/navigation/cleanup + this._compactUploadTargets = new Map(); // tabId -> { pageUrl, targets: Map(targetId, internal candidate) } + this._compactUploadTargetCounter = 0; // Productive browsing often mixes reads and scrolling, so exact-call loop // detection cannot tell when the agent already has enough evidence to // answer. Track long observation-only streaks and remind it to deliver a @@ -1720,6 +1722,7 @@ export class Agent extends LoopDetector { _clearPageLoopState(tabId) { super._clearPageLoopState(tabId); this._uploadSelectorRecoveryRequired.delete(tabId); + this._compactUploadTargets.delete(tabId); this.deliveryObservationStreaks.delete(tabId); this.bulkApiMutationClicks.delete(tabId); this.bulkApiMutationHints.delete(tabId); @@ -1744,6 +1747,164 @@ export class Agent extends LoopDetector { return true; } + // COMPACT_UPLOAD_TARGET_HELPERS_START + _toolResultTrustName(name, result) { + return name === 'upload_file' && result?.discoveryOnly + ? 'get_file_input_targets' + : name; + } + + async _readCompactUploadFileInputs(tabId) { + const response = await this.executeTool(tabId, 'get_file_input_targets', {}); + if (!Array.isArray(response)) { + return { + ok: false, + error: response?.error || 'Could not inspect this page for file inputs.', + }; + } + const fileInputs = response.filter(element => ( + element?.tag === 'input' + && String(element.type || '').toLowerCase() === 'file' + )); + return { + ok: true, + fileInputs, + usable: fileInputs.filter(element => ( + typeof element.selector === 'string' && element.selector.trim().length > 0 + )), + }; + } + + _compactUploadTargetKey(element) { + return JSON.stringify([ + String(element?.selector || ''), + String(element?.id || ''), + String(element?.name || ''), + element?.accept == null ? null : String(element.accept), + element?.multiple === true, + element?.inShadowDOM === true, + ]); + } + + async _publishCompactUploadTargets(tabId, inventory, prefix = '') { + this._compactUploadTargets.delete(tabId); + if (!inventory?.ok) { + return { + success: false, + dispatched: false, + noDispatch: true, + discoveryOnly: true, + requiresTarget: true, + error: `${prefix}${inventory?.error || 'Could not inspect this page for file inputs.'}`, + }; + } + + const maxCandidates = 12; + const pageUrl = await this._currentUrl(tabId); + const targets = new Map(); + const candidates = inventory.usable.slice(0, maxCandidates).map((element, index) => { + const targetId = `file_target_${(++this._compactUploadTargetCounter).toString(36)}`; + targets.set(targetId, { + selector: element.selector.trim(), + key: this._compactUploadTargetKey(element), + }); + const label = String(element.text || element.name || element.id || `File input ${index + 1}`) + .replace(/[\r\n]+/g, ' ') + .trim() + .slice(0, 100); + return { + targetId, + label: label || `File input ${index + 1}`, + ...(element.name ? { name: String(element.name).slice(0, 100) } : {}), + ...(element.accept != null ? { accept: String(element.accept).slice(0, 200) } : {}), + multiple: element.multiple === true, + inShadowDOM: element.inShadowDOM === true, + }; + }); + + if (targets.size) this._compactUploadTargets.set(tabId, { pageUrl, targets }); + const candidateCount = inventory.fileInputs.length; + const addressableCount = inventory.usable.length; + if (!candidates.length) { + const foundButUnsafe = candidateCount > 0; + return { + success: false, + dispatched: false, + noDispatch: true, + discoveryOnly: true, + requiresTarget: true, + candidateCount, + addressableCount, + candidates: [], + initializerSuggested: !foundButUnsafe, + error: `${prefix}${foundButUnsafe + ? 'File inputs were found, but none had a verified unique target. Re-read the page and expose the intended upload widget before repeating upload_file without targetId.' + : 'No file input is currently available. If the upload widget creates one lazily, make one guarded click on its add-files control, re-read the page, then repeat upload_file without targetId.'}`, + }; + } + + return { + success: false, + dispatched: false, + noDispatch: true, + discoveryOnly: true, + requiresTarget: true, + candidateCount, + addressableCount, + candidates, + truncated: addressableCount > candidates.length, + error: `${prefix}Choose the intended file input from candidates, then retry upload_file with that exact targetId and the same attachmentId if one was provided. Never guess or modify a targetId.`, + }; + } + + async _discoverCompactUploadTargets(tabId, prefix = '') { + return this._publishCompactUploadTargets( + tabId, + await this._readCompactUploadFileInputs(tabId), + prefix, + ); + } + + async _resolveCompactUploadTarget(tabId, targetId) { + const normalizedTargetId = typeof targetId === 'string' ? targetId.trim() : ''; + const state = this._compactUploadTargets.get(tabId); + const saved = normalizedTargetId ? state?.targets?.get(normalizedTargetId) : null; + if (!saved) { + return { + ok: false, + result: await this._discoverCompactUploadTargets( + tabId, + 'That targetId is missing, expired, or was not returned by the latest discovery. ', + ), + }; + } + + const pageUrl = await this._currentUrl(tabId); + const inventory = await this._readCompactUploadFileInputs(tabId); + const current = inventory?.ok + ? inventory.usable.find(element => ( + element.selector.trim() === saved.selector + && this._compactUploadTargetKey(element) === saved.key + )) + : null; + if (this._normalizeUrl(pageUrl) !== this._normalizeUrl(state.pageUrl) || !current) { + return { + ok: false, + result: await this._publishCompactUploadTargets( + tabId, + inventory, + 'The page changed and that targetId expired. ', + ), + }; + } + + // Compact target handles are one-use. A retry must rediscover so a page + // that consumed or replaced the input cannot receive a stale attachment. + this._compactUploadTargets.delete(tabId); + return { ok: true, selector: saved.selector }; + } + // COMPACT_UPLOAD_TARGET_HELPERS_END + _rememberAxScope(tabId, documentToken, pageUrl = '') { const next = { documentToken: String(documentToken || ''), @@ -4780,6 +4941,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d onUpdate, { completionBatchStartState, + promptTier, dispatchBinding: toolbarPreflight.probe?.dispatchBinding || null, iframeTargetUnresolved: toolbarPreflight.iframeTargetUnresolved === true, }, @@ -5190,7 +5352,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // Wrap page-derived results as untrusted DATA BEFORE appending any of // our own trusted notes (the loop nudge), so the nudge stays outside the // box and is read as an instruction, not data. - let resultContent = this._wrapUntrusted(fnName, this._limitToolResult(toolResult)); + const resultTrustName = this._toolResultTrustName(fnName, toolResult); + let resultContent = this._wrapUntrusted(resultTrustName, this._limitToolResult(toolResult)); if (toolResult?.errorCode === 'chrome_protected_page') { resultContent += '\n[TRUSTED RUNTIME ROUTING: Chrome blocks extension DOM/debugger access on this dashboard. Do not call another DOM, accessibility, wait, script, iframe, WebMCP, or upload_file tool here. Continue manually in the dashboard.]'; onUpdate('warning', { message: 'Chrome-protected dashboard detected; DOM automation is unavailable.' }); @@ -17694,7 +17857,46 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (name === 'upload_file') { args = args || {}; - if (this._uploadSelectorRecoveryRequired.has(tabId)) { + const compactUpload = ( + executionContext?.promptTier || this._resolvePromptTier() + ) === 'compact'; + let attachmentPayload = null; + if (compactUpload) { + if ( + args.selector != null + || args.downloadId != null + || args.filePath != null + ) { + return { + success: false, + dispatched: false, + noDispatch: true, + denied: true, + error: 'Compact upload_file accepts only attachmentId and a targetId returned by its own discovery phase. Do not provide selector, downloadId, or filePath.', + }; + } + if (args.attachmentId == null || !String(args.attachmentId).trim()) { + return { + success: false, + dispatched: false, + noDispatch: true, + error: 'Compact Chrome upload_file requires attachmentId from the current user-attachment notice.', + }; + } + const resolvedAttachment = this._resolveUserAttachment(tabId, args.attachmentId); + if (!resolvedAttachment.ok) return { success: false, error: resolvedAttachment.error }; + attachmentPayload = resolvedAttachment; + if (args.targetId == null || !String(args.targetId).trim()) { + return await this._discoverCompactUploadTargets(tabId); + } + const resolvedTarget = await this._resolveCompactUploadTarget(tabId, args.targetId); + if (!resolvedTarget.ok) return resolvedTarget.result; + args = { + attachmentId: String(args.attachmentId), + selector: resolvedTarget.selector, + }; + } + if (!compactUpload && this._uploadSelectorRecoveryRequired.has(tabId)) { return { success: false, dispatched: false, @@ -17705,8 +17907,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d error: 'A previous upload selector matched multiple file inputs. Call get_interactive_elements now and use the exact selector returned on the intended file-input record before retrying upload_file; do not guess another selector variant.', }; } - let attachmentPayload = null; - if (args.attachmentId != null) { + if (!attachmentPayload && args.attachmentId != null) { if (args.downloadId != null || (typeof args.filePath === 'string' && args.filePath.trim())) { return { success: false, error: 'upload_file accepts only one source when attachmentId is used. Remove downloadId/filePath and retry with the current attachmentId.' }; } @@ -17758,9 +17959,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d uploadQuery = await cdpClient.querySelectorPierce(tabId, args.selector); const objectIds = uploadQuery?.objectIds || []; if (objectIds.length === 0) { + if (compactUpload) { + return await this._discoverCompactUploadTargets( + tabId, + 'The selected file input changed before attachment. ', + ); + } return { success: false, error: `File input not found for selector "${args.selector}". Re-inspect the page with get_interactive_elements or get_accessibility_tree to find the real (some upload widgets hide it until you click their "add files" button first).` }; } if (objectIds.length > 1) { + if (compactUpload) { + return await this._discoverCompactUploadTargets( + tabId, + 'The selected file input became ambiguous before attachment. ', + ); + } this._uploadSelectorRecoveryRequired.set(tabId, objectIds.length); return { success: false, @@ -19724,6 +19937,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const actionMap = { 'read_page': 'get_page_info_cdp', 'get_interactive_elements': 'get_interactive_elements_cdp', + // Internal only: Compact upload_file turns these selectors into opaque, + // one-use targetIds before exposing the bounded candidate list. + 'get_file_input_targets': 'get_file_input_targets', // Accessibility-tree path (preferred). Ported from Claude for Chrome — // flat indented text output with persistent WeakRef-backed ref_ids. 'get_accessibility_tree': 'get_accessibility_tree', @@ -19780,6 +19996,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d name === 'type_text' || name === 'type_ax' || name === 'set_field' || name === 'press_keys' || name === 'scroll' || name === 'get_accessibility_tree' || name === 'get_interactive_elements' || + name === 'get_file_input_targets' || name === 'extract_data' || name === 'inspect_element_styles' || name === 'wait_for_element' || name === 'wait_for_stable' || name === 'get_selection' || name === 'find_text' diff --git a/src/chrome/src/agent/permission-gate.js b/src/chrome/src/agent/permission-gate.js index 706f7b713..50808bcad 100644 --- a/src/chrome/src/agent/permission-gate.js +++ b/src/chrome/src/agent/permission-gate.js @@ -59,6 +59,8 @@ export const UNTRUSTED_CONTENT_TOOLS = new Set([ 'read_page', 'get_accessibility_tree', 'get_interactive_elements', + // Hidden Compact-upload discovery returns page-authored file-input labels. + 'get_file_input_targets', 'get_shadow_dom', 'shadow_dom_query', 'get_frames', diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.js index 2d03f4531..e967481fe 100644 --- a/src/chrome/src/agent/tools.js +++ b/src/chrome/src/agent/tools.js @@ -1268,6 +1268,36 @@ const WATCH_BEEP_TOOL = { }, }; +// Compact has no download tools, so the only file it can legitimately reach is +// the one the user attached to this run. Dropping downloadId and filePath is +// therefore not just prompt economy: filePath is a CDP-backed read of any local +// path into an untrusted page's input, and compact omits the full-tier guidance +// that exists to stop the model inventing one. Compact also replaces selector +// with an opaque targetId returned by upload_file's own discovery phase, so a +// small model never has to choose another inspection tool or construct CSS. +const COMPACT_UPLOAD_HIDDEN_PARAMS = ['selector', 'downloadId', 'filePath']; + +function compactUploadFileTool(tool) { + const properties = { ...tool.function.parameters.properties }; + for (const key of COMPACT_UPLOAD_HIDDEN_PARAMS) delete properties[key]; + return { + ...tool, + function: { + ...tool.function, + description: 'Attach the current user-provided file through a two-step Compact workflow. First call with attachmentId only: this is read-only and returns opaque targetId choices for the page\'s file inputs. Then call again with the same attachmentId and one returned targetId. Never invent or modify a targetId. This proves only local page attachment, not remote upload or submission. If no file input exists because a widget creates it lazily, make one guarded click on its add-files control, then repeat the discovery call.', + parameters: { + ...tool.function.parameters, + properties: { + ...properties, + attachmentId: { type: 'string', description: 'Opaque id from the current user-attachment notice. Never guess an id.' }, + targetId: { type: 'string', description: 'Opaque file-input target returned by a prior upload_file discovery call in this run. Never guess or modify it.' }, + }, + required: ['attachmentId'], + }, + }, + }; +} + /** * Get tools filtered by mode. * @@ -1287,7 +1317,9 @@ export function getToolsForMode(mode, opts = {}) { } else if (devCompactBlocked) { base = []; } else if (tier === 'compact') { - base = AGENT_TOOLS.filter(t => COMPACT_TOOL_NAMES.has(t.function.name)); + base = AGENT_TOOLS + .filter(t => COMPACT_TOOL_NAMES.has(t.function.name)) + .map(t => (t.function.name === 'upload_file' ? compactUploadFileTool(t) : t)); } else if (tier === 'mid') { base = AGENT_TOOLS.filter(t => MID_TOOL_NAMES.has(t.function.name)); } else { @@ -1706,6 +1738,7 @@ export const COMPACT_TOOL_NAMES = new Set([ 'click', 'type_text', 'press_keys', 'navigate', 'new_tab', 'wait_for_element', 'fetch_url', + 'upload_file', 'scratchpad_write', 'progress_update', 'progress_read', 'clarify', 'done', ]); @@ -1751,6 +1784,7 @@ TOOLS — use ONLY these: - new_tab({url}): Open a URL in a background tab for user reference. It does not activate or retarget the current run, so never use it as a site-permission workaround. - wait_for_element({selector}): Wait for an element to appear. - fetch_url({url}): Fetch a URL for its content. +- upload_file({attachmentId, targetId?}): Two steps: first call with the current attachmentId only to discover file inputs; then call again with the same attachmentId and one returned targetId. Never guess a targetId. If discovery finds no input because the widget creates it lazily, make one guarded initializer click and repeat discovery. Verify the page shows the attachment before submitting. - scratchpad_write({text}): Save notes that persist across steps. - progress_update({items}) / progress_read({status}): Structured progress ledger for the active repeated item/action task. On GitHub stargazers, only "Follow USER" buttons are follow targets when following is allowed by the task; "Unfollow USER" means skip/already followed unless the ledger shows acted. - done({summary, outcome}): Signal success, partial progress, or a failed blocker. diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js index 6cf1eab38..c759803f1 100644 --- a/src/chrome/src/content/content.js +++ b/src/chrome/src/content/content.js @@ -3011,6 +3011,39 @@ return ''; } + // Internal Compact-upload discovery. This is intentionally not a model + // tool: it returns only file inputs, and the agent replaces each selector + // with a run-scoped opaque targetId before the result reaches the model. + function getFileInputTargets() { + const targets = []; + const seen = new Set(); + const visit = (root, inShadowDOM = false) => { + try { + root.querySelectorAll('input').forEach(el => { + if (!(el instanceof HTMLInputElement) || el.type !== 'file' || seen.has(el)) return; + seen.add(el); + const selector = _uniqueFileInputSelector(el); + targets.push({ + tag: 'input', + type: 'file', + text: _siteInteractionText(el).slice(0, 100), + id: el.id || '', + name: el.name || '', + accept: el.getAttribute('accept'), + multiple: el.hasAttribute('multiple'), + inShadowDOM, + ...(selector ? { selector } : {}), + }); + }); + root.querySelectorAll('*').forEach(host => { + if (host.shadowRoot) visit(host.shadowRoot, true); + }); + } catch {} + }; + visit(document); + return targets; + } + function getInteractiveElementsFull() { return queryInteractiveFull().map((c, i) => { const el = c.el; @@ -4247,6 +4280,7 @@ 'get_page_info_cdp': () => getPageInfoFull(msg.params || {}), 'get_interactive_elements': () => getInteractiveElements(), 'get_interactive_elements_cdp': () => getInteractiveElementsFull(), + 'get_file_input_targets': () => getFileInputTargets(), 'click': () => clickElement(msg.params || {}), 'consume_file_picker_guard': () => consumeFilePickerGuard(msg.params?.guardId), 'type': () => typeText(msg.params || {}), diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index fd4bb974a..ba01ebd85 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -401,6 +401,8 @@ export class Agent extends LoopDetector { this._richTextToolbarGuard = new RichTextToolbarGuard(); this._richTextToolbarProbe = new RichTextToolbarProbe(this); this._uploadSelectorRecoveryRequired = new Map(); // tabId -> prior ambiguous match count; cleared only by inspection/navigation/cleanup + this._compactUploadTargets = new Map(); // tabId -> { pageUrl, targets: Map(targetId, internal candidate) } + this._compactUploadTargetCounter = 0; // Productive browsing often mixes reads and scrolling, so exact-call loop // detection cannot tell when the agent already has enough evidence to // answer. Track long observation-only streaks and remind it to deliver a @@ -1794,6 +1796,7 @@ export class Agent extends LoopDetector { _clearPageLoopState(tabId) { super._clearPageLoopState(tabId); this._uploadSelectorRecoveryRequired.delete(tabId); + this._compactUploadTargets.delete(tabId); this.deliveryObservationStreaks.delete(tabId); this.bulkApiMutationClicks.delete(tabId); this.bulkApiMutationHints.delete(tabId); @@ -1818,6 +1821,164 @@ export class Agent extends LoopDetector { return true; } + // COMPACT_UPLOAD_TARGET_HELPERS_START + _toolResultTrustName(name, result) { + return name === 'upload_file' && result?.discoveryOnly + ? 'get_file_input_targets' + : name; + } + + async _readCompactUploadFileInputs(tabId) { + const response = await this.executeTool(tabId, 'get_file_input_targets', {}); + if (!Array.isArray(response)) { + return { + ok: false, + error: response?.error || 'Could not inspect this page for file inputs.', + }; + } + const fileInputs = response.filter(element => ( + element?.tag === 'input' + && String(element.type || '').toLowerCase() === 'file' + )); + return { + ok: true, + fileInputs, + usable: fileInputs.filter(element => ( + typeof element.selector === 'string' && element.selector.trim().length > 0 + )), + }; + } + + _compactUploadTargetKey(element) { + return JSON.stringify([ + String(element?.selector || ''), + String(element?.id || ''), + String(element?.name || ''), + element?.accept == null ? null : String(element.accept), + element?.multiple === true, + element?.inShadowDOM === true, + ]); + } + + async _publishCompactUploadTargets(tabId, inventory, prefix = '') { + this._compactUploadTargets.delete(tabId); + if (!inventory?.ok) { + return { + success: false, + dispatched: false, + noDispatch: true, + discoveryOnly: true, + requiresTarget: true, + error: `${prefix}${inventory?.error || 'Could not inspect this page for file inputs.'}`, + }; + } + + const maxCandidates = 12; + const pageUrl = await this._currentUrl(tabId); + const targets = new Map(); + const candidates = inventory.usable.slice(0, maxCandidates).map((element, index) => { + const targetId = `file_target_${(++this._compactUploadTargetCounter).toString(36)}`; + targets.set(targetId, { + selector: element.selector.trim(), + key: this._compactUploadTargetKey(element), + }); + const label = String(element.text || element.name || element.id || `File input ${index + 1}`) + .replace(/[\r\n]+/g, ' ') + .trim() + .slice(0, 100); + return { + targetId, + label: label || `File input ${index + 1}`, + ...(element.name ? { name: String(element.name).slice(0, 100) } : {}), + ...(element.accept != null ? { accept: String(element.accept).slice(0, 200) } : {}), + multiple: element.multiple === true, + inShadowDOM: element.inShadowDOM === true, + }; + }); + + if (targets.size) this._compactUploadTargets.set(tabId, { pageUrl, targets }); + const candidateCount = inventory.fileInputs.length; + const addressableCount = inventory.usable.length; + if (!candidates.length) { + const foundButUnsafe = candidateCount > 0; + return { + success: false, + dispatched: false, + noDispatch: true, + discoveryOnly: true, + requiresTarget: true, + candidateCount, + addressableCount, + candidates: [], + initializerSuggested: !foundButUnsafe, + error: `${prefix}${foundButUnsafe + ? 'File inputs were found, but none had a verified unique target. Re-read the page and expose the intended upload widget before repeating upload_file without targetId.' + : 'No file input is currently available. If the upload widget creates one lazily, make one guarded click on its add-files control, re-read the page, then repeat upload_file without targetId.'}`, + }; + } + + return { + success: false, + dispatched: false, + noDispatch: true, + discoveryOnly: true, + requiresTarget: true, + candidateCount, + addressableCount, + candidates, + truncated: addressableCount > candidates.length, + error: `${prefix}Choose the intended file input from candidates, then retry upload_file with that exact targetId and the same attachmentId if one was provided. Never guess or modify a targetId.`, + }; + } + + async _discoverCompactUploadTargets(tabId, prefix = '') { + return this._publishCompactUploadTargets( + tabId, + await this._readCompactUploadFileInputs(tabId), + prefix, + ); + } + + async _resolveCompactUploadTarget(tabId, targetId) { + const normalizedTargetId = typeof targetId === 'string' ? targetId.trim() : ''; + const state = this._compactUploadTargets.get(tabId); + const saved = normalizedTargetId ? state?.targets?.get(normalizedTargetId) : null; + if (!saved) { + return { + ok: false, + result: await this._discoverCompactUploadTargets( + tabId, + 'That targetId is missing, expired, or was not returned by the latest discovery. ', + ), + }; + } + + const pageUrl = await this._currentUrl(tabId); + const inventory = await this._readCompactUploadFileInputs(tabId); + const current = inventory?.ok + ? inventory.usable.find(element => ( + element.selector.trim() === saved.selector + && this._compactUploadTargetKey(element) === saved.key + )) + : null; + if (this._normalizeUrl(pageUrl) !== this._normalizeUrl(state.pageUrl) || !current) { + return { + ok: false, + result: await this._publishCompactUploadTargets( + tabId, + inventory, + 'The page changed and that targetId expired. ', + ), + }; + } + + // Compact target handles are one-use. A retry must rediscover so a page + // that consumed or replaced the input cannot receive a stale attachment. + this._compactUploadTargets.delete(tabId); + return { ok: true, selector: saved.selector }; + } + // COMPACT_UPLOAD_TARGET_HELPERS_END + _rememberAxScope(tabId, documentToken, pageUrl = '') { const next = { documentToken: String(documentToken || ''), @@ -4367,6 +4528,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d onUpdate, { completionBatchStartState, + promptTier, dispatchBinding: toolbarPreflight.probe?.dispatchBinding || null, iframeTargetUnresolved: toolbarPreflight.iframeTargetUnresolved === true, }, @@ -4768,7 +4930,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // Wrap page-derived results as untrusted DATA BEFORE appending any of // our own trusted notes (the loop nudge), so the nudge stays outside the // box and is read as an instruction, not data. - let resultContent = this._wrapUntrusted(fnName, this._limitToolResult(toolResult)); + const resultTrustName = this._toolResultTrustName(fnName, toolResult); + let resultContent = this._wrapUntrusted(resultTrustName, this._limitToolResult(toolResult)); if (captchaGateDecision?.status === 'solve_required') { resultContent += '\n[TRUSTED CAPTCHA GATE: A supported verification challenge is active. Call solve_captcha once now. Do not dismiss or close the dialog, click Continue/Submit, or use another page-changing tool until solve_captcha returns.]'; onUpdate('warning', { message: 'Supported verification challenge detected; solve_captcha is required.' }); @@ -14548,7 +14711,47 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (name === 'upload_file') { const UPLOAD_MAX_BYTES = 25 * 1024 * 1024; try { - if (this._uploadSelectorRecoveryRequired.has(tabId)) { + args = args || {}; + const compactUpload = ( + executionContext?.promptTier || this._resolvePromptTier() + ) === 'compact'; + let compactAttachmentPayload = null; + if (compactUpload) { + if ( + args.selector != null + || args.downloadId != null + || args.filePath != null + ) { + return { + success: false, + dispatched: false, + noDispatch: true, + denied: true, + error: 'Compact upload_file accepts only attachmentId and a targetId returned by its own discovery phase. Do not provide selector, downloadId, or filePath.', + }; + } + if (args.attachmentId != null) { + const resolvedAttachment = this._resolveUserAttachment( + tabId, + args.attachmentId, + UPLOAD_MAX_BYTES, + ); + if (!resolvedAttachment.ok) { + return { success: false, error: resolvedAttachment.error }; + } + compactAttachmentPayload = resolvedAttachment; + } + if (args.targetId == null || !String(args.targetId).trim()) { + return await this._discoverCompactUploadTargets(tabId); + } + const resolvedTarget = await this._resolveCompactUploadTarget(tabId, args.targetId); + if (!resolvedTarget.ok) return resolvedTarget.result; + args = { + ...(args.attachmentId != null ? { attachmentId: String(args.attachmentId) } : {}), + selector: resolvedTarget.selector, + }; + } + if (!compactUpload && this._uploadSelectorRecoveryRequired.has(tabId)) { return { success: false, dispatched: false, @@ -14574,7 +14777,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (args.downloadId != null) { return { success: false, error: 'upload_file accepts only one source. Remove downloadId and retry with the current attachmentId.' }; } - const resolved = this._resolveUserAttachment(tabId, args.attachmentId, UPLOAD_MAX_BYTES); + const resolved = compactAttachmentPayload + || this._resolveUserAttachment(tabId, args.attachmentId, UPLOAD_MAX_BYTES); if (!resolved.ok) return { success: false, error: resolved.error }; ({ base64, filename, mimeType } = resolved); } else if (args.downloadId != null) { @@ -14798,6 +15002,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } const res = results && results[0]; if (!res || !res.success) { + if (compactUpload && res?.dispatched !== true) { + return await this._discoverCompactUploadTargets( + tabId, + 'The selected file input changed before attachment. ', + ); + } if (res?.ambiguous) this._uploadSelectorRecoveryRequired.set(tabId, Number(res.matchCount) || 0); return { success: false, @@ -15691,6 +15901,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const actionMap = { 'read_page': 'get_page_info_cdp', 'get_interactive_elements': 'get_interactive_elements_cdp', + // Internal only: Compact upload_file turns these selectors into opaque, + // one-use targetIds before exposing the bounded candidate list. + 'get_file_input_targets': 'get_file_input_targets', 'get_shadow_dom': 'get_shadow_dom', 'get_frames': 'get_frames', 'click': 'click', @@ -15757,6 +15970,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d name === 'press_keys' || name === 'scroll' || name === 'hover' || name === 'drag_drop' || name === 'get_accessibility_tree' || name === 'get_interactive_elements' || + name === 'get_file_input_targets' || name === 'extract_data' || name === 'inspect_element_styles' || name === 'wait_for_element' || name === 'wait_for_stable' || name === 'get_selection' || name === 'find_text' || name === 'execute_js' diff --git a/src/firefox/src/agent/permission-gate.js b/src/firefox/src/agent/permission-gate.js index a957fb4bc..c55190b7e 100644 --- a/src/firefox/src/agent/permission-gate.js +++ b/src/firefox/src/agent/permission-gate.js @@ -57,6 +57,8 @@ export const UNTRUSTED_CONTENT_TOOLS = new Set([ 'read_page', 'get_accessibility_tree', 'get_interactive_elements', + // Hidden Compact-upload discovery returns page-authored file-input labels. + 'get_file_input_targets', 'get_shadow_dom', 'shadow_dom_query', 'get_frames', diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.js index 7f82baef0..30f6b7124 100644 --- a/src/firefox/src/agent/tools.js +++ b/src/firefox/src/agent/tools.js @@ -995,6 +995,7 @@ export const COMPACT_TOOL_NAMES = new Set([ 'click', 'type_text', 'press_keys', 'navigate', 'new_tab', 'wait_for_element', 'fetch_url', + 'upload_file', 'scratchpad_write', 'progress_update', 'progress_read', 'clarify', 'done', ]); @@ -1116,6 +1117,34 @@ const WATCH_BEEP_TOOL = { }, }; +// Compact has no download tools, so the only file it can legitimately reach is +// the one the user attached to this run, or one the user picks themselves. +// Compact replaces selector with an opaque targetId returned by upload_file's +// own discovery phase, so a small model never has to choose another inspection +// tool or construct CSS. Firefox never had filePath (no CDP). +const COMPACT_UPLOAD_HIDDEN_PARAMS = ['selector', 'downloadId', 'filePath']; + +function compactUploadFileTool(tool) { + const properties = { ...tool.function.parameters.properties }; + for (const key of COMPACT_UPLOAD_HIDDEN_PARAMS) delete properties[key]; + return { + ...tool, + function: { + ...tool.function, + description: 'Attach a file through a two-step Compact workflow. First call without targetId: this is read-only and returns opaque targetId choices for the page\'s file inputs. Then call again with one returned targetId and the current attachmentId, or omit attachmentId on that second call to open WebBrain\'s user-controlled picker. Never invent or modify a targetId. This proves only local page attachment, not remote upload or submission. If no file input exists because a widget creates it lazily, make one guarded click on its add-files control, then repeat the discovery call.', + parameters: { + ...tool.function.parameters, + properties: { + ...properties, + attachmentId: { type: 'string', description: 'Opaque id from the current user-attachment notice. Omit only to ask the user through WebBrain\'s file picker; never guess an id.' }, + targetId: { type: 'string', description: 'Opaque file-input target returned by a prior upload_file discovery call in this run. Never guess or modify it.' }, + }, + required: [], + }, + }, + }; +} + /** * Get tools filtered by mode. * @@ -1134,7 +1163,9 @@ export function getToolsForMode(mode, opts = {}) { } else if (devCompactBlocked) { base = []; } else if (tier === 'compact') { - base = AGENT_TOOLS.filter(t => COMPACT_TOOL_NAMES.has(t.function.name)); + base = AGENT_TOOLS + .filter(t => COMPACT_TOOL_NAMES.has(t.function.name)) + .map(t => (t.function.name === 'upload_file' ? compactUploadFileTool(t) : t)); } else if (tier === 'mid') { base = AGENT_TOOLS.filter(t => MID_TOOL_NAMES.has(t.function.name)); } else { @@ -1234,6 +1265,7 @@ TOOLS - use only these: - new_tab({url}): Open a URL in a background tab for user reference. It does not activate or retarget the current run, so never use it as a site-permission workaround. - wait_for_element({selector}): Wait for an element to appear. - fetch_url({url}): Fetch other URLs for reading only; do not use it to re-read the active tab. +- upload_file({attachmentId?, targetId?}): Two steps: first call without targetId to discover file inputs; then call again with one returned targetId and the current attachmentId, or omit attachmentId on the second call for WebBrain's picker. Never guess a targetId. If discovery finds no input because the widget creates it lazily, make one guarded initializer click and repeat discovery. Verify the page shows the attachment before submitting. - scratchpad_write({text}): Save notes that persist across steps. - progress_update({items}) / progress_read({status}): Structured progress ledger for the active repeated item/action task. On GitHub stargazers, only "Follow USER" buttons are follow targets when following is allowed by the task; "Unfollow USER" means skip/already followed unless the ledger shows acted. - clarify({question, options?}): Ask the user only when materially blocked or ambiguous. Unanswered clarifies auto-select options[0] after timeout (source=timeout is not user approval for high-risk steps; source=auto Instant is intentional auto-approve). diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js index 88579d1fc..121081928 100644 --- a/src/firefox/src/content/content.js +++ b/src/firefox/src/content/content.js @@ -960,6 +960,39 @@ return ''; } + // Internal Compact-upload discovery. This is intentionally not a model + // tool: it returns only file inputs, and the agent replaces each selector + // with a run-scoped opaque targetId before the result reaches the model. + function getFileInputTargets() { + const targets = []; + const seen = new Set(); + const visit = (root, inShadowDOM = false) => { + try { + root.querySelectorAll('input').forEach(el => { + if (!(el instanceof HTMLInputElement) || el.type !== 'file' || seen.has(el)) return; + seen.add(el); + const selector = _uniqueFileInputSelector(el); + targets.push({ + tag: 'input', + type: 'file', + text: _siteInteractionText(el).slice(0, 100), + id: el.id || '', + name: el.name || '', + accept: el.getAttribute('accept'), + multiple: el.hasAttribute('multiple'), + inShadowDOM, + ...(selector ? { selector } : {}), + }); + }); + root.querySelectorAll('*').forEach(host => { + if (host.shadowRoot) visit(host.shadowRoot, true); + }); + } catch {} + }; + visit(document); + return targets; + } + window.__wb_resolve_click_target_for_submit_probe = function resolveClickTargetForSubmitProbe(params = {}) { if (params?.index == null) return null; const index = Number(params.index); @@ -3491,6 +3524,7 @@ 'get_page_info_cdp': () => getPageInfoFull(msg.params || {}), 'get_interactive_elements': () => getInteractiveElements(), 'get_interactive_elements_cdp': () => getInteractiveElementsFull(), + 'get_file_input_targets': () => getFileInputTargets(), 'click': () => clickElement(msg.params || {}), 'consume_file_picker_guard': () => consumeFilePickerGuard(msg.params?.guardId), 'type': () => typeText(msg.params || {}), diff --git a/test/run.js b/test/run.js index ad498c6cb..a6e6f4ea9 100644 --- a/test/run.js +++ b/test/run.js @@ -12839,6 +12839,7 @@ test('getToolsForMode: compact mode restricts act tools in both browsers', () => [...compactNames].sort(), ); assert.ok(compactNamesActual.includes('done'), `[${label}] compact mode must keep done`); + assert.ok(compactNamesActual.includes('upload_file'), `[${label}] compact mode must expose upload_file`); for (const excluded of ['resize_window', 'download_social_media', 'solve_captcha']) { assert.equal(compactNamesActual.includes(excluded), false, `[${label}] compact mode must omit ${excluded}`); } @@ -12846,6 +12847,183 @@ test('getToolsForMode: compact mode restricts act tools in both browsers', () => } }); +test('compact Act exposes a self-targeting upload workflow without a general selector tool', () => { + for (const [label, getTools, prompt] of [ + ['chrome', getToolsForModeCh, SYSTEM_PROMPT_ACT_COMPACT_CH], + ['firefox', getToolsForModeFx, SYSTEM_PROMPT_ACT_COMPACT_FX], + ]) { + const askNames = getTools('ask').map(tool => tool.function.name); + const compactTools = getTools('act', { tier: 'compact' }); + const compactNames = compactTools.map(tool => tool.function.name); + const upload = compactTools.find(tool => tool.function.name === 'upload_file'); + const fullUpload = getTools('act').find(tool => tool.function.name === 'upload_file'); + + assert.ok(upload, `[${label}] compact Act must expose upload_file`); + assert.equal(askNames.includes('upload_file'), false, `[${label}] Ask must remain read-only`); + for (const unavailable of [ + 'download_files', + 'download_resource_from_page', + 'list_downloads', + 'read_downloaded_file', + ]) { + assert.equal(compactNames.includes(unavailable), false, `[${label}] compact Act exposed ${unavailable}`); + } + assert.match(upload.function.description, /two-step Compact workflow/i); + assert.match(upload.function.description, /read-only/i); + assert.match(upload.function.description, /targetId choices/i); + assert.match(upload.function.description, /Never invent or modify a targetId/i); + assert.match(upload.function.description, /one guarded click/i); + assert.doesNotMatch(upload.function.description, /download_files|list_downloads|downloadId/i); + assert.doesNotMatch(upload.function.description, /get_interactive_elements|CSS selector/i); + assert.ok(upload.function.parameters.properties.attachmentId, `[${label}] compact upload must accept attachmentId`); + assert.ok(upload.function.parameters.properties.targetId, `[${label}] compact upload must accept its own targetId`); + assert.ok(fullUpload.function.parameters.properties.downloadId, `[${label}] full upload must retain downloadId`); + assert.ok(fullUpload.function.parameters.properties.selector, `[${label}] full upload must retain selector`); + assert.equal(fullUpload.function.parameters.properties.targetId, undefined, `[${label}] full upload must not gain compact targetId`); + + // Compact can reach only the file the user attached to this run: it has no + // download tools to produce a downloadId, and no way to learn a local path + // that the model did not invent. + for (const hidden of ['selector', 'downloadId', 'filePath']) { + assert.equal( + upload.function.parameters.properties[hidden], + undefined, + `[${label}] compact upload must hide ${hidden}`, + ); + } + assert.deepEqual( + Object.keys(upload.function.parameters.properties).sort(), + ['attachmentId', 'targetId'], + `[${label}] compact upload must expose exactly attachmentId + targetId`, + ); + assert.equal( + compactNames.includes('get_interactive_elements'), + false, + `[${label}] compact upload discovery must not expose the general selector tool`, + ); + + // Assert on the prompt's own upload_file bullet rather than on character + // distances, so rewording the neighbouring bullets cannot break these. + const uploadLine = prompt.split('\n').find(line => line.startsWith('- upload_file(')); + assert.ok(uploadLine, `[${label}] compact prompt must document upload_file`); + assert.match(uploadLine, /Two steps/i); + assert.match(uploadLine, /targetId/i); + assert.match(uploadLine, /Never guess a targetId/i); + assert.match(uploadLine, /one guarded initializer click/i); + assert.doesNotMatch(uploadLine, /selector|get_interactive_elements/i); + assert.equal( + prompt.split('\n').some(line => line.startsWith('- get_interactive_elements')), + false, + `[${label}] compact prompt must not advertise the removed general selector tool`, + ); + assert.doesNotMatch(prompt, /download_files|list_downloads|downloadId/i); + + if (label === 'firefox') { + assert.match(upload.function.description, /user-controlled picker/i); + assert.deepEqual(upload.function.parameters.required, []); + } else { + assert.deepEqual(upload.function.parameters.required, ['attachmentId']); + } + } +}); + +test('upload targeting is tier-scoped: compact self-discovers while mid/full keep selectors', () => { + for (const [label, getTools] of [['chrome', getToolsForModeCh], ['firefox', getToolsForModeFx]]) { + const compact = getTools('act', { tier: 'compact' }); + const compactNames = new Set(compact.map(tool => tool.function.name)); + assert.equal(compactNames.has('upload_file'), true, `[${label}] compact must keep upload_file`); + assert.equal(compactNames.has('get_interactive_elements'), false, `[${label}] compact must avoid the overlapping inspection tool`); + + for (const tier of ['mid', 'full']) { + const tools = getTools('act', { tier }); + const names = new Set(tools.map(tool => tool.function.name)); + const upload = tools.find(tool => tool.function.name === 'upload_file'); + assert.equal(names.has('upload_file'), true, `[${label}] act/${tier} must keep upload_file`); + assert.equal(names.has('get_interactive_elements'), true, `[${label}] act/${tier} must keep selector recovery`); + assert.ok(upload.function.parameters.properties.selector, `[${label}] act/${tier} lost selector`); + assert.equal(upload.function.parameters.properties.targetId, undefined, `[${label}] act/${tier} gained compact targetId`); + assert.deepEqual(upload.function.parameters.required, ['selector'], `[${label}] act/${tier} selector contract changed`); + } + } +}); + +test('Compact upload discovery uses a hidden file-input-only page action', async () => { + for (const [label, AgentClass, getTools, untrustedTools] of [ + ['chrome', AgentCh, getToolsForModeCh, UNTRUSTED_CONTENT_TOOLS_CH], + ['firefox', AgentFx, getToolsForModeFx, UNTRUSTED_CONTENT_TOOLS], + ]) { + const agent = Object.create(AgentClass.prototype); + let invocation = null; + agent.executeTool = async (tabId, name, args) => { + invocation = { tabId, name, args }; + return [ + { tag: 'input', type: 'file', selector: '#avatar' }, + { tag: 'button', type: 'button', selector: '#submit' }, + ]; + }; + + const inventory = await agent._readCompactUploadFileInputs(42); + assert.deepEqual(invocation, { + tabId: 42, + name: 'get_file_input_targets', + args: {}, + }, `[${label}] Compact discovery must call the narrow internal action`); + assert.equal(inventory.ok, true); + assert.deepEqual(inventory.fileInputs, [{ tag: 'input', type: 'file', selector: '#avatar' }]); + assert.deepEqual(inventory.usable, [{ tag: 'input', type: 'file', selector: '#avatar' }]); + assert.equal(untrustedTools.has('get_file_input_targets'), true); + assert.equal( + agent._toolResultTrustName('upload_file', { discoveryOnly: true }), + 'get_file_input_targets', + `[${label}] page-derived discovery labels must use the untrusted boundary`, + ); + assert.equal( + agent._toolResultTrustName('upload_file', { success: true }), + 'upload_file', + `[${label}] ordinary Mid/Full upload results must retain their existing trust classification`, + ); + + for (const [mode, tiers] of [['ask', ['full']], ['act', ['compact', 'mid', 'full']], ['dev', ['compact', 'mid', 'full']]]) { + for (const tier of tiers) { + const names = getTools(mode, { tier }).map(tool => tool.function.name); + assert.equal( + names.includes('get_file_input_targets'), + false, + `[${label}] ${mode}/${tier} must not expose the internal discovery action`, + ); + } + } + } +}); + +test('Compact upload target helpers stay byte-identical across Chrome and Firefox', () => { + const startMarker = ' // COMPACT_UPLOAD_TARGET_HELPERS_START'; + const endMarker = ' // COMPACT_UPLOAD_TARGET_HELPERS_END'; + const helperBlock = (browser) => { + const source = fs.readFileSync(path.join(ROOT, `src/${browser}/src/agent/agent.js`), 'utf8'); + const start = source.indexOf(startMarker); + const end = source.indexOf(endMarker, start + startMarker.length); + assert.ok(start >= 0 && end > start, `${browser}: Compact upload helper parity markers are missing or reversed`); + assert.equal( + source.indexOf(startMarker, start + startMarker.length), + -1, + `${browser}: Compact upload helper start marker must be unique`, + ); + assert.equal( + source.indexOf(endMarker, end + endMarker.length), + -1, + `${browser}: Compact upload helper end marker must be unique`, + ); + return source.slice(start, end + endMarker.length); + }; + + assert.equal( + helperBlock('chrome'), + helperBlock('firefox'), + 'Chrome and Firefox Compact upload target helpers must remain byte-identical', + ); +}); + test('getToolsForMode: mode/tier redesign exposes the intended normal and Dev tools', () => { for (const [label, getTools] of [ ['chrome', getToolsForModeCh], @@ -52350,7 +52528,7 @@ test('user attachment upload guidance follows the active tier tool catalog', () ]); for (const [mode, tier, shouldAdvertiseUpload] of [ - ['act', 'compact', false], + ['act', 'compact', true], ['act', 'mid', true], ['act', 'full', true], ['ask', 'full', false], @@ -52819,7 +52997,7 @@ test('upload_file prefers a valid downloadId and falls back to filePath for an i const agent = new AgentCh({}); const args = { selector: 'input[type=file]', downloadId: 9123, filePath: stalePath }; - const result = await agent.executeTool(42, 'upload_file', args); + const result = await agent.executeTool(42, 'upload_file', args, null, { promptTier: 'mid' }); assert.equal(result.success, true); assert.equal(result.file, realPath); @@ -52906,6 +53084,20 @@ test('upload_file prefers a valid downloadId and falls back to filePath for an i true, ); assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false); + + // The retry the recovery exists to enable: once the inspection has supplied + // a unique selector, the corrected upload must actually dispatch. Without + // this the latch could clear and still leave uploads wedged. + selectorMatches = ['input-501']; + const recoveredRetry = await agent.executeTool(42, 'upload_file', { + selector: 'input[type=file]:not([accept])', + downloadId: 9123, + }); + assert.equal(recoveredRetry.success, true, 'a corrected selector must upload after recovery'); + assert.equal(recoveredRetry.file, realPath); + assert.equal(recoveredRetry.attachmentState, 'input_attached'); + assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false, 'a successful retry must leave the latch clear'); + agent._uploadSelectorRecoveryRequired.set(42, 2); agent._clearRunLoopState(42); assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false, 'run cleanup must clear upload recovery'); @@ -52914,8 +53106,8 @@ test('upload_file prefers a valid downloadId and falls back to filePath for an i assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false, 'navigation cleanup must clear upload recovery'); assert.deepEqual( releasedGroups, - ['upload-query-1', 'upload-query-2', 'upload-query-3', 'upload-query-4'], - 'early upload failures must release selector handles', + ['upload-query-1', 'upload-query-2', 'upload-query-3', 'upload-query-4', 'upload-query-5'], + 'early upload failures and the post-recovery retry must release selector handles', ); } finally { if (originalChrome === undefined) delete globalThis.chrome; @@ -52924,6 +53116,197 @@ test('upload_file prefers a valid downloadId and falls back to filePath for an i } }); +test('Compact Chrome upload_file discovers opaque targets, rejects hidden full-tier inputs, and attaches on retry', async () => { + const originalCdp = { + attach: cdpClientCh.attach, + querySelectorPierce: cdpClientCh.querySelectorPierce, + releaseObjectGroup: cdpClientCh.releaseObjectGroup, + setFileInputData: cdpClientCh.setFileInputData, + getFileInputFiles: cdpClientCh.getFileInputFiles, + }; + let cdpQueries = 0; + let attachedPayload = null; + try { + cdpClientCh.attach = async () => ({ attached: true }); + cdpClientCh.querySelectorPierce = async (_tabId, selector) => { + cdpQueries++; + assert.equal(selector, '#resume-upload', 'the model-facing targetId must resolve to the internal verified selector'); + return { objectIds: ['input-501'], objectGroup: 'compact-upload-query' }; + }; + cdpClientCh.releaseObjectGroup = async () => {}; + cdpClientCh.setFileInputData = async (_tabId, objectId, payload) => { + assert.equal(objectId, 'input-501'); + attachedPayload = payload; + return { success: true, dispatched: true }; + }; + cdpClientCh.getFileInputFiles = async () => [{ name: 'resume.pdf', size: 42 }]; + + const agent = new AgentCh({}); + agent._currentUrl = async () => 'https://example.com/apply'; + agent._resolveUserAttachment = (_tabId, attachmentId) => ({ + ok: true, + attachmentId, + filename: 'resume.pdf', + mimeType: 'application/pdf', + base64: 'JVBERi0=', + }); + agent._readCompactUploadFileInputs = async () => ({ + ok: true, + fileInputs: [ + { + tag: 'input', + type: 'file', + selector: '#resume-upload', + text: 'Resume', + name: 'resume', + accept: '.pdf', + multiple: false, + inShadowDOM: false, + }, + ], + usable: [ + { + tag: 'input', + type: 'file', + selector: '#resume-upload', + text: 'Resume', + name: 'resume', + accept: '.pdf', + multiple: false, + inShadowDOM: false, + }, + ], + }); + + const denied = await agent.executeTool(42, 'upload_file', { + attachmentId: 'att_1', + selector: '#resume-upload', + }, null, { promptTier: 'compact' }); + assert.equal(denied.denied, true); + assert.equal(denied.noDispatch, true); + assert.match(denied.error, /only attachmentId and a targetId/i); + + const discovery = await agent.executeTool(42, 'upload_file', { + attachmentId: 'att_1', + }, null, { promptTier: 'compact' }); + assert.equal(discovery.success, false); + assert.equal(discovery.discoveryOnly, true); + assert.equal(discovery.requiresTarget, true); + assert.equal(discovery.dispatched, false); + assert.equal(discovery.candidates.length, 1); + assert.equal(discovery.candidates[0].label, 'Resume'); + assert.equal(discovery.candidates[0].accept, '.pdf'); + assert.equal(typeof discovery.candidates[0].targetId, 'string'); + assert.equal('selector' in discovery.candidates[0], false, 'Compact must never expose CSS to the model'); + assert.equal(cdpQueries, 0, 'discovery must not attach or query through the upload mutation path'); + + const result = await agent.executeTool(42, 'upload_file', { + attachmentId: 'att_1', + targetId: discovery.candidates[0].targetId, + }, null, { promptTier: 'compact' }); + assert.equal(result.success, true); + assert.equal(result.file, 'resume.pdf'); + assert.equal(result.attachmentState, 'input_attached'); + assert.equal(cdpQueries, 1); + assert.equal(attachedPayload.filename, 'resume.pdf'); + assert.equal(agent._compactUploadTargets.has(42), false, 'targetId must be one-use'); + + const stale = await agent.executeTool(42, 'upload_file', { + attachmentId: 'att_1', + targetId: discovery.candidates[0].targetId, + }, null, { promptTier: 'compact' }); + assert.equal(stale.discoveryOnly, true); + assert.match(stale.error, /missing, expired/i); + assert.equal(cdpQueries, 1, 'a stale targetId must fail before attachment'); + + agent._clearPageLoopState(42); + assert.equal(agent._compactUploadTargets.has(42), false, 'navigation cleanup must clear compact targets'); + } finally { + Object.assign(cdpClientCh, originalCdp); + } +}); + +test('Compact Firefox upload_file discovers before opening its picker and attaches only to a returned targetId', async () => { + const originalBrowser = globalThis.browser; + const scripts = []; + let pickerEvent = null; + try { + globalThis.browser = { + tabs: { + async executeScript(_tabId, details) { + scripts.push(details.code); + if (details.code.includes('WebBrain file attachment settle probe')) { + return [{ attachmentState: 'input_attached' }]; + } + return [{ success: true, dispatched: true, file: 'resume.pdf', size: 4, attachmentState: 'input_attached' }]; + }, + }, + }; + + const candidate = { + tag: 'input', + type: 'file', + selector: '#resume-upload', + text: 'Resume', + name: 'resume', + accept: '.pdf', + multiple: false, + inShadowDOM: false, + }; + const agent = new AgentFx({}); + agent._currentUrl = async () => 'https://example.com/apply'; + agent._readCompactUploadFileInputs = async () => ({ + ok: true, + fileInputs: [candidate], + usable: [candidate], + }); + + const denied = await agent.executeTool(42, 'upload_file', { + selector: '#resume-upload', + }, null, { promptTier: 'compact' }); + assert.equal(denied.denied, true); + assert.equal(denied.noDispatch, true); + + const discovery = await agent.executeTool( + 42, + 'upload_file', + {}, + (evt, data) => { if (evt === 'upload_picker') pickerEvent = data; }, + { promptTier: 'compact' }, + ); + assert.equal(discovery.discoveryOnly, true); + assert.equal(discovery.candidates.length, 1); + assert.equal(pickerEvent, null, 'read-only discovery must not open Firefox\'s picker'); + assert.equal(scripts.length, 0, 'discovery must not inject attachment code'); + + const uploadPromise = agent.executeTool( + 42, + 'upload_file', + { targetId: discovery.candidates[0].targetId }, + (evt, data) => { if (evt === 'upload_picker') pickerEvent = data; }, + { promptTier: 'compact' }, + ); + await new Promise(resolve => setTimeout(resolve, 10)); + assert.ok(pickerEvent?.pickerId, 'the picker should open only after a valid targetId is selected'); + agent.submitUploadPickerResponse(42, pickerEvent.pickerId, { + base64: 'JVBERg==', + name: 'resume.pdf', + type: 'application/pdf', + size: 4, + }); + const result = await uploadPromise; + assert.equal(result.success, true); + assert.equal(result.file, 'resume.pdf'); + assert.equal(result.attachmentState, 'input_attached'); + assert.equal(scripts.length, 2); + assert.match(scripts[0], /const selector = "#resume-upload"/); + assert.equal(agent._compactUploadTargets.has(42), false, 'targetId must be one-use'); + } finally { + if (originalBrowser === undefined) delete globalThis.browser; + else globalThis.browser = originalBrowser; + } +}); + test('upload_file schema accepts downloadId and no longer hard-requires filePath (firefox)', () => { const tools = getToolsForModeFx('act', {}); const up = tools.find(t => t.function?.name === 'upload_file'); @@ -52986,7 +53369,7 @@ test('Firefox upload_file injects the exact user attachment bytes without re-fet const result = await agent.executeTool(42, 'upload_file', { selector: 'input[type=file]', attachmentId, - }); + }, null, { promptTier: 'mid' }); assert.equal(result.success, true); assert.equal(result.attachmentId, attachmentId);