From cdbb022e86798d1311ce3368c79b46ac4213b868 Mon Sep 17 00:00:00 2001 From: waterdoog Date: Sat, 31 Oct 2026 11:36:39 +0800 Subject: [PATCH] feat(mcp): preserve bounded inline image tool results Let hosted desktop tools return the actual screenshot to native MCP harnesses while retaining the text summary. Validate inline formats and bounds and exclude failed results; metadata remains a presentation hint, not authority. --- docs/api/sharedos-mcp.md | 9 +++++ packages/mcp/README.md | 9 +++++ packages/mcp/src/image-results.test.ts | 46 ++++++++++++++++++++++++++ packages/mcp/src/server.ts | 32 +++++++++++++++++- 4 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 packages/mcp/src/image-results.test.ts diff --git a/docs/api/sharedos-mcp.md b/docs/api/sharedos-mcp.md index e36f50b3..dc4420b3 100644 --- a/docs/api/sharedos-mcp.md +++ b/docs/api/sharedos-mcp.md @@ -102,6 +102,15 @@ and [ADR 0014](https://github.com/systemind-team/SharedOS/blob/main/docs/adr/001 SharedOS is currently a `1.0.0` preview. +## Image results + +A successful tool may include `metadata["sharedos/content"]` with inline MCP +image blocks (`type: "image"`, `mimeType`, base64 `data`). The MCP adapter retains +the ordinary JSON/text summary and appends valid JPEG, PNG or WebP images, bounded +to four images and 4 MiB of encoded data per result. Other metadata, remote URLs, +and images on failed or denied results are not forwarded as content. This is a +presentation hint; it grants no authority and does not change tool discovery. + ## Classes ### McpToolServer diff --git a/packages/mcp/README.md b/packages/mcp/README.md index 27a6a7b9..ee77ac99 100644 --- a/packages/mcp/README.md +++ b/packages/mcp/README.md @@ -95,3 +95,12 @@ Full design: [MCP toolshare](https://github.com/systemind-team/SharedOS/blob/mai and [ADR 0014](https://github.com/systemind-team/SharedOS/blob/main/docs/adr/0014-mcp-toolshare.md). SharedOS is currently a `1.0.0` preview. + +## Image results + +A successful tool may include `metadata["sharedos/content"]` with inline MCP +image blocks (`type: "image"`, `mimeType`, base64 `data`). The MCP adapter retains +the ordinary JSON/text summary and appends valid JPEG, PNG or WebP images, bounded +to four images and 4 MiB of encoded data per result. Other metadata, remote URLs, +and images on failed or denied results are not forwarded as content. This is a +presentation hint; it grants no authority and does not change tool discovery. diff --git a/packages/mcp/src/image-results.test.ts b/packages/mcp/src/image-results.test.ts new file mode 100644 index 00000000..c2085af7 --- /dev/null +++ b/packages/mcp/src/image-results.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { toCallToolResult } from "./server.js"; +import type { ToolResult } from "@aicoo/sharedos-contracts"; + +const result = (content: unknown): ToolResult => ({ + callId: "call", + tool: "screen", + completedAt: "2026-10-31T00:00:00Z", + status: "succeeded", + output: { epoch: 3 }, + metadata: { "sharedos/content": content } as ToolResult["metadata"], +}); +describe("image presentation through the authorized MCP result", () => { + it("preserves structured output and exposes an image to the harness", () => { + const image = { type: "image", mimeType: "image/png", data: "aGVsbG8=" }; + expect(toCallToolResult(result([image])).content).toEqual([ + { type: "text", text: '{"epoch":3}' }, + image, + ]); + }); + it("does not fetch URLs or pass unsupported metadata content", () => { + expect( + toCallToolResult( + result([ + { type: "resource_link", uri: "https://example.com" }, + { type: "image", mimeType: "text/html", data: "aGVsbG8=" }, + ]), + ).content, + ).toHaveLength(1); + }); + it("bounds image size and rejects malformed base64", () => { + for (const data of ["!not-base64", "A".repeat(4 * 1024 * 1024 + 4)]) + expect( + toCallToolResult(result([{ type: "image", mimeType: "image/png", data }])).content, + ).toHaveLength(1); + }); + it("never exposes presentation hints from denied results", () => { + const denied = { + ...result([{ type: "image", mimeType: "image/png", data: "aGVsbG8=" }]), + status: "denied", + error: { code: "denied", message: "No grant" }, + } as ToolResult; + expect(toCallToolResult(denied).content).toHaveLength(1); + expect(toCallToolResult(denied).isError).toBe(true); + }); +}); diff --git a/packages/mcp/src/server.ts b/packages/mcp/src/server.ts index 599f4917..acdfde79 100644 --- a/packages/mcp/src/server.ts +++ b/packages/mcp/src/server.ts @@ -338,7 +338,10 @@ export function toCallToolResult( ? { structuredContent: result.output } : {}; return { - content: [{ type: "text", text: renderText(result.output) }], + content: [ + { type: "text", text: renderText(result.output) }, + ...imageContent(result.metadata?.["sharedos/content"]), + ], ...structured, isError: false, }; @@ -357,6 +360,33 @@ export function toCallToolResult( }; } +/** Optional, host-neutral presentation hints. Output remains the structured + * result. Only bounded inline images are recognized: no URLs are fetched and + * arbitrary metadata cannot introduce executable or remote resources. + */ +function imageContent(value: unknown): JsonObject[] { + if (!Array.isArray(value) || value.length > 4) return []; + const images: JsonObject[] = []; + let bytes = 0; + for (const raw of value) { + if (!raw || typeof raw !== "object") continue; + const { type, data, mimeType } = raw as Record; + if ( + type !== "image" || + typeof data !== "string" || + !["image/png", "image/jpeg", "image/webp"].includes(String(mimeType)) || + data.length === 0 || + data.length % 4 !== 0 || + !/^[A-Za-z0-9+/]*={0,2}$/.test(data) + ) + continue; + bytes += data.length; + if (bytes > 4 * 1024 * 1024) return []; + images.push({ type: "image", data, mimeType: String(mimeType) }); + } + return images; +} + function renderText(output: JsonValue): string { return typeof output === "string" ? output : JSON.stringify(output); }