diff --git a/docs/api/sharedos-mcp.md b/docs/api/sharedos-mcp.md index e36f50b3..dc4420b3 100644 --- a/docs/api/sharedos-mcp.md +++ b/docs/api/sharedos-mcp.md @@ -102,6 +102,15 @@ and [ADR 0014](https://github.com/systemind-team/SharedOS/blob/main/docs/adr/001 SharedOS is currently a `1.0.0` preview. +## Image results + +A successful tool may include `metadata["sharedos/content"]` with inline MCP +image blocks (`type: "image"`, `mimeType`, base64 `data`). The MCP adapter retains +the ordinary JSON/text summary and appends valid JPEG, PNG or WebP images, bounded +to four images and 4 MiB of encoded data per result. Other metadata, remote URLs, +and images on failed or denied results are not forwarded as content. This is a +presentation hint; it grants no authority and does not change tool discovery. + ## Classes ### McpToolServer diff --git a/packages/mcp/README.md b/packages/mcp/README.md index 27a6a7b9..ee77ac99 100644 --- a/packages/mcp/README.md +++ b/packages/mcp/README.md @@ -95,3 +95,12 @@ Full design: [MCP toolshare](https://github.com/systemind-team/SharedOS/blob/mai and [ADR 0014](https://github.com/systemind-team/SharedOS/blob/main/docs/adr/0014-mcp-toolshare.md). SharedOS is currently a `1.0.0` preview. + +## Image results + +A successful tool may include `metadata["sharedos/content"]` with inline MCP +image blocks (`type: "image"`, `mimeType`, base64 `data`). The MCP adapter retains +the ordinary JSON/text summary and appends valid JPEG, PNG or WebP images, bounded +to four images and 4 MiB of encoded data per result. Other metadata, remote URLs, +and images on failed or denied results are not forwarded as content. This is a +presentation hint; it grants no authority and does not change tool discovery. diff --git a/packages/mcp/src/image-results.test.ts b/packages/mcp/src/image-results.test.ts new file mode 100644 index 00000000..c2085af7 --- /dev/null +++ b/packages/mcp/src/image-results.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { toCallToolResult } from "./server.js"; +import type { ToolResult } from "@aicoo/sharedos-contracts"; + +const result = (content: unknown): ToolResult => ({ + callId: "call", + tool: "screen", + completedAt: "2026-10-31T00:00:00Z", + status: "succeeded", + output: { epoch: 3 }, + metadata: { "sharedos/content": content } as ToolResult["metadata"], +}); +describe("image presentation through the authorized MCP result", () => { + it("preserves structured output and exposes an image to the harness", () => { + const image = { type: "image", mimeType: "image/png", data: "aGVsbG8=" }; + expect(toCallToolResult(result([image])).content).toEqual([ + { type: "text", text: '{"epoch":3}' }, + image, + ]); + }); + it("does not fetch URLs or pass unsupported metadata content", () => { + expect( + toCallToolResult( + result([ + { type: "resource_link", uri: "https://example.com" }, + { type: "image", mimeType: "text/html", data: "aGVsbG8=" }, + ]), + ).content, + ).toHaveLength(1); + }); + it("bounds image size and rejects malformed base64", () => { + for (const data of ["!not-base64", "A".repeat(4 * 1024 * 1024 + 4)]) + expect( + toCallToolResult(result([{ type: "image", mimeType: "image/png", data }])).content, + ).toHaveLength(1); + }); + it("never exposes presentation hints from denied results", () => { + const denied = { + ...result([{ type: "image", mimeType: "image/png", data: "aGVsbG8=" }]), + status: "denied", + error: { code: "denied", message: "No grant" }, + } as ToolResult; + expect(toCallToolResult(denied).content).toHaveLength(1); + expect(toCallToolResult(denied).isError).toBe(true); + }); +}); diff --git a/packages/mcp/src/server.ts b/packages/mcp/src/server.ts index 599f4917..acdfde79 100644 --- a/packages/mcp/src/server.ts +++ b/packages/mcp/src/server.ts @@ -338,7 +338,10 @@ export function toCallToolResult( ? { structuredContent: result.output } : {}; return { - content: [{ type: "text", text: renderText(result.output) }], + content: [ + { type: "text", text: renderText(result.output) }, + ...imageContent(result.metadata?.["sharedos/content"]), + ], ...structured, isError: false, }; @@ -357,6 +360,33 @@ export function toCallToolResult( }; } +/** Optional, host-neutral presentation hints. Output remains the structured + * result. Only bounded inline images are recognized: no URLs are fetched and + * arbitrary metadata cannot introduce executable or remote resources. + */ +function imageContent(value: unknown): JsonObject[] { + if (!Array.isArray(value) || value.length > 4) return []; + const images: JsonObject[] = []; + let bytes = 0; + for (const raw of value) { + if (!raw || typeof raw !== "object") continue; + const { type, data, mimeType } = raw as Record; + if ( + type !== "image" || + typeof data !== "string" || + !["image/png", "image/jpeg", "image/webp"].includes(String(mimeType)) || + data.length === 0 || + data.length % 4 !== 0 || + !/^[A-Za-z0-9+/]*={0,2}$/.test(data) + ) + continue; + bytes += data.length; + if (bytes > 4 * 1024 * 1024) return []; + images.push({ type: "image", data, mimeType: String(mimeType) }); + } + return images; +} + function renderText(output: JsonValue): string { return typeof output === "string" ? output : JSON.stringify(output); }