Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs/api/sharedos-mcp.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,15 @@ and [ADR 0014](https://github.com/systemind-team/SharedOS/blob/main/docs/adr/001

SharedOS is currently a `1.0.0` preview.

## Image results

A successful tool may include `metadata["sharedos/content"]` with inline MCP
image blocks (`type: "image"`, `mimeType`, base64 `data`). The MCP adapter retains
the ordinary JSON/text summary and appends valid JPEG, PNG or WebP images, bounded
to four images and 4 MiB of encoded data per result. Other metadata, remote URLs,
and images on failed or denied results are not forwarded as content. This is a
presentation hint; it grants no authority and does not change tool discovery.

## Classes

### McpToolServer
Expand Down
9 changes: 9 additions & 0 deletions packages/mcp/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -95,3 +95,12 @@ Full design: [MCP toolshare](https://github.com/systemind-team/SharedOS/blob/mai
and [ADR 0014](https://github.com/systemind-team/SharedOS/blob/main/docs/adr/0014-mcp-toolshare.md).

SharedOS is currently a `1.0.0` preview.

## Image results

A successful tool may include `metadata["sharedos/content"]` with inline MCP
image blocks (`type: "image"`, `mimeType`, base64 `data`). The MCP adapter retains
the ordinary JSON/text summary and appends valid JPEG, PNG or WebP images, bounded
to four images and 4 MiB of encoded data per result. Other metadata, remote URLs,
and images on failed or denied results are not forwarded as content. This is a
presentation hint; it grants no authority and does not change tool discovery.
46 changes: 46 additions & 0 deletions packages/mcp/src/image-results.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import { describe, expect, it } from "vitest";
import { toCallToolResult } from "./server.js";
import type { ToolResult } from "@aicoo/sharedos-contracts";

const result = (content: unknown): ToolResult => ({
callId: "call",
tool: "screen",
completedAt: "2026-10-31T00:00:00Z",
status: "succeeded",
output: { epoch: 3 },
metadata: { "sharedos/content": content } as ToolResult["metadata"],
});
describe("image presentation through the authorized MCP result", () => {
it("preserves structured output and exposes an image to the harness", () => {
const image = { type: "image", mimeType: "image/png", data: "aGVsbG8=" };
expect(toCallToolResult(result([image])).content).toEqual([
{ type: "text", text: '{"epoch":3}' },
image,
]);
});
it("does not fetch URLs or pass unsupported metadata content", () => {
expect(
toCallToolResult(
result([
{ type: "resource_link", uri: "https://example.com" },
{ type: "image", mimeType: "text/html", data: "aGVsbG8=" },
]),
).content,
).toHaveLength(1);
});
it("bounds image size and rejects malformed base64", () => {
for (const data of ["!not-base64", "A".repeat(4 * 1024 * 1024 + 4)])
expect(
toCallToolResult(result([{ type: "image", mimeType: "image/png", data }])).content,
).toHaveLength(1);
});
it("never exposes presentation hints from denied results", () => {
const denied = {
...result([{ type: "image", mimeType: "image/png", data: "aGVsbG8=" }]),
status: "denied",
error: { code: "denied", message: "No grant" },
} as ToolResult;
expect(toCallToolResult(denied).content).toHaveLength(1);
expect(toCallToolResult(denied).isError).toBe(true);
});
});
32 changes: 31 additions & 1 deletion packages/mcp/src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,10 @@ export function toCallToolResult(
? { structuredContent: result.output }
: {};
return {
content: [{ type: "text", text: renderText(result.output) }],
content: [
{ type: "text", text: renderText(result.output) },
...imageContent(result.metadata?.["sharedos/content"]),
],
...structured,
isError: false,
};
Expand All @@ -357,6 +360,33 @@ export function toCallToolResult(
};
}

/** Optional, host-neutral presentation hints. Output remains the structured
* result. Only bounded inline images are recognized: no URLs are fetched and
* arbitrary metadata cannot introduce executable or remote resources.
*/
function imageContent(value: unknown): JsonObject[] {
if (!Array.isArray(value) || value.length > 4) return [];
const images: JsonObject[] = [];
let bytes = 0;
for (const raw of value) {
if (!raw || typeof raw !== "object") continue;
const { type, data, mimeType } = raw as Record<string, unknown>;
if (
type !== "image" ||
typeof data !== "string" ||
!["image/png", "image/jpeg", "image/webp"].includes(String(mimeType)) ||
data.length === 0 ||
data.length % 4 !== 0 ||
!/^[A-Za-z0-9+/]*={0,2}$/.test(data)
)
continue;
bytes += data.length;
if (bytes > 4 * 1024 * 1024) return [];
images.push({ type: "image", data, mimeType: String(mimeType) });
}
return images;
}

function renderText(output: JsonValue): string {
return typeof output === "string" ? output : JSON.stringify(output);
}
Expand Down
Loading