diff --git a/src/lib/mcp/analytics.test.ts b/src/lib/mcp/analytics.test.ts index d1b3623..a08eb7f 100644 --- a/src/lib/mcp/analytics.test.ts +++ b/src/lib/mcp/analytics.test.ts @@ -33,8 +33,10 @@ import { MCP_REMOVED_PARAMS_PROPERTY, OAUTH_TOKEN_EXCHANGE_EVENT, sanitizeMcpAnalyticsEvent, + UNREGISTERED_RESOURCE_NAME, } from "@/lib/mcp/analytics"; import { connectTestMcp, toolResultJSON } from "@/lib/mcp/mcp-test-fixtures"; +import { MANAGED_AUTH_RESOURCE_URI } from "@/lib/mcp/tools/auth-login-app"; import { KERNEL_FEEDBACK_TOOL_NAME } from "@/lib/mcp/tools/feedback"; import { registerMcpCapabilities } from "@/lib/mcp/register"; import { z } from "zod"; @@ -1937,3 +1939,181 @@ describe("instrumentMcpAnalytics (SDK integration)", () => { expect(toolsList.distinctId).toStartWith("ses_"); }); }); + +describe("instrumentMcpAnalytics client-named targets", () => { + const SESSION_ID = "ses_synthetic9f2c"; + const PROFILE_NAME = "profile-synthetic-4d1e"; + const APP_NAME = "app-synthetic-8b7a"; + const MISSING_SESSION_ID = "ses_missing0c3d"; + const TOKEN = "sk_synthetic_Zq8vN3pL5tR7wX2y"; + const BASE = "kernel://orgs/org_test/projects/proj_test"; + + type CapturedEvent = { + event?: string; + properties: Record; + }; + + async function connectInstrumented() { + const captured: CapturedEvent[] = []; + const kernelClient = { + browsers: { + retrieve: async (id: string) => + id === SESSION_ID ? { session_id: id } : null, + list: async function* () { + yield { session_id: SESSION_ID }; + }, + }, + profiles: { + retrieve: async (name: string) => ({ name }), + }, + apps: { + list: async () => ({ + getPaginatedItems: () => [{ app_name: APP_NAME }], + }), + }, + }; + const mcp = await connectTestMcp((server, dependencies) => { + instrumentMcpAnalytics(server, { + capture: (event: CapturedEvent) => captured.push(event), + } as unknown as PostHog); + registerMcpCapabilities(server, { mcpApps: true, dependencies }); + }, kernelClient); + return { ...mcp, captured }; + } + + async function capturedEvents(captured: CapturedEvent[], name: string) { + await new Promise((resolve) => setTimeout(resolve, 50)); + return captured.filter(({ event }) => event === name); + } + + test("reports the registered resource name instead of the URI", async () => { + const { client, close, captured } = await connectInstrumented(); + try { + for (const uri of [ + `${BASE}/browsers/${SESSION_ID}`, + `${BASE}/browsers`, + `${BASE}/profiles/${PROFILE_NAME}`, + `${BASE}/apps/${APP_NAME}`, + MANAGED_AUTH_RESOURCE_URI, + ]) { + await client.readResource({ uri }); + } + + const reads = await capturedEvents( + captured, + PostHogMCPAnalyticsEvent.ResourceRead, + ); + expect( + reads.map( + ({ properties }) => + properties[PostHogMCPAnalyticsProperty.ResourceName], + ), + ).toEqual([ + "browser", + "browsers", + "profile", + "app", + "kernel-managed-auth-login", + ]); + for (const { properties } of reads) { + expect(properties[PostHogMCPAnalyticsProperty.IsError]).toBe(false); + expect(typeof properties[PostHogMCPAnalyticsProperty.DurationMs]).toBe( + "number", + ); + } + const serialized = JSON.stringify(reads); + for (const value of [ + "kernel://", + "ui://", + "proj_test", + SESSION_ID, + PROFILE_NAME, + APP_NAME, + ]) { + expect(serialized).not.toContain(value); + } + } finally { + await close(); + } + }); + + test("reports failed reads without the URI", async () => { + const { client, close, captured } = await connectInstrumented(); + try { + for (const uri of [ + `${BASE}/browsers/${MISSING_SESSION_ID}`, + `${BASE}/secrets/${TOKEN}`, + `https://${TOKEN}@files.example.test/${TOKEN}?key=${TOKEN}`, + ]) { + await expect(client.readResource({ uri })).rejects.toThrow(); + } + + const reads = await capturedEvents( + captured, + PostHogMCPAnalyticsEvent.ResourceRead, + ); + expect( + reads.map(({ properties }) => [ + properties[PostHogMCPAnalyticsProperty.ResourceName], + properties[PostHogMCPAnalyticsProperty.IsError], + ]), + ).toEqual([ + ["browser", true], + [UNREGISTERED_RESOURCE_NAME, true], + [UNREGISTERED_RESOURCE_NAME, true], + ]); + const serialized = JSON.stringify(captured); + for (const value of ["kernel://", MISSING_SESSION_ID, TOKEN]) { + expect(serialized).not.toContain(value); + } + } finally { + await close(); + } + }); + + test("drops calls to unregistered tools", async () => { + const { client, close, captured } = await connectInstrumented(); + try { + for (const name of [TOKEN, "constructor"]) { + await client + .callTool({ name, arguments: { context: "Calling a tool." } }) + .catch(() => undefined); + } + + await new Promise((resolve) => setTimeout(resolve, 50)); + expect( + captured.filter( + ({ event }) => event === PostHogMCPAnalyticsEvent.ToolCall, + ), + ).toEqual([]); + expect(JSON.stringify(captured)).not.toContain(TOKEN); + } finally { + await close(); + } + }); + + test("leaves tool call names unchanged", async () => { + const { client, close, captured } = await connectInstrumented(); + try { + await client.callTool({ + name: "get_connection_context", + arguments: { context: "Checking tool analytics next to resources." }, + }); + + const [toolCall] = await capturedEvents( + captured, + PostHogMCPAnalyticsEvent.ToolCall, + ); + expect(toolCall.properties).toMatchObject({ + [PostHogMCPAnalyticsProperty.ToolName]: "get_connection_context", + [PostHogMCPAnalyticsProperty.ResourceName]: "get_connection_context", + [PostHogMCPAnalyticsProperty.IsError]: false, + }); + expect( + typeof toolCall.properties[PostHogMCPAnalyticsProperty.DurationMs], + ).toBe("number"); + } finally { + await close(); + } + }); +}); diff --git a/src/lib/mcp/analytics.ts b/src/lib/mcp/analytics.ts index 941719c..bf1f5ff 100644 --- a/src/lib/mcp/analytics.ts +++ b/src/lib/mcp/analytics.ts @@ -14,6 +14,7 @@ import { CLIENT_CAPABILITIES_META_KEY, CLIENT_INFO_META_KEY, type McpServer, + type ResourceTemplate, } from "@modelcontextprotocol/server"; import { PostHog } from "posthog-node"; import type { @@ -402,6 +403,42 @@ function sanitizeIntent(intent: string) { } const ANALYTICS_CONTEXT_PROPERTY = "__mcp_connection_analytics_context"; +const RESOURCE_CATEGORY_PROPERTY = "__mcp_resource_category"; +const UNREGISTERED_TOOL_PROPERTY = "__mcp_unregistered_tool"; + +export const UNREGISTERED_RESOURCE_NAME = "unregistered"; + +type ResourceRegistry = { + _registeredResources?: Record; + _registeredResourceTemplates?: Record< + string, + { resourceTemplate: ResourceTemplate } + >; +}; + +/** + * Names the registered resource a resources/read URI resolves to, matched the way + * McpServer's read handler matches it. The SDK reports the requested URI, which carries + * whatever the client put in it; the registered name is server-defined. + */ +function resourceReadCategory(server: McpServer, uri: unknown) { + if (typeof uri !== "string") return UNREGISTERED_RESOURCE_NAME; + let normalized: string; + try { + normalized = new URL(uri).toString(); + } catch { + return UNREGISTERED_RESOURCE_NAME; + } + const registry = server as unknown as ResourceRegistry; + const resource = registry._registeredResources?.[normalized]; + if (resource) return resource.name; + for (const [name, template] of Object.entries( + registry._registeredResourceTemplates ?? {}, + )) { + if (template.resourceTemplate.uriTemplate.match(normalized)) return name; + } + return UNREGISTERED_RESOURCE_NAME; +} // PostHog reads the `context` argument only for tools whose schema leaves it to the SDK. // Declaring `context` on a tool (the analytics-backed tools, manage_browsers) stops that @@ -423,6 +460,13 @@ export const sanitizeMcpAnalyticsEvent: BeforeSendFn = (event) => { const properties = event.properties; if (!properties) return event; + // The SDK names a call to an unregistered tool after whatever the client sent. + if ( + event.event === PostHogMCPAnalyticsEvent.ToolCall && + properties[UNREGISTERED_TOOL_PROPERTY] === true + ) { + return null; + } enrichMcpAnalyticsEvent(event); applyDerivedSession(event); if (event.event === PostHogMCPAnalyticsEvent.ToolCall) { @@ -440,6 +484,14 @@ export const sanitizeMcpAnalyticsEvent: BeforeSendFn = (event) => { properties[PostHogMCPAnalyticsProperty.ErrorType] = "validation"; } } + if (event.event === PostHogMCPAnalyticsEvent.ResourceRead) { + const category = properties[RESOURCE_CATEGORY_PROPERTY]; + if (typeof category === "string") { + properties[PostHogMCPAnalyticsProperty.ResourceName] = category; + } else { + delete properties[PostHogMCPAnalyticsProperty.ResourceName]; + } + } for (const key of Object.keys(properties)) { if (!SENT_PROPERTIES.has(key)) delete properties[key]; @@ -1022,7 +1074,24 @@ export function instrumentMcpAnalytics( } const sessionKey = analyticsSessionKey(extra); if (sessionKey) properties[SESSION_KEY_PROPERTY] = sessionKey; + if (request.method === "resources/read") { + properties[RESOURCE_CATEGORY_PROPERTY] = resourceReadCategory( + server, + request.params?.uri, + ); + } if (request.method === "tools/call") { + const { _registeredTools: registeredTools } = server as unknown as { + _registeredTools: Record; + }; + if ( + !Object.prototype.hasOwnProperty.call( + registeredTools, + String(request.params?.name), + ) + ) { + properties[UNREGISTERED_TOOL_PROPERTY] = true; + } const model = clientMetadataModel(request); if (model) { properties[PostHogMCPAnalyticsProperty.LlmModel] = model;