From 216114127dc12851d542ceb2fbc1a91a620ba49e Mon Sep 17 00:00:00 2001 From: Makisuo Date: Tue, 4 Aug 2026 19:46:17 +0200 Subject: [PATCH] feat(llm): attribute and tag OpenRouter requests Every OpenRouter request Maple made was anonymous: the dashboard showed one undifferentiated pile of spend with no way to tell chat from AI triage from the Slack agent, or to attribute cost to an org. Maple also had no app page on openrouter.ai at all, since that is only created for traffic carrying an HTTP-Referer header. Attribution rides on every OpenRouter call now (HTTP-Referer: https://maple.dev, X-Title: Maple), and resolveTriageModel takes optional LlmCallTags that it folds into the request body as route defaults - user (org id), session_id, and trace.trace_name. Being route defaults, every generate/stream/generateObject made with that model inherits them, including triage's final structured pass, without each call site threading anything through. The Workers AI branch ignores the tags entirely: those are OpenRouter's body fields. apps/slack-agent uses the provider's first-class appUrl/appName plus a static trace_name, deliberately with the same referer as apps/api - the referer is the app's identity, so a second value would mint a second app entry and split the rankings. MCP evals stay unattributed so CI spend stays out of the product's numbers. Nothing in lib/llm changed: ModelOptions already carries headers and http, and user/session_id/trace are not in the transport's body-overlay denylist. docs/openrouter-tracing.md described headers and a BYOK table that no longer exist and pointed at files deleted in c3c7cc1a8; rewritten to match reality. --- apps/api/src/chat/turn-runner.ts | 6 +- apps/api/src/mcp/__evals__/model.ts | 11 +- apps/api/src/platform/Llm.test.ts | 115 ++++++++++++++++++ apps/api/src/platform/Llm.ts | 48 +++++++- .../api/src/workflows/AiTriageWorkflow.run.ts | 7 +- apps/slack-agent/agent/agent.ts | 9 ++ docs/openrouter-tracing.md | 103 ++++++++++------ 7 files changed, 250 insertions(+), 49 deletions(-) create mode 100644 apps/api/src/platform/Llm.test.ts diff --git a/apps/api/src/chat/turn-runner.ts b/apps/api/src/chat/turn-runner.ts index 8ff199bc2..95bd17f58 100644 --- a/apps/api/src/chat/turn-runner.ts +++ b/apps/api/src/chat/turn-runner.ts @@ -136,7 +136,11 @@ export const runChatSessionTurn = async (input: RunChatSessionTurnInput): Promis const program = Effect.gen(function* () { const investigations = yield* InvestigationService const history = input.session.history() - const model = resolveTriageModel(input.env) + const model = resolveTriageModel(input.env, { + surface: "chat", + orgId: tenant.orgId, + sessionId: input.sessionId, + }) // Shared with the turn so `submit_diagnosis` can report what the investigation cost. See // `TurnUsage` — the tool is invoked mid-turn, so there is no later moment to hand it a total. const usage = agent.makeTurnUsage() diff --git a/apps/api/src/mcp/__evals__/model.ts b/apps/api/src/mcp/__evals__/model.ts index 71b0be8ae..d5230fd19 100644 --- a/apps/api/src/mcp/__evals__/model.ts +++ b/apps/api/src/mcp/__evals__/model.ts @@ -2,9 +2,7 @@ import { createOpenAICompatible } from "@ai-sdk/openai-compatible" import type { LanguageModel } from "ai" /** - * Default eval model — matches apps/chat-agent's prod default - * (`DEFAULT_MODEL_ID` in apps/chat-agent/src/lib/openrouter.ts), so evals - * reflect what real users run. Override with `MCP_EVAL_MODEL`. + * Default eval model. Override with `MCP_EVAL_MODEL`. */ const DEFAULT_EVAL_MODEL = "moonshotai/kimi-k2.7-code" @@ -14,8 +12,11 @@ const evalModelId = (): string => process.env.MCP_EVAL_MODEL ?? DEFAULT_EVAL_MOD export const hasEvalCredentials = (): boolean => Boolean(process.env.OPENROUTER_API_KEY) /** - * Build the eval model via OpenRouter — same wiring as - * apps/chat-agent/src/lib/openrouter.ts `createChatModel`. + * Build the eval model via OpenRouter. + * + * Deliberately *not* app-attributed or tagged the way `apps/api/src/platform/Llm.ts` is: this is + * CI-only traffic, and keeping it off Maple's OpenRouter app page keeps eval spend out of the + * product's numbers. See `docs/openrouter-tracing.md`. */ export const createEvalModel = (): LanguageModel => { const apiKey = process.env.OPENROUTER_API_KEY diff --git a/apps/api/src/platform/Llm.test.ts b/apps/api/src/platform/Llm.test.ts new file mode 100644 index 000000000..39c5739ae --- /dev/null +++ b/apps/api/src/platform/Llm.test.ts @@ -0,0 +1,115 @@ +/** + * Wire-level proof that OpenRouter calls are attributed and tagged. + * + * `LLMClient.prepare` is deliberately not used here: it returns the *protocol* body, which is built + * before `http.body` is overlaid onto it, so it cannot see the tags at all. The only place the tags + * and the attribution headers exist together is the outgoing HTTP request — so the test swaps + * `FetchHttpClient.Fetch` for a capture and reads what would have gone over the wire. + * + * The fake responds 400, which `@maple/llm` classifies as non-retryable. That keeps the run to a + * single request with no backoff; the resulting failure is expected and ignored. + */ +import { LLM } from "@maple/llm" +import { Effect, Layer } from "effect" +import { FetchHttpClient } from "effect/unstable/http" +import { describe, expect, it } from "vitest" +import { layerLlm, resolveTriageModel, type LlmCallTags, type LlmEnv } from "./Llm" + +interface CapturedRequest { + readonly url: string + readonly headers: Record + readonly body: Record +} + +/** + * Run one `LLM.generate` against a fetch that records the request instead of sending it. + */ +const captureRequest = async (env: LlmEnv, tags?: LlmCallTags): Promise => { + let captured: CapturedRequest | undefined + + const fakeFetch: typeof globalThis.fetch = async (input, init) => { + const headers: Record = {} + new Headers(init?.headers).forEach((value, key) => { + headers[key.toLowerCase()] = value + }) + // The body arrives as bytes, not a string — `Response` is the cheapest correct decoder. + const bodyText = await new Response(init?.body ?? "{}").text() + captured = { + url: String(input), + headers, + body: JSON.parse(bodyText) as Record, + } + return new Response(JSON.stringify({ error: "captured" }), { status: 400 }) + } + + const request = LLM.request({ + model: resolveTriageModel(env, tags), + system: "You are concise.", + prompt: "hi", + }) + + await Effect.runPromise( + LLM.generate(request).pipe( + Effect.ignore, + Effect.provide( + layerLlm(env).pipe(Layer.provide(Layer.succeed(FetchHttpClient.Fetch, fakeFetch))), + ), + ), + ) + + if (captured === undefined) throw new Error("no request reached the transport") + return captured +} + +const openRouterEnv: LlmEnv = { OPENROUTER_API_KEY: "test-key" } + +const tags: LlmCallTags = { surface: "chat", orgId: "org_123", sessionId: "chat_abc" } + +describe("resolveTriageModel — OpenRouter attribution", () => { + it("sends the app-attribution headers on every OpenRouter call", async () => { + const captured = await captureRequest(openRouterEnv) + + expect(captured.url).toContain("openrouter.ai") + // `HTTP-Referer` is what creates the app page — a title alone does nothing. + expect(captured.headers["http-referer"]).toBe("https://maple.dev") + expect(captured.headers["x-title"]).toBe("Maple") + }) + + it("tags the request body with surface, org and session", async () => { + const captured = await captureRequest(openRouterEnv, tags) + + expect(captured.body).toMatchObject({ + user: "org_123", + session_id: "chat_abc", + trace: { trace_name: "chat" }, + }) + }) + + it("omits session_id when the caller has no session to group by", async () => { + const captured = await captureRequest(openRouterEnv, { surface: "ai-triage", orgId: "org_123" }) + + expect(captured.body).toMatchObject({ user: "org_123", trace: { trace_name: "ai-triage" } }) + expect(captured.body).not.toHaveProperty("session_id") + }) + + it("truncates an over-long session id to OpenRouter's 256-character limit", async () => { + const captured = await captureRequest(openRouterEnv, { ...tags, sessionId: "s".repeat(400) }) + + expect(captured.body.session_id).toHaveLength(256) + }) + + it("keeps the headers and tags off the Workers AI path", async () => { + const captured = await captureRequest( + { MAPLE_LLM_PROVIDER: "workers-ai", CLOUDFLARE_API_KEY: "test-key" }, + tags, + ) + + expect(captured.url).not.toContain("openrouter.ai") + expect(captured.headers).not.toHaveProperty("http-referer") + expect(captured.headers).not.toHaveProperty("x-title") + // These are OpenRouter's body fields; Cloudflare must never be sent them. + expect(captured.body).not.toHaveProperty("user") + expect(captured.body).not.toHaveProperty("session_id") + expect(captured.body).not.toHaveProperty("trace") + }) +}) diff --git a/apps/api/src/platform/Llm.ts b/apps/api/src/platform/Llm.ts index 09109ef9b..ec62784df 100644 --- a/apps/api/src/platform/Llm.ts +++ b/apps/api/src/platform/Llm.ts @@ -31,6 +31,40 @@ import { layerWorkersAi } from "./WorkersAiHttpClient" */ export const DEFAULT_OPENROUTER_MODEL = "openai/gpt-5.6-luna" +/** + * OpenRouter app attribution. `HTTP-Referer` is the header that actually creates the app page — + * a title on its own does nothing — so both are sent together or not at all. One URL for every + * surface on purpose: a second referer would mint a second app entry and split the rankings. + */ +const OPENROUTER_APP_URL = "https://maple.dev" +const OPENROUTER_APP_TITLE = "Maple" + +/** + * Where a model call came from and what it is running for. + * + * OpenRouter surfaces these three in different places, which is why all three are worth sending: + * `user` shows up on the activity page and in usage exports, `session_id` groups a conversation + * (and makes OpenRouter route the whole session to one provider, so prompt caches actually hit), + * and `trace` is forwarded to any configured Broadcast destination. + */ +export interface LlmCallTags { + readonly surface: "chat" | "ai-triage" + readonly orgId: string + /** Groups one conversation or investigation. OpenRouter caps this at 256 characters. */ + readonly sessionId?: string +} + +/** + * The tag fields as OpenRouter's request body wants them. Kept next to the `configure` call that + * uses it because these are OpenRouter's field names, not Maple's — the Workers AI branch must + * never see them. + */ +const openRouterTagBody = (tags: LlmCallTags) => ({ + user: tags.orgId, + ...(tags.sessionId === undefined ? {} : { session_id: tags.sessionId.slice(0, 256) }), + trace: { trace_name: tags.surface }, +}) + /** * Default triage/chat model on Workers AI. Carried over unchanged from the pre-`@maple/llm` chat * backend (`cloudflare/@cf/moonshotai/kimi-k2.6`, minus that runtime's `provider/` prefix), so the @@ -82,8 +116,16 @@ export const resolveLlmProvider = (env: LlmEnv): LlmProvider => ? "workers-ai" : DEFAULT_LLM_PROVIDER -/** Resolve the model the triage/chat agents should run on, from env. */ -export const resolveTriageModel = (env: LlmEnv): Model => +/** + * Resolve the model the triage/chat agents should run on, from env. + * + * `tags` is optional and OpenRouter-only. Attribution headers ride on every OpenRouter call + * regardless; the per-call tags are folded into the request body as route defaults, so every + * `LLM.request`/`generate`/`stream` made with the returned model carries them without each call + * site having to thread them through. The Workers AI branch deliberately ignores `tags` — they are + * OpenRouter's body fields and have no meaning to Cloudflare. + */ +export const resolveTriageModel = (env: LlmEnv, tags?: LlmCallTags): Model => resolveLlmProvider(env) === "workers-ai" ? CloudflareWorkersAI.configure({ accountId: readString(env, "CLOUDFLARE_ACCOUNT_ID") ?? BINDING_PLACEHOLDER, @@ -91,6 +133,8 @@ export const resolveTriageModel = (env: LlmEnv): Model => }).model(readString(env, "MAPLE_TRIAGE_MODEL_WORKERS_AI") ?? DEFAULT_WORKERS_AI_MODEL) : OpenRouter.configure({ apiKey: readString(env, "OPENROUTER_API_KEY") ?? "", + headers: { "HTTP-Referer": OPENROUTER_APP_URL, "X-Title": OPENROUTER_APP_TITLE }, + ...(tags === undefined ? {} : { http: { body: openRouterTagBody(tags) } }), }).model(readString(env, "MAPLE_TRIAGE_MODEL_OPENROUTER") ?? DEFAULT_OPENROUTER_MODEL) /** diff --git a/apps/api/src/workflows/AiTriageWorkflow.run.ts b/apps/api/src/workflows/AiTriageWorkflow.run.ts index 588dc63d4..df43b2609 100644 --- a/apps/api/src/workflows/AiTriageWorkflow.run.ts +++ b/apps/api/src/workflows/AiTriageWorkflow.run.ts @@ -99,6 +99,7 @@ const invokeTriageWorkflow = async ({ env, orgId, incidentKind, + incidentId, context, }: InvokeTriageInput): Promise => { if (!canReachModel(env)) throw new Error("llm_unavailable") @@ -127,7 +128,11 @@ const invokeTriageWorkflow = async ({ orgId, incidentKind, context, - model: resolveTriageModel(env), + model: resolveTriageModel(env, { + surface: "ai-triage", + orgId, + sessionId: `triage_${incidentKind}_${incidentId}`, + }), tenant: { orgId: decodeOrgId(orgId), userId: internalServiceUserId, diff --git a/apps/slack-agent/agent/agent.ts b/apps/slack-agent/agent/agent.ts index e1cb656b1..2891762c5 100644 --- a/apps/slack-agent/agent/agent.ts +++ b/apps/slack-agent/agent/agent.ts @@ -3,9 +3,18 @@ import { defineAgent } from "eve" /** * OpenRouter over its REST API. + * + * `appUrl`/`appName` set `HTTP-Referer`/`X-OpenRouter-Title`, which is what attributes this + * traffic to Maple's app page on openrouter.ai. Same URL and title as `apps/api` on purpose: the + * referer is the app's identity, so a different one here would mint a second app entry and split + * the rankings. Surfaces are told apart by `trace.trace_name` instead — static, because this + * process only ever is the Slack agent. */ const openrouter = createOpenRouter({ apiKey: process.env.OPENROUTER_API_KEY ?? "", + appUrl: "https://maple.dev", + appName: "Maple", + extraBody: { trace: { trace_name: "slack" } }, }) /** diff --git a/docs/openrouter-tracing.md b/docs/openrouter-tracing.md index 580dcc736..3e23f6835 100644 --- a/docs/openrouter-tracing.md +++ b/docs/openrouter-tracing.md @@ -1,40 +1,63 @@ -# OpenRouter Tracing +# OpenRouter Attribution And Tracing -OpenRouter Broadcast can send each OpenRouter completion as an OTLP/HTTP trace to any backend that accepts JSON OTLP on `/v1/traces`. Maple can receive those traces through the normal ingest gateway, so chat-agent LLM calls can appear alongside application traces. +Maple attributes every OpenRouter request to its app page on openrouter.ai and tags it with the +surface it came from, the org it ran for, and the session it belongs to. Separately, OpenRouter +Broadcast can send each completion as an OTLP/HTTP trace to any backend that accepts JSON OTLP on +`/v1/traces` — including Maple's own ingest gateway. -References verified on June 4, 2026: +References verified on August 4, 2026: -- OpenRouter Broadcast to OpenTelemetry Collector: https://openrouter.ai/docs/guides/features/broadcast/otel-collector -- OpenRouter API request fields and app attribution headers: https://openrouter.ai/docs/api/reference/overview +- App attribution: https://openrouter.ai/docs/app-attribution +- User tracking: https://openrouter.ai/docs/guides/guides/administration/user-tracking +- Broadcast to OpenTelemetry Collector: https://openrouter.ai/docs/guides/features/broadcast/otel-collector -## What Maple Sends +## Where OpenRouter Is Called -`apps/chat-agent/src/lib/openrouter.ts` centralizes OpenRouter request setup. +| Path | Client | Surfaces | +| --- | --- | --- | +| `apps/api/src/platform/Llm.ts` | `@maple/llm` (`OpenRouter.configure`) | chat turns (`src/chat/turn-runner.ts`), AI triage (`src/workflows/AiTriageWorkflow.run.ts`) | +| `apps/slack-agent/agent/agent.ts` | `@openrouter/ai-sdk-provider` | the Slack agent | +| `apps/api/src/mcp/__evals__/model.ts` | `@ai-sdk/openai-compatible` | MCP evals in CI — **not** attributed or tagged | -Every chat turn sent through `apps/chat-agent/src/index.ts` includes `providerOptions.openrouter.trace`, which the OpenAI-compatible AI SDK forwards into the OpenRouter request body. OpenRouter documents these fields as Broadcast trace metadata: +`apps/api` can also run on Cloudflare Workers AI instead (`MAPLE_LLM_PROVIDER=workers-ai`). None of +the attribution below applies on that path — the headers and tag fields are OpenRouter's, and +`resolveTriageModel` deliberately keeps them off the Workers AI branch. -| Field | Maple value | -| ----------------------- | ---------------------------------------------------------------------------- | -| `trace.trace_id` | Chat request id, or a generated turn id when the request has none. | -| `trace.trace_name` | `Maple Chat Agent`. | -| `trace.generation_name` | `Chat Turn`. | -| `session_id` | The chat Durable Object name, currently `:`. | -| `trace.orgId` | Maple org id, surfaced downstream as `trace.metadata.orgId`. | -| `trace.operation` | `chat.turn`. | -| `trace.mode` | Chat mode, such as `default`, `dashboard_builder`, `alert`, or `widget-fix`. | -| `trace.environment` | `MAPLE_ENVIRONMENT` when configured. | -| `trace.isByok` | Whether the org's own OpenRouter key was used. | +## App Attribution -The OpenRouter provider also sends app attribution headers: +`HTTP-Referer` is what creates the app page on openrouter.ai; a title on its own does nothing and +usage without a referer never appears in the rankings. Both the API and the Slack agent send the +same URL and title on purpose — the referer *is* the app's identity, so a second value would mint a +second app entry and split the rankings. -| Header | Maple value | -| -------------------- | ------------------------------------- | -| `HTTP-Referer` | `MAPLE_APP_BASE_URL` when configured. | -| `X-OpenRouter-Title` | `Maple` by default. | +| Header | Value | Set at | +| --- | --- | --- | +| `HTTP-Referer` | `https://maple.dev` | `apps/api/src/platform/Llm.ts` (`OPENROUTER_APP_URL`), `apps/slack-agent/agent/agent.ts` (`appUrl`) | +| `X-Title` / `X-OpenRouter-Title` | `Maple` | same, `OPENROUTER_APP_TITLE` / `appName` | + +Per-app analytics then live at https://openrouter.ai/apps. + +## Per-Request Tags + +`resolveTriageModel(env, tags)` takes an optional `LlmCallTags` (`surface`, `orgId`, `sessionId`) +and folds it into the OpenRouter request body as route defaults, so every `LLM.request` / +`generate` / `stream` made with the returned model carries it without each call site threading it +through. + +| Field | Maple value | Where it shows up | +| --- | --- | --- | +| `user` | Maple org id | the `/activity` page, activity exports, and the `/generations` API. OpenRouter folds it into a hashed identity and never forwards it raw upstream. | +| `session_id` | `` or `triage__`, truncated to OpenRouter's 256-character limit | groups the requests of one conversation or investigation, and makes OpenRouter route the whole session to a single provider so prompt caches actually hit | +| `trace.trace_name` | `chat`, `ai-triage`, or `slack` | forwarded to configured Broadcast destinations only — it does **not** appear in the OpenRouter dashboard | + +The Slack agent sends a static `trace: { trace_name: "slack" }` via the provider's `extraBody`, +since that process is a single surface. It does not send `user` or `session_id`; wiring the Slack +team and thread through would need a per-request hook from `eve`. ## Configure OpenRouter Broadcast To Maple -Use this when you want OpenRouter-generated LLM traces to land in Maple. +Use this when you want OpenRouter-generated LLM traces to land in Maple. This is also the only way +the `trace` metadata above becomes visible anywhere. 1. In Maple, copy the org's private ingest key from Settings -> Ingestion. It has the `maple_sk_...` prefix. 2. In OpenRouter, open Settings -> Observability and enable Broadcast. @@ -61,34 +84,34 @@ https:///v1/traces 6. Use OpenRouter's Test Connection action, then send a Maple chat message. -OpenRouter only emits Broadcast traces for traffic under the OpenRouter account or workspace where Broadcast is enabled. If an org uses BYOK in Maple Settings -> AI, configure Broadcast in that org's OpenRouter account. If the org falls back to Maple's default OpenRouter key, Broadcast must be configured on Maple's OpenRouter account. +OpenRouter only emits Broadcast traces for traffic under the OpenRouter account or workspace where +Broadcast is enabled. Maple has no BYOK path — every org's traffic runs on Maple's own +`OPENROUTER_API_KEY` — so Broadcast is configured once, on Maple's OpenRouter account. ## Querying In Maple -OpenRouter Broadcast traces use standard GenAI semantic convention attributes such as `gen_ai.*` for model, usage, and cost data. Maple also receives the custom metadata above under OpenRouter's `trace.metadata.*` namespace. +OpenRouter Broadcast traces use standard GenAI semantic convention attributes such as `gen_ai.*` for +model, usage, and cost data. The tag fields arrive under OpenRouter's `trace.metadata.*` namespace. Useful filters: ```text -trace.metadata.orgId = "" -trace.metadata.operation = "chat.turn" -trace.metadata.mode = "dashboard_builder" -session.id = ":" +trace.metadata.trace_name = "ai-triage" +session.id = "triage_error_" ``` -If prompt or completion content should not leave OpenRouter, enable Privacy Mode for the OpenRouter observability destination. OpenRouter's docs state that Privacy Mode excludes prompt and completion content while still sending timing, model, token usage, cost, and metadata. +If prompt or completion content should not leave OpenRouter, enable Privacy Mode for the OpenRouter +observability destination. OpenRouter's docs state that Privacy Mode excludes prompt and completion +content while still sending timing, model, token usage, cost, and metadata. ## Local Test Coverage -The OpenRouter request contract is covered by: +The attribution and tagging contract is covered by: ```bash -TMPDIR=/tmp bun test apps/chat-agent/src/lib/openrouter.test.ts +bun run --cwd apps/api vitest run src/platform/Llm.test.ts ``` -The tests assert that Maple: - -- sends the OpenRouter app attribution headers; -- forwards trace correlation metadata under `providerOptions.openrouter`; -- omits blank optional metadata instead of sending empty attributes; -- rejects an empty `traceId`. +Those tests swap `FetchHttpClient.Fetch` for a capture and assert, on the outgoing request, that +Maple sends the attribution headers, the `user` / `session_id` / `trace` fields, omits `session_id` +when there is no session, truncates an over-long one, and sends none of it on the Workers AI path.