From 4ae86e61b2c86f38ae14caf2f6bf5e7c8d2d31b7 Mon Sep 17 00:00:00 2001 From: Marc Seiler Date: Mon, 9 Nov 2026 12:35:21 -0500 Subject: [PATCH] feat(step): add option to fold reasoning tokens into output OpenCode V2 reports output with reasoning subtracted, while Claude Code includes thinking in output. OPENCODE_COMBINE_REASONING_TOKENS (or the combineReasoningTokens option) restores Claude Code semantics for the token.usage metric, api_request log, and LLM span completion count. --- AGENTS.md | 1 + README.md | 4 +++- src/config.ts | 3 +++ src/handlers/step.ts | 17 +++++++++----- src/plugin.ts | 1 + src/types.ts | 1 + tests/config.test.ts | 9 ++++++++ tests/handlers/step.test.ts | 44 +++++++++++++++++++++++++++++++++++++ tests/helpers.ts | 2 ++ 9 files changed, 75 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 916519f..bfb92b4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -64,6 +64,7 @@ src/ - **All env vars are `OPENCODE_` prefixed** — `OPENCODE_ENABLE_TELEMETRY`, `OPENCODE_OTLP_ENDPOINT`, `OPENCODE_OTLP_METRICS_INTERVAL`, `OPENCODE_OTLP_LOGS_INTERVAL`, `OPENCODE_METRIC_PREFIX`, `OPENCODE_CAPTURE_PROMPT_IN_LOGS`, `OPENCODE_OTLP_HEADERS`, `OPENCODE_RESOURCE_ATTRIBUTES`, `OPENCODE_SPAN_ATTRIBUTES`. Never use bare `OTEL_*` names for plugin config. Headers are passed directly to exporters; `loadConfig` copies `OPENCODE_RESOURCE_ATTRIBUTES` → `OTEL_RESOURCE_ATTRIBUTES` before the SDK initializes. - **`OPENCODE_ENABLE_TELEMETRY`** — all OTel instrumentation is gated on this env var (or the `enabled` plugin option). The plugin always loads regardless. - **`OPENCODE_METRIC_PREFIX`** — defaults to `opencode.`; set to `claude_code.` for Claude Code dashboard compatibility. +- **Reasoning tokens** — OpenCode V2 reports `output` with reasoning subtracted (`visibleOutputTokens`), whereas Claude Code's `output` includes thinking. `OPENCODE_COMBINE_REASONING_TOKENS` folds reasoning back into `output` (metric, `api_request` log, span completion count) and suppresses the separate `reasoning` metric/log field so sums never double-count. Session totals and `llm.token_count.total` are unaffected. - **Plugin options** — read from `ctx.options` during `setup`. Precedence is option → `OPENCODE_*` env → default. Keep option keys 1:1 with `PluginConfig` field names. ## Commit message format diff --git a/README.md b/README.md index 5b87b73..a6f54cf 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ the parent run span; a background child may outlive its already-ended dispatch t | Metric | Type | Description | |--------|------|-------------| | `opencode.session.count` | Counter | Incremented on `session.created`, or lazily when a pre-existing session is first observed | -| `opencode.token.usage` | Counter | Per token type: `input`, `output`, `reasoning`, `cacheRead`, `cacheCreation` (per `session.step.ended`) | +| `opencode.token.usage` | Counter | Per token type: `input`, `output`, `reasoning`, `cacheRead`, `cacheCreation` (per `session.step.ended`). With `OPENCODE_COMBINE_REASONING_TOKENS`, `reasoning` is folded into `output` and not emitted separately | | `opencode.cost.usage` | Counter | USD cost per completed LLM step | | `opencode.commit.count` | Counter | `git commit` commands observed in successful, executed shell-tool calls; Git's resulting repository state is not verified | | `opencode.tool.duration` | Histogram | Tool execution time in milliseconds | @@ -145,6 +145,7 @@ The environment variables (set them in your shell profile — `~/.zshrc`, `~/.ba | `OPENCODE_DISABLE_LOGS` | *(unset)* | Set to any non-empty value to suppress all OTLP log events while leaving metrics and traces unchanged | | `OPENCODE_CAPTURE_PROMPT_IN_LOGS` | *(unset)* | Set to any non-empty value to include the full prompt text in the `prompt` attribute of `user_prompt` log events. **Log events only** — trace spans carry an observed prompt in `input.value` regardless of this flag (disable span-level capture separately via `OPENCODE_DISABLE_TRACES`). **Off by default — prompts may contain secrets or PII; enable only for trusted collectors.** | | `OPENCODE_CAPTURE_MODEL_CONTEXT` | *(unset)* | Set to any non-empty value to attach a bounded text-only preview of the primary model-visible context to LLM spans. Off by default; can include system instructions, earlier messages, and tool text. | +| `OPENCODE_COMBINE_REASONING_TOKENS` | *(unset)* | Set to any non-empty value to report reasoning tokens as part of `output` instead of a separate `reasoning` type. OpenCode V2 excludes reasoning from `output`, while Claude Code includes it, so enable this for Claude Code dashboard and cost parity. Applies to the `token.usage` metric, `api_request` `output_tokens` (with `reasoning_tokens` omitted), and the LLM span `llm.token_count.completion`; `llm.token_count.completion_details.reasoning` still carries the reasoning subset. | | `OPENCODE_DISABLE_TRACES` | *(unset)* | Comma-separated list of trace types to disable (`session`, `llm`, `tool`). Use `all`, `*`, `true`, or `1` to disable every trace type | | `OPENCODE_OTLP_HEADERS` | *(unset)* | Comma-separated `key=value` headers added to all OTLP exports. **Keep out of version control — may contain sensitive auth tokens.** | | `OPENCODE_OTLP_HEADERS_HELPER` | *(unset)* | Executable script/binary that returns dynamic OTLP headers as JSON after an auth failure. Helper headers override `OPENCODE_OTLP_HEADERS`. | @@ -188,6 +189,7 @@ Option keys mirror the resolved config and map to the environment variables: | `logsEnabled` | `OPENCODE_DISABLE_LOGS` (inverted) | | `capturePromptInLogs` | `OPENCODE_CAPTURE_PROMPT_IN_LOGS` | | `captureModelContext` | `OPENCODE_CAPTURE_MODEL_CONTEXT` | +| `combineReasoningTokens` | `OPENCODE_COMBINE_REASONING_TOKENS` | | `logLevel` | *(none — option only)*: `debug`, `info`, `warn`, `error` | | `endpoint` | `OPENCODE_OTLP_ENDPOINT` | | `protocol` | `OPENCODE_OTLP_PROTOCOL` | diff --git a/src/config.ts b/src/config.ts index 887865f..a0425a9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -15,6 +15,7 @@ export type PluginConfig = { logsEnabled: boolean capturePromptInLogs: boolean captureModelContext: boolean + combineReasoningTokens: boolean logLevel: string | undefined endpoint: string protocol: "grpc" | "http/protobuf" | "http/json" @@ -61,6 +62,7 @@ export type OtelPluginOptions = { logsEnabled?: boolean capturePromptInLogs?: boolean captureModelContext?: boolean + combineReasoningTokens?: boolean logLevel?: string endpoint?: string protocol?: "grpc" | "http/protobuf" | "http/json" @@ -199,6 +201,7 @@ export function loadConfig(options: OtelPluginOptions = {}): PluginConfig { logsEnabled: pickBoolean(resolvedOptions.logsEnabled) ?? !hasNonEmptyEnv("OPENCODE_DISABLE_LOGS"), capturePromptInLogs: pickBoolean(resolvedOptions.capturePromptInLogs) ?? hasNonEmptyEnv("OPENCODE_CAPTURE_PROMPT_IN_LOGS"), captureModelContext: pickBoolean(resolvedOptions.captureModelContext) ?? hasNonEmptyEnv("OPENCODE_CAPTURE_MODEL_CONTEXT"), + combineReasoningTokens: pickBoolean(resolvedOptions.combineReasoningTokens) ?? hasNonEmptyEnv("OPENCODE_COMBINE_REASONING_TOKENS"), logLevel: pickString(resolvedOptions.logLevel), endpoint: pickString(resolvedOptions.endpoint) ?? process.env["OPENCODE_OTLP_ENDPOINT"] ?? "http://localhost:4317", protocol, diff --git a/src/handlers/step.ts b/src/handlers/step.ts index b87d11b..3bf155c 100644 --- a/src/handlers/step.ts +++ b/src/handlers/step.ts @@ -35,6 +35,7 @@ import { markSeen, resolveRunContext, setBoundedMap, + type TokenInfo, } from "../util.ts" const OPENINFERENCE_SPAN_KIND = SemanticConventions.OPENINFERENCE_SPAN_KIND @@ -44,6 +45,10 @@ function billedTokens(tokens: { input: number; output: number; reasoning: number return (tokens?.input ?? 0) + (tokens?.output ?? 0) + (tokens?.reasoning ?? 0) } +function completionTokens(tokens: TokenInfo, ctx: HandlerContext): number { + return ctx.combineReasoningTokens ? tokens.output + tokens.reasoning : tokens.output +} + function allTokens(tokens: { input: number; output: number; reasoning: number; cache: { read: number; write: number } } | undefined): number { if (!tokens) return 0 return tokens.input + tokens.output + tokens.reasoning + tokens.cache.read + tokens.cache.write @@ -181,7 +186,7 @@ export function handleStepEnded(e: EventOf<"session.step.ended">, ctx: HandlerCo [AGENT_NAME]: agent, "agent.type": agentType, [LLM_TOKEN_COUNT_PROMPT]: d.tokens.input, - [LLM_TOKEN_COUNT_COMPLETION]: d.tokens.output, + [LLM_TOKEN_COUNT_COMPLETION]: completionTokens(d.tokens, ctx), [LLM_TOKEN_COUNT_COMPLETION_DETAILS_REASONING]: d.tokens.reasoning, [LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHE_READ]: d.tokens.cache.read, [LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHE_WRITE]: d.tokens.cache.write, @@ -217,8 +222,8 @@ export function handleStepEnded(e: EventOf<"session.step.ended">, ctx: HandlerCo cost_usd: d.cost, duration_ms: durationMs, input_tokens: d.tokens.input, - output_tokens: d.tokens.output, - reasoning_tokens: d.tokens.reasoning, + output_tokens: completionTokens(d.tokens, ctx), + ...(ctx.combineReasoningTokens ? {} : { reasoning_tokens: d.tokens.reasoning }), cache_read_tokens: d.tokens.cache.read, cache_creation_tokens: d.tokens.cache.write, ...ctx.commonAttrs, @@ -252,7 +257,7 @@ export function handleStepFailed(e: EventOf<"session.step.failed">, ctx: Handler duration_ms: durationMs, ...(d.tokens ? { [LLM_TOKEN_COUNT_PROMPT]: d.tokens.input, - [LLM_TOKEN_COUNT_COMPLETION]: d.tokens.output, + [LLM_TOKEN_COUNT_COMPLETION]: completionTokens(d.tokens, ctx), [LLM_TOKEN_COUNT_COMPLETION_DETAILS_REASONING]: d.tokens.reasoning, [LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHE_READ]: d.tokens.cache.read, [LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHE_WRITE]: d.tokens.cache.write, @@ -298,8 +303,8 @@ function recordUsageMetrics( if (tokens && isMetricEnabled("token.usage", ctx)) { const { tokenCounter } = ctx.instruments tokenCounter.add(tokens.input, { ...base, type: "input" }) - tokenCounter.add(tokens.output, { ...base, type: "output" }) - tokenCounter.add(tokens.reasoning, { ...base, type: "reasoning" }) + tokenCounter.add(completionTokens(tokens, ctx), { ...base, type: "output" }) + if (!ctx.combineReasoningTokens) tokenCounter.add(tokens.reasoning, { ...base, type: "reasoning" }) tokenCounter.add(tokens.cache.read, { ...base, type: "cacheRead" }) tokenCounter.add(tokens.cache.write, { ...base, type: "cacheCreation" }) } diff --git a/src/plugin.ts b/src/plugin.ts index a416d61..2ecb0e9 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -103,6 +103,7 @@ export async function setup(ctx: OpenCodeContext): Promise<() => Promise> commonAttrs: parseAttributePairs(config.spanAttributes), disabledMetrics: config.disabledMetrics, disabledTraces: config.disabledTraces, + combineReasoningTokens: config.combineReasoningTokens, tracer: shared.tracer, tracePrefix: config.metricPrefix, rootContext: remoteContext ? () => remoteContext : () => ROOT_CONTEXT, diff --git a/src/types.ts b/src/types.ts index 07468e0..5c4ec11 100644 --- a/src/types.ts +++ b/src/types.ts @@ -182,6 +182,7 @@ export type HandlerContext = { commonAttrs: CommonAttrs disabledMetrics: Set disabledTraces: Set + combineReasoningTokens: boolean tracer: Tracer tracePrefix: string rootContext: () => Context diff --git a/tests/config.test.ts b/tests/config.test.ts index 7b6a8dc..f3d0d5c 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -76,6 +76,7 @@ describe("loadConfig", () => { "OPENCODE_DISABLE_LOGS", "OPENCODE_CAPTURE_PROMPT_IN_LOGS", "OPENCODE_CAPTURE_MODEL_CONTEXT", + "OPENCODE_COMBINE_REASONING_TOKENS", "OPENCODE_DISABLE_TRACES", "OPENCODE_TRACE_PROPAGATION_PROVIDERS", "OTEL_EXPORTER_OTLP_HEADERS", @@ -185,6 +186,14 @@ describe("loadConfig", () => { expect(loadConfig({ captureModelContext: false }).captureModelContext).toBe(false) }) + test("combines reasoning tokens only when explicitly enabled", () => { + expect(loadConfig().combineReasoningTokens).toBe(false) + expect(loadConfig({ combineReasoningTokens: true }).combineReasoningTokens).toBe(true) + process.env["OPENCODE_COMBINE_REASONING_TOKENS"] = "1" + expect(loadConfig().combineReasoningTokens).toBe(true) + expect(loadConfig({ combineReasoningTokens: false }).combineReasoningTokens).toBe(false) + }) + test("does not set OTEL_EXPORTER_OTLP_HEADERS when OPENCODE_OTLP_HEADERS is unset", () => { delete process.env["OPENCODE_OTLP_HEADERS"] loadConfig() diff --git a/tests/handlers/step.test.ts b/tests/handlers/step.test.ts index ee36269..14a367b 100644 --- a/tests/handlers/step.test.ts +++ b/tests/handlers/step.test.ts @@ -59,6 +59,37 @@ describe("handleStepEnded", () => { expect(llm.attributes["llm.output_messages"]).toBe('[{"role":"assistant","content":"hello\\nworld"}]') }) + test("reports reasoning separately from output by default", () => { + const { ctx, tracer, counters, logger } = makeCtx() + seedSession(ctx) + handleStepStarted(stepStarted(), ctx) + handleStepEnded(evt("session.step.ended", { sessionID: "ses_1", assistantMessageID: "msg_1", finish: "stop", cost: 0, tokens: tokens(10, 5, 2) }), ctx) + const byType = Object.fromEntries(counters.token.calls.map((c) => [c.attrs["type"], c.value])) + expect(byType["output"]).toBe(5) + expect(byType["reasoning"]).toBe(2) + const attrs = logger.records.at(-1)!.attributes! + expect(attrs["output_tokens"]).toBe(5) + expect(attrs["reasoning_tokens"]).toBe(2) + expect(tracer.spans.find((s) => s.name === "opencode.llm")!.attributes["llm.token_count.completion"]).toBe(5) + }) + + test("folds reasoning into output when combineReasoningTokens is enabled", () => { + const { ctx, tracer, counters, logger } = makeCtx("proj_test", [], [], true, {}, true) + seedSession(ctx) + handleStepStarted(stepStarted(), ctx) + handleStepEnded(evt("session.step.ended", { sessionID: "ses_1", assistantMessageID: "msg_1", finish: "stop", cost: 0, tokens: tokens(10, 5, 2, 3, 1) }), ctx) + const tokenTypes = counters.token.calls.map((c) => c.attrs["type"]) + expect(tokenTypes).toEqual(["input", "output", "cacheRead", "cacheCreation"]) + expect(counters.token.calls.find((c) => c.attrs["type"] === "output")!.value).toBe(7) + const attrs = logger.records.at(-1)!.attributes! + expect(attrs["output_tokens"]).toBe(7) + expect(attrs).not.toHaveProperty("reasoning_tokens") + const llm = tracer.spans.find((s) => s.name === "opencode.llm")! + expect(llm.attributes["llm.token_count.completion"]).toBe(7) + expect(llm.attributes["llm.token_count.completion_details.reasoning"]).toBe(2) + expect(llm.attributes["llm.token_count.total"]).toBe(21) + }) + test("counts a message once across multiple steps", () => { const { ctx, counters } = makeCtx() seedSession(ctx) @@ -90,6 +121,19 @@ describe("handleStepFailed", () => { expect(counters.modelUsage.calls).toHaveLength(1) }) + test("folds reasoning into output for failed steps when combineReasoningTokens is enabled", () => { + const { ctx, tracer, counters } = makeCtx("proj_test", [], [], true, {}, true) + seedSession(ctx) + handleStepStarted(stepStarted(), ctx) + handleStepFailed( + evt("session.step.failed", { sessionID: "ses_1", assistantMessageID: "msg_1", error: { type: "x", message: "y" }, tokens: tokens(10, 5, 2) }), + ctx, + ) + expect(counters.token.calls.some((c) => c.attrs["type"] === "reasoning")).toBe(false) + expect(counters.token.calls.find((c) => c.attrs["type"] === "output")!.value).toBe(7) + expect(tracer.spans.find((s) => s.name === "opencode.llm")!.attributes["llm.token_count.completion"]).toBe(7) + }) + test("records a failed step cost even when token usage is absent", () => { const { ctx, counters } = makeCtx() handleStepFailed(evt("session.step.failed", { sessionID: "s", assistantMessageID: "m", error: { type: "x", message: "y" }, cost: 2 }), ctx) diff --git a/tests/helpers.ts b/tests/helpers.ts index 09175ee..2af8128 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -200,6 +200,7 @@ export function makeCtx( disabledTraces: string[] = [], logsEnabled = true, extraCommonAttrs: Record = {}, + combineReasoningTokens = false, ): MockContext { const session = makeCounter() const token = makeCounter() @@ -244,6 +245,7 @@ export function makeCtx( commonAttrs: { "project.id": projectID, ...extraCommonAttrs }, disabledMetrics: new Set(disabledMetrics), disabledTraces: new Set(disabledTraces), + combineReasoningTokens, tracer: tracer as unknown as Tracer, tracePrefix: "opencode.", rootContext: () => ROOT_CONTEXT,