Skip to content

fix: normalize cached and reasoning usage details #28

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
NeuralEmpowerment wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: normalize cached and reasoning usage details #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions plugins/tracing/dist/index.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46984,11 +46984,13 @@ async function seededTraceParent(config$1, sessionMeta, turnNumber) {
function toUsageDetails(usage) {
if (!usage) return void 0;
const details = {};
if (typeof usage.input_tokens === "number") details.input = usage.input_tokens;
if (typeof usage.output_tokens === "number") details.output = usage.output_tokens;
const cachedInputTokens = usage.cached_input_tokens ?? 0;
const reasoningOutputTokens = usage.reasoning_output_tokens ?? 0;
if (typeof usage.input_tokens === "number") details.input = Math.max(usage.input_tokens - cachedInputTokens, 0);
if (cachedInputTokens > 0) details.cache_read_input_tokens = cachedInputTokens;
if (typeof usage.output_tokens === "number") details.output = Math.max(usage.output_tokens - reasoningOutputTokens, 0);
if (reasoningOutputTokens > 0) details.output_reasoning_tokens = reasoningOutputTokens;
if (typeof usage.total_tokens === "number") details.total = usage.total_tokens;
if (typeof usage.cached_input_tokens === "number") details.cache_read_input_tokens = usage.cached_input_tokens;
if (typeof usage.reasoning_output_tokens === "number") details.reasoning_tokens = usage.reasoning_output_tokens;
return Object.keys(details).length > 0 ? details : void 0;
}
/** Build a clip() that truncates long strings to `maxChars`. */
Expand Down
23 changes: 16 additions & 7 deletions plugins/tracing/src/trace.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,15 +112,24 @@ async function seededTraceParent(
function toUsageDetails(usage: TokenUsage | undefined): Record<string, number> | undefined {
if (!usage) return undefined;
const details: Record<string, number> = {};
if (typeof usage.input_tokens === "number") details.input = usage.input_tokens;
if (typeof usage.output_tokens === "number") details.output = usage.output_tokens;
if (typeof usage.total_tokens === "number") details.total = usage.total_tokens;
if (typeof usage.cached_input_tokens === "number") {
details.cache_read_input_tokens = usage.cached_input_tokens;
const cachedInputTokens = usage.cached_input_tokens ?? 0;
const reasoningOutputTokens = usage.reasoning_output_tokens ?? 0;

// Codex reports cached and reasoning tokens as subsets of input/output. Langfuse
// pricing treats usage-detail keys as additive buckets, so make them exclusive.
if (typeof usage.input_tokens === "number") {
details.input = Math.max(usage.input_tokens - cachedInputTokens, 0);
}
if (cachedInputTokens > 0) {
details.cache_read_input_tokens = cachedInputTokens;
}
if (typeof usage.reasoning_output_tokens === "number") {
details.reasoning_tokens = usage.reasoning_output_tokens;
if (typeof usage.output_tokens === "number") {
details.output = Math.max(usage.output_tokens - reasoningOutputTokens, 0);
}
if (reasoningOutputTokens > 0) {
details.output_reasoning_tokens = reasoningOutputTokens;
}
if (typeof usage.total_tokens === "number") details.total = usage.total_tokens;
return Object.keys(details).length > 0 ? details : undefined;
}

Expand Down
21 changes: 16 additions & 5 deletions plugins/tracing/test/trace.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,22 @@ describe("convertRollout", () => {
expect(parentId(gen)).toBe(root!.spanContext().spanId);
expect(attr(gen, "langfuse.observation.model.name")).toBe("gpt-5.4");
}
// First generation carries token usage.
const usage = generations
.map((g) => attr(g, "langfuse.observation.usage_details"))
.find((u) => u.includes("120"));
expect(usage, "expected usage details with 120 total tokens").toBeTruthy();
// Usage buckets must be mutually exclusive for Langfuse cost inference.
const usageDetails = generations.map(
(g) => JSON.parse(attr(g, "langfuse.observation.usage_details")) as Record<string, number>,
);
expect(usageDetails).toContainEqual({
input: 100,
output: 15,
output_reasoning_tokens: 5,
total: 120,
});
expect(usageDetails).toContainEqual({
input: 100,
cache_read_input_tokens: 50,
output: 30,
total: 180,
});

// One tool span, nested under a generation, with the captured command output.
const tools = spans.filter((s) => obsType(s) === "tool");
Expand Down