Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ After each Codex turn, the plugin reads the session's rollout transcript and upl

- **Turn** (`Codex Turn`, an [agent observation](https://langfuse.com/docs/observability/features/observation-types)) — one trace per turn, from your prompt to the final answer.
- **Generations** — one per model response within the turn, named `LLM` (or `LLM Subagent` inside subagent threads), with the model recorded on the observation plus reasoning, assistant text, the tool calls it requested, and token usage.
- **Tool calls** — shell commands, `apply_patch`, `spawn_agent`, MCP tools, web searches, etc., each with its input, output, and error status. Observation names include what was called — the shell command (`exec_command: git status`), the search query (`web_search: …`), or the MCP server and tool (`server.tool`) — and failed commands are flagged as errors.
- **Tool calls** — shell commands, `apply_patch`, `spawn_agent`, MCP tools, web searches, etc., each with its input, output, and error status. MCP calls are named `server.tool`, and failed commands are flagged as errors.
- **Subagents** — subagent threads are resolved from their own rollout files and nested under the spawning turn as `Codex Subagent Turn`.
- **Sessions** — all turns from one Codex session are grouped via the Codex thread id, so you can replay the whole session in Langfuse's [Sessions](https://langfuse.com/docs/observability/features/sessions) view.

Expand Down
46 changes: 6 additions & 40 deletions plugins/tracing/dist/index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -47011,50 +47011,16 @@ function buildGenerationOutput(step, clip) {
}));
return Object.keys(output).length > 0 ? output : void 0;
}
/** Tools that execute a shell command; the command makes a better span name. */
const SHELL_TOOL_NAMES = new Set([
"shell",
"exec_command",
"local_shell",
"container.exec"
]);
/** Collapse a value to a short single line usable inside a span name. */
function previewText(value, max = 60) {
const oneLine = value.trim().replace(/\s+/g, " ");
return oneLine.length > max ? `${oneLine.slice(0, max - 1)}…` : oneLine;
}
function extractCommandText(args) {
if (args == null || typeof args !== "object") return void 0;
const cmd = args.command ?? args.cmd;
if (typeof cmd === "string" && cmd.trim()) return cmd;
if (Array.isArray(cmd) && cmd.length > 0 && cmd.every((c) => typeof c === "string")) {
const parts = cmd;
if (parts.length === 3 && /^(ba|z|fi)?sh$/.test(parts[0]) && /^-l?c$/.test(parts[1])) return parts[2];
return parts.join(" ");
}
}
/**
* Observation name for a tool call: the tool name enriched with what it was
* called with (shell command, search query, MCP server/tool), so the trace
* tree shows *what* ran, not just which tool.
* Observation name for a tool call. MCP calls use the clean `server.tool`
* split from the mcp_tool_call_* events instead of the mangled function name;
* everything else uses the plain tool name. Call arguments (shell command,
* search query, …) stay out of the name — they belong to the observation
* input.
*/
function toolObservationName(tc) {
if (tc.mcp) return `${tc.mcp.server}.${tc.mcp.tool}`;
const name = tc.name || "tool";
if (SHELL_TOOL_NAMES.has(name)) {
const command = extractCommandText(tc.args);
if (command) return `${name}: ${previewText(command)}`;
}
if (name === "web_search" && tc.args != null && typeof tc.args === "object") {
const a = tc.args;
const detail = [
a.query,
a.url,
a.pattern
].find((v) => typeof v === "string" && v);
if (detail) return `${name}: ${previewText(detail)}`;
}
return name;
return tc.name || "tool";
}
/** Emit a single turn (and its subagents) as a Langfuse observation tree. */
async function emitTurn(turn, sessionMeta, ctx) {
Expand Down
45 changes: 6 additions & 39 deletions plugins/tracing/src/trace.ts
Original file line number Diff line number Diff line change
Expand Up @@ -155,49 +155,16 @@ function buildGenerationOutput(step: ModelStep, clip: Clip): Record<string, unkn
return Object.keys(output).length > 0 ? output : undefined;
}

/** Tools that execute a shell command; the command makes a better span name. */
const SHELL_TOOL_NAMES = new Set(["shell", "exec_command", "local_shell", "container.exec"]);

/** Collapse a value to a short single line usable inside a span name. */
function previewText(value: string, max = 60): string {
const oneLine = value.trim().replace(/\s+/g, " ");
return oneLine.length > max ? `${oneLine.slice(0, max - 1)}…` : oneLine;
}

function extractCommandText(args: unknown): string | undefined {
if (args == null || typeof args !== "object") return undefined;
const cmd =
(args as { command?: unknown; cmd?: unknown }).command ?? (args as { cmd?: unknown }).cmd;
if (typeof cmd === "string" && cmd.trim()) return cmd;
if (Array.isArray(cmd) && cmd.length > 0 && cmd.every((c) => typeof c === "string")) {
const parts = cmd as string[];
// Codex wraps most commands as ["bash", "-lc", "<script>"]; show the script.
if (parts.length === 3 && /^(ba|z|fi)?sh$/.test(parts[0]) && /^-l?c$/.test(parts[1])) {
return parts[2];
}
return parts.join(" ");
}
return undefined;
}

/**
* Observation name for a tool call: the tool name enriched with what it was
* called with (shell command, search query, MCP server/tool), so the trace
* tree shows *what* ran, not just which tool.
* Observation name for a tool call. MCP calls use the clean `server.tool`
* split from the mcp_tool_call_* events instead of the mangled function name;
* everything else uses the plain tool name. Call arguments (shell command,
* search query, …) stay out of the name — they belong to the observation
* input.
*/
function toolObservationName(tc: ToolCall): string {
if (tc.mcp) return `${tc.mcp.server}.${tc.mcp.tool}`;
const name = tc.name || "tool";
if (SHELL_TOOL_NAMES.has(name)) {
const command = extractCommandText(tc.args);
if (command) return `${name}: ${previewText(command)}`;
}
if (name === "web_search" && tc.args != null && typeof tc.args === "object") {
const a = tc.args as { query?: unknown; url?: unknown; pattern?: unknown };
const detail = [a.query, a.url, a.pattern].find((v) => typeof v === "string" && v);
if (detail) return `${name}: ${previewText(detail as string)}`;
}
return name;
return tc.name || "tool";
}

/** Emit a single turn (and its subagents) as a Langfuse observation tree. */
Expand Down
13 changes: 5 additions & 8 deletions plugins/tracing/test/trace.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ describe("convertRollout", () => {
// One tool span, nested under a generation, with the captured command output.
const tools = spans.filter((s) => obsType(s) === "tool");
expect(tools).toHaveLength(1);
expect(tools[0].name).toBe("exec_command: ls");
expect(tools[0].name).toBe("exec_command");
expect(attr(tools[0], "langfuse.observation.metadata.codex.tool_name")).toBe("exec_command");
expect(attr(tools[0], "langfuse.observation.output")).toContain("file1.txt");
expect(generations.map((g) => g.spanContext().spanId)).toContain(parentId(tools[0]));
Expand Down Expand Up @@ -149,16 +149,13 @@ describe("convertRollout", () => {
.filter((s) => obsType(s) === "tool")
.map((s) => s.name)
.sort();
expect(toolNames).toEqual([
"linear.create_issue",
"local_shell: git status",
"web_search: langfuse codex plugin",
]);
// Call arguments (command, query) stay out of the name — they are the input.
expect(toolNames).toEqual(["linear.create_issue", "local_shell", "web_search"]);

const webSearch = spans.find((s) => s.name.startsWith("web_search"))!;
const webSearch = spans.find((s) => s.name === "web_search")!;
expect(attr(webSearch, "langfuse.observation.input")).toContain("langfuse codex plugin");

const shell = spans.find((s) => s.name.startsWith("local_shell"))!;
const shell = spans.find((s) => s.name === "local_shell")!;
expect(attr(shell, "langfuse.observation.output")).toContain("clean");
});

Expand Down
Loading