Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion plugins/tracing/dist/index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -47116,6 +47116,26 @@ function emitToolCall(tc, parent, clip, fallbackEnd) {
parentSpanContext: parent.otelSpan.spanContext()
}).end(new Date(tc.endTime ?? fallbackEnd));
}
const SKILL_MD_PATH_RE = /(?:^|[\s"'`=(/])skills\/([A-Za-z0-9_.-]+)\/SKILL\.md/g;
/**
* Collect `skill:<name>` tags for every skill whose SKILL.md was read by a
* tool call in this turn. Mirrors the Claude Code plugin's skill tags so the
* two projects share the same tag semantics: `skill:` means "read in the
* top-level conversation". Subagent rollouts are intentionally out of scope
* (the Claude plugin keeps those in a separate `subagent-skill:` namespace;
* add the equivalent here if subagent skill reads become worth tracking).
*/
function collectSkillTags(turn) {
const tags = [];
for (const step of turn.steps) for (const call of step.toolCalls) {
const raw = typeof call.args === "string" ? call.args : toText(call.args ?? "");
for (const match of raw.matchAll(SKILL_MD_PATH_RE)) {
const tag = `skill:${match[1]}`;
if (!tags.includes(tag)) tags.push(tag);
}
}
return tags;
}
/**
* Convert a Codex rollout file into Langfuse traces.
*
Expand All @@ -47139,11 +47159,12 @@ async function convertRollout(rolloutFile, options) {
const turn = turns[turnIndex];
if (turn.completed && turn.turnId && uploaded.has(turn.turnId)) continue;
const seededParent = await seededTraceParent(options.config, sessionMeta, turnIndex + 1);
const tags = [...options.config.tags ?? [], ...collectSkillTags(turn)];
await propagateAttributes({
sessionId: sessionMeta.sessionId,
traceName: sessionMeta.isSubagentThread ? "Codex Subagent Turn" : "Codex Turn",
...options.config.user_id ? { userId: options.config.user_id } : {},
...options.config.tags ? { tags: options.config.tags } : {},
...tags.length ? { tags } : {},
...options.config.metadata ? { metadata: options.config.metadata } : {}
}, async () => {
await emitTurn(turn, sessionMeta, {
Expand Down
29 changes: 28 additions & 1 deletion plugins/tracing/src/trace.ts
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,32 @@ function emitToolCall(
tool.end(new Date(tc.endTime ?? fallbackEnd));
}

// Matches a skill's SKILL.md path anywhere inside tool-call arguments (shell
// commands read skills as e.g. `cat ~/.codex/skills/<name>/SKILL.md`).
const SKILL_MD_PATH_RE = /(?:^|[\s"'`=(/])skills\/([A-Za-z0-9_.-]+)\/SKILL\.md/g;

/**
* Collect `skill:<name>` tags for every skill whose SKILL.md was read by a
* tool call in this turn. Mirrors the Claude Code plugin's skill tags so the
* two projects share the same tag semantics: `skill:` means "read in the
* top-level conversation". Subagent rollouts are intentionally out of scope
* (the Claude plugin keeps those in a separate `subagent-skill:` namespace;
* add the equivalent here if subagent skill reads become worth tracking).
*/
export function collectSkillTags(turn: Turn): string[] {
const tags: string[] = [];
for (const step of turn.steps) {
for (const call of step.toolCalls) {
const raw = typeof call.args === "string" ? call.args : toText(call.args ?? "");
for (const match of raw.matchAll(SKILL_MD_PATH_RE)) {
const tag = `skill:${match[1]}`;
if (!tags.includes(tag)) tags.push(tag);
}
}
}
return tags;
}

/**
* Convert a Codex rollout file into Langfuse traces.
*
Expand Down Expand Up @@ -358,12 +384,13 @@ export async function convertRollout(
// skipped by dedup above) so the derived id is stable across hook runs.
const seededParent = await seededTraceParent(options.config, sessionMeta, turnIndex + 1);

const tags = [...(options.config.tags ?? []), ...collectSkillTags(turn)];
await propagateAttributes(
{
sessionId: sessionMeta.sessionId,
traceName: sessionMeta.isSubagentThread ? "Codex Subagent Turn" : "Codex Turn",
...(options.config.user_id ? { userId: options.config.user_id } : {}),
...(options.config.tags ? { tags: options.config.tags } : {}),
...(tags.length ? { tags } : {}),
...(options.config.metadata ? { metadata: options.config.metadata } : {}),
},
async () => {
Expand Down
49 changes: 48 additions & 1 deletion plugins/tracing/test/trace.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";

import type { Config } from "../src/config.js";
import { convertRollout } from "../src/trace.js";
import { collectSkillTags, convertRollout } from "../src/trace.js";

const exporter = new InMemorySpanExporter();
let provider: NodeTracerProvider;
Expand Down Expand Up @@ -330,3 +330,50 @@ describe("deterministic trace ids (trace_seed)", () => {
expect(roots[0].spanContext().traceId).toBe(seededTraceId(`${seed}:2`));
});
});

describe("collectSkillTags", () => {
const makeTurn = (calls: unknown[]): Parameters<typeof collectSkillTags>[0] =>
({
startTime: 0,
endTime: 1,
steps: [
{
startTime: 0,
endTime: 1,
toolCalls: calls.map((args, i) => ({
callId: `c${i}`,
name: "shell",
args,
startTime: 0,
})),
},
],
subagentThreadIds: [],
completed: true,
aborted: false,
}) as Parameters<typeof collectSkillTags>[0];

it("tags a SKILL.md read from shell command args", () => {
const turn = makeTurn([
{ command: ["bash", "-lc", "cat /Users/u/.codex/skills/goal-first/SKILL.md"] },
]);
expect(collectSkillTags(turn)).toEqual(["skill:goal-first"]);
});

it("dedupes repeated reads and collects multiple skills", () => {
const turn = makeTurn([
{ command: ["cat", "/home/u/dotfiles/codex/skills/goal-first/SKILL.md"] },
{ command: ["sed", "-n", "1,40p", "skills/goal-first/SKILL.md"] },
{ command: ["cat", "skills/structured-answer/SKILL.md"] },
]);
expect(collectSkillTags(turn)).toEqual(["skill:goal-first", "skill:structured-answer"]);
});

it("ignores non-SKILL.md paths and unrelated commands", () => {
const turn = makeTurn([
{ command: ["cat", "skills/goal-first/references/deep.md"] },
{ command: ["ls", "src/"] },
]);
expect(collectSkillTags(turn)).toEqual([]);
});
});