From 42d88f315a2d24852a50b25db036e5b2bf62c6af Mon Sep 17 00:00:00 2001 From: lilpacy Date: Fri, 4 Sep 2026 05:08:17 +0900 Subject: [PATCH] feat: tag traces with skill: when a SKILL.md is read by a tool call --- plugins/tracing/dist/index.mjs | 23 +++++++++++++- plugins/tracing/src/trace.ts | 29 +++++++++++++++++- plugins/tracing/test/trace.test.ts | 49 +++++++++++++++++++++++++++++- 3 files changed, 98 insertions(+), 3 deletions(-) diff --git a/plugins/tracing/dist/index.mjs b/plugins/tracing/dist/index.mjs index f6887d3..bd64c4e 100644 --- a/plugins/tracing/dist/index.mjs +++ b/plugins/tracing/dist/index.mjs @@ -47116,6 +47116,26 @@ function emitToolCall(tc, parent, clip, fallbackEnd) { parentSpanContext: parent.otelSpan.spanContext() }).end(new Date(tc.endTime ?? fallbackEnd)); } +const SKILL_MD_PATH_RE = /(?:^|[\s"'`=(/])skills\/([A-Za-z0-9_.-]+)\/SKILL\.md/g; +/** +* Collect `skill:` tags for every skill whose SKILL.md was read by a +* tool call in this turn. Mirrors the Claude Code plugin's skill tags so the +* two projects share the same tag semantics: `skill:` means "read in the +* top-level conversation". Subagent rollouts are intentionally out of scope +* (the Claude plugin keeps those in a separate `subagent-skill:` namespace; +* add the equivalent here if subagent skill reads become worth tracking). +*/ +function collectSkillTags(turn) { + const tags = []; + for (const step of turn.steps) for (const call of step.toolCalls) { + const raw = typeof call.args === "string" ? call.args : toText(call.args ?? ""); + for (const match of raw.matchAll(SKILL_MD_PATH_RE)) { + const tag = `skill:${match[1]}`; + if (!tags.includes(tag)) tags.push(tag); + } + } + return tags; +} /** * Convert a Codex rollout file into Langfuse traces. * @@ -47139,11 +47159,12 @@ async function convertRollout(rolloutFile, options) { const turn = turns[turnIndex]; if (turn.completed && turn.turnId && uploaded.has(turn.turnId)) continue; const seededParent = await seededTraceParent(options.config, sessionMeta, turnIndex + 1); + const tags = [...options.config.tags ?? [], ...collectSkillTags(turn)]; await propagateAttributes({ sessionId: sessionMeta.sessionId, traceName: sessionMeta.isSubagentThread ? "Codex Subagent Turn" : "Codex Turn", ...options.config.user_id ? { userId: options.config.user_id } : {}, - ...options.config.tags ? { tags: options.config.tags } : {}, + ...tags.length ? { tags } : {}, ...options.config.metadata ? { metadata: options.config.metadata } : {} }, async () => { await emitTurn(turn, sessionMeta, { diff --git a/plugins/tracing/src/trace.ts b/plugins/tracing/src/trace.ts index 24b4f0e..f43d532 100644 --- a/plugins/tracing/src/trace.ts +++ b/plugins/tracing/src/trace.ts @@ -320,6 +320,32 @@ function emitToolCall( tool.end(new Date(tc.endTime ?? fallbackEnd)); } +// Matches a skill's SKILL.md path anywhere inside tool-call arguments (shell +// commands read skills as e.g. `cat ~/.codex/skills//SKILL.md`). +const SKILL_MD_PATH_RE = /(?:^|[\s"'`=(/])skills\/([A-Za-z0-9_.-]+)\/SKILL\.md/g; + +/** + * Collect `skill:` tags for every skill whose SKILL.md was read by a + * tool call in this turn. Mirrors the Claude Code plugin's skill tags so the + * two projects share the same tag semantics: `skill:` means "read in the + * top-level conversation". Subagent rollouts are intentionally out of scope + * (the Claude plugin keeps those in a separate `subagent-skill:` namespace; + * add the equivalent here if subagent skill reads become worth tracking). + */ +export function collectSkillTags(turn: Turn): string[] { + const tags: string[] = []; + for (const step of turn.steps) { + for (const call of step.toolCalls) { + const raw = typeof call.args === "string" ? call.args : toText(call.args ?? ""); + for (const match of raw.matchAll(SKILL_MD_PATH_RE)) { + const tag = `skill:${match[1]}`; + if (!tags.includes(tag)) tags.push(tag); + } + } + } + return tags; +} + /** * Convert a Codex rollout file into Langfuse traces. * @@ -358,12 +384,13 @@ export async function convertRollout( // skipped by dedup above) so the derived id is stable across hook runs. const seededParent = await seededTraceParent(options.config, sessionMeta, turnIndex + 1); + const tags = [...(options.config.tags ?? []), ...collectSkillTags(turn)]; await propagateAttributes( { sessionId: sessionMeta.sessionId, traceName: sessionMeta.isSubagentThread ? "Codex Subagent Turn" : "Codex Turn", ...(options.config.user_id ? { userId: options.config.user_id } : {}), - ...(options.config.tags ? { tags: options.config.tags } : {}), + ...(tags.length ? { tags } : {}), ...(options.config.metadata ? { metadata: options.config.metadata } : {}), }, async () => { diff --git a/plugins/tracing/test/trace.test.ts b/plugins/tracing/test/trace.test.ts index c3f66a9..05f42ca 100644 --- a/plugins/tracing/test/trace.test.ts +++ b/plugins/tracing/test/trace.test.ts @@ -13,7 +13,7 @@ import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"; import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import type { Config } from "../src/config.js"; -import { convertRollout } from "../src/trace.js"; +import { collectSkillTags, convertRollout } from "../src/trace.js"; const exporter = new InMemorySpanExporter(); let provider: NodeTracerProvider; @@ -330,3 +330,50 @@ describe("deterministic trace ids (trace_seed)", () => { expect(roots[0].spanContext().traceId).toBe(seededTraceId(`${seed}:2`)); }); }); + +describe("collectSkillTags", () => { + const makeTurn = (calls: unknown[]): Parameters[0] => + ({ + startTime: 0, + endTime: 1, + steps: [ + { + startTime: 0, + endTime: 1, + toolCalls: calls.map((args, i) => ({ + callId: `c${i}`, + name: "shell", + args, + startTime: 0, + })), + }, + ], + subagentThreadIds: [], + completed: true, + aborted: false, + }) as Parameters[0]; + + it("tags a SKILL.md read from shell command args", () => { + const turn = makeTurn([ + { command: ["bash", "-lc", "cat /Users/u/.codex/skills/goal-first/SKILL.md"] }, + ]); + expect(collectSkillTags(turn)).toEqual(["skill:goal-first"]); + }); + + it("dedupes repeated reads and collects multiple skills", () => { + const turn = makeTurn([ + { command: ["cat", "/home/u/dotfiles/codex/skills/goal-first/SKILL.md"] }, + { command: ["sed", "-n", "1,40p", "skills/goal-first/SKILL.md"] }, + { command: ["cat", "skills/structured-answer/SKILL.md"] }, + ]); + expect(collectSkillTags(turn)).toEqual(["skill:goal-first", "skill:structured-answer"]); + }); + + it("ignores non-SKILL.md paths and unrelated commands", () => { + const turn = makeTurn([ + { command: ["cat", "skills/goal-first/references/deep.md"] }, + { command: ["ls", "src/"] }, + ]); + expect(collectSkillTags(turn)).toEqual([]); + }); +});