From f87be0232d600be68b327183c3e8318e20e5a647 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 22 Jun 2026 02:29:45 +0200 Subject: [PATCH] feat(dashboard): serve trace session artifacts --- apps/cli/src/commands/results/serve.ts | 174 ++++ apps/cli/test/commands/results/serve.test.ts | 319 +++++- apps/dashboard/package.json | 1 + apps/dashboard/src/lib/trace-read-model.ts | 871 +--------------- apps/dashboard/src/lib/types.ts | 162 +-- bun.lock | 9 +- .../evaluation/dashboard-trace-read-model.ts | 970 ++++++++++++++++++ packages/core/src/index.ts | 1 + 8 files changed, 1502 insertions(+), 1005 deletions(-) create mode 100644 packages/core/src/evaluation/dashboard-trace-read-model.ts diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index 315ae4969..2c7dcc980 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -13,6 +13,8 @@ * - GET /api/runs/:filename/phoenix-session — read linked Phoenix session data * - GET /api/runs/:filename/evals/:evalId/files/* — read artifact files as JSON, * or as raw/downloadable text with ?raw=1 / ?download=1 + * - GET /api/runs/:filename/evals/:evalId/trace-session — read an AgentV + * trace sidecar through the Dashboard trace/session read model * - GET /api/feedback — read feedback reviews * - POST /api/feedback — write feedback reviews * - GET /api/projects — list registered projects @@ -55,6 +57,7 @@ import { type EvaluationResult, type ExternalTraceMetadata, type ExternalTraceMetadataWire, + type TraceSessionResponse, addProject, externalTraceMetadataFromRecord, getProject, @@ -65,6 +68,7 @@ import { syncProjects, toExternalTraceMetadataWire, touchProject, + traceEnvelopeToTraceSessionResponse, } from '@agentv/core'; import type { Context } from 'hono'; import { Hono } from 'hono'; @@ -598,6 +602,74 @@ function missingTranscriptMessage(): string { ].join(' '); } +const TRACE_SESSION_ARTIFACT_RESPONSE_SCHEMA_VERSION = + 'agentv.dashboard.trace_artifact.v1' as const; + +type TraceSessionArtifactStatus = + | 'ok' + | 'missing' + | 'unsupported' + | 'dangling' + | 'invalid' + | 'rejected'; + +type TraceSessionArtifactResponse = + | { + schema_version: typeof TRACE_SESSION_ARTIFACT_RESPONSE_SCHEMA_VERSION; + status: 'ok'; + trace_path: string; + trace_session: TraceSessionResponse; + pointer?: string; + } + | { + schema_version: typeof TRACE_SESSION_ARTIFACT_RESPONSE_SCHEMA_VERSION; + status: Exclude; + message: string; + trace_path?: string; + pointer?: string; + }; + +type TraceSessionArtifactResponseInput = + | Omit, 'schema_version'> + | Omit, 'schema_version'>; + +function traceSessionArtifactResponse( + response: TraceSessionArtifactResponseInput, +): TraceSessionArtifactResponse { + return { + schema_version: TRACE_SESSION_ARTIFACT_RESPONSE_SCHEMA_VERSION, + ...response, + } as TraceSessionArtifactResponse; +} + +function missingTraceMessage(): string { + return [ + 'This result does not include canonical outputs/trace.json metadata.', + 'Dashboard trace sessions require an agentv.trace.v1 sidecar artifact.', + ].join(' '); +} + +function parseTraceArtifactJson(content: string): { value?: unknown; message?: string } { + try { + return { value: JSON.parse(content) }; + } catch (error) { + return { + message: `Trace artifact is not valid JSON: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } +} + +function supportedTraceSessionEnvelope(value: unknown): value is Record { + if (!isRecord(value)) { + return false; + } + const schemaVersion = nonEmptyString(value.schema_version); + const trace = isRecord(value.trace) ? value.trace : undefined; + return schemaVersion === 'agentv.trace.v1' && Array.isArray(trace?.spans); +} + function stripHeavyFields(results: readonly EvaluationResult[]) { return results.map((r) => { const { requests, trace, ...rest } = r as EvaluationResult & Record; @@ -1328,6 +1400,102 @@ async function handleEvalFileContent(c: C, { searchDir, projectId }: DataContext } } +async function handleEvalTraceSession(c: C, { searchDir, projectId }: DataContext) { + const filename = c.req.param('filename') ?? ''; + const evalId = c.req.param('evalId'); + const meta = await findRunById(searchDir, filename, projectId); + if (!meta) return c.json({ error: 'Run not found' }, 404); + + try { + const records = await parseManifestForMeta(searchDir, meta, projectId); + const record = records.find((r) => r.test_id === evalId); + if (!record) return c.json({ error: 'Eval not found' }, 404); + + const baseDir = path.dirname(meta.path); + const trace = resolveRecordArtifactPointer(record, 'trace'); + + if (!trace.path) { + return c.json( + traceSessionArtifactResponse({ + status: trace.unsupportedReason ? 'unsupported' : 'missing', + message: trace.unsupportedReason ?? missingTraceMessage(), + ...(trace.description && { pointer: trace.description }), + }), + ); + } + + const resolvedTrace = resolveReadableRunArtifactFile(baseDir, trace.path); + if (resolvedTrace.error) { + return c.json( + traceSessionArtifactResponse({ + status: 'rejected', + trace_path: trace.path, + message: resolvedTrace.error ?? 'Trace artifact path could not be resolved.', + ...(trace.description && { pointer: trace.description }), + }), + 403, + ); + } + + let content: string | undefined; + if (resolvedTrace.absolutePath) { + content = readFileSync(resolvedTrace.absolutePath, 'utf8'); + } else { + content = await readSidecarArtifactText(searchDir, projectId, meta, trace); + } + + if (content === undefined) { + const refMessage = trace.ref ? ` on ${trace.ref}` : ''; + return c.json( + traceSessionArtifactResponse({ + status: 'dangling', + trace_path: trace.path, + message: `Trace artifact pointer${refMessage} is present, but ${trace.path} is not available in this run workspace.`, + ...(trace.description && { pointer: trace.description }), + }), + ); + } + + const parsed = parseTraceArtifactJson(content); + if (parsed.message) { + return c.json( + traceSessionArtifactResponse({ + status: 'invalid', + trace_path: trace.path, + message: parsed.message, + ...(trace.description && { pointer: trace.description }), + }), + ); + } + + if (!supportedTraceSessionEnvelope(parsed.value)) { + return c.json( + traceSessionArtifactResponse({ + status: 'unsupported', + trace_path: trace.path, + message: + 'Trace artifact is not an agentv.trace.v1 envelope with trace.spans. Normalization is handled by the trace normalization pipeline.', + ...(trace.description && { pointer: trace.description }), + }), + ); + } + + return c.json( + traceSessionArtifactResponse({ + status: 'ok', + trace_path: trace.path, + trace_session: traceEnvelopeToTraceSessionResponse(parsed.value, { + runId: filename, + artifactPath: trace.path, + }), + ...(trace.description && { pointer: trace.description }), + }), + ); + } catch { + return c.json({ error: 'Failed to load trace artifact' }, 500); + } +} + async function handleEvalTranscript(c: C, { searchDir, projectId }: DataContext) { const filename = c.req.param('filename') ?? ''; const evalId = c.req.param('evalId'); @@ -2451,6 +2619,9 @@ export function createApp( handleCategorySuites(c, defaultCtx), ); app.get('/api/runs/:filename/evals/:evalId', (c) => handleEvalDetail(c, defaultCtx)); + app.get('/api/runs/:filename/evals/:evalId/trace-session', (c) => + handleEvalTraceSession(c, defaultCtx), + ); app.get('/api/runs/:filename/evals/:evalId/transcript', (c) => handleEvalTranscript(c, defaultCtx), ); @@ -2575,6 +2746,9 @@ export function createApp( app.get('/api/projects/:projectId/runs/:filename/evals/:evalId', (c) => withProject(c, handleEvalDetail), ); + app.get('/api/projects/:projectId/runs/:filename/evals/:evalId/trace-session', (c) => + withProject(c, handleEvalTraceSession), + ); app.get('/api/projects/:projectId/runs/:filename/evals/:evalId/transcript', (c) => withProject(c, handleEvalTranscript), ); diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index d2e992c8a..1af8b1b8b 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -93,6 +93,69 @@ function toJsonl(...records: object[]): string { return `${records.map((r) => JSON.stringify(r)).join('\n')}\n`; } +function traceSessionEnvelope(input?: { + runId?: string; + testId?: string; + target?: string; + spanName?: string; +}): object { + const testId = input?.testId ?? 'test-greeting'; + const target = input?.target ?? 'gpt-4o'; + return { + schema_version: 'agentv.trace.v1', + artifact_id: `${testId}-trace`, + created_at: '2026-03-25T10:00:00.000Z', + eval: { + run_id: input?.runId ?? 'trace-run', + test_id: testId, + suite: 'demo', + target, + }, + trace: { + format: 'otlp_openinference_spans', + trace_id: `${testId}-trace-id`, + root_span_id: 'root-span', + spans: [ + { + trace_id: `${testId}-trace-id`, + span_id: 'root-span', + parent_span_id: null, + name: input?.spanName ?? 'invoke_agent gpt-4o', + kind: 'INTERNAL', + start_time_unix_nano: '1000000000', + end_time_unix_nano: '1500000000', + status: { code: 'OK' }, + attributes: { + 'gen_ai.usage.input_tokens': 7, + 'gen_ai.usage.output_tokens': 5, + }, + events: [ + { + name: 'agentv.score', + time_unix_nano: '1400000000', + attributes: { + event_id: 'score-1', + score: 1, + text: 'Trace score', + passed: true, + }, + }, + ], + }, + ], + }, + source: { + kind: 'agentv_run', + provider: target, + format: 'agentv_result', + version: '1', + }, + artifacts: { + trace_path: 'outputs/trace.json', + }, + }; +} + function cleanGitEnv(): Record { const env: Record = {}; for (const [key, value] of Object.entries(process.env)) { @@ -3108,6 +3171,225 @@ describe('serve app', () => { }); }); + describe('GET /api/runs/:filename/evals/:evalId/trace-session', () => { + function writeLocalTraceRun( + projectDir: string, + experiment: string, + timestamp: string, + traceArtifactPath: string, + traceContent: string, + recordOverrides?: Record, + ): string { + const runId = `${experiment}::${timestamp}`; + const runDir = path.join(projectDir, '.agentv', 'results', 'runs', experiment, timestamp); + const tracePath = path.join(runDir, traceArtifactPath); + mkdirSync(path.dirname(tracePath), { recursive: true }); + writeFileSync(tracePath, traceContent); + writeFileSync( + path.join(runDir, 'index.jsonl'), + toJsonl({ + ...RESULT_A, + experiment, + trace_path: traceArtifactPath, + ...recordOverrides, + }), + ); + return runId; + } + + it('projects a local AgentV trace sidecar through the Dashboard read model', async () => { + const traceArtifactPath = 'demo/test-greeting/outputs/trace.json'; + const runId = 'with-trace::2026-03-25T09-00-00-000Z'; + writeLocalTraceRun( + tempDir, + 'with-trace', + '2026-03-25T09-00-00-000Z', + traceArtifactPath, + `${JSON.stringify(traceSessionEnvelope({ runId, spanName: 'local root' }))}\n`, + ); + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const res = await app.request( + `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/trace-session`, + ); + + expect(res.status).toBe(200); + const data = (await res.json()) as { + schema_version: string; + status: string; + trace_path: string; + trace_session: { + schema_version: string; + run_id?: string; + test_id?: string; + spans: Array<{ name: string; token_usage?: { input?: number } }>; + events: Array<{ kind: string; score?: number }>; + }; + }; + expect(data.schema_version).toBe('agentv.dashboard.trace_artifact.v1'); + expect(data.status).toBe('ok'); + expect(data.trace_path).toBe(traceArtifactPath); + expect(data.trace_session).toMatchObject({ + schema_version: 'agentv.dashboard.trace_session.v1', + run_id: runId, + test_id: 'test-greeting', + }); + expect(data.trace_session.spans[0]).toMatchObject({ + name: 'local root', + token_usage: { input: 7 }, + }); + expect(data.trace_session.events[0]).toMatchObject({ kind: 'score', score: 1 }); + }); + + it('returns equivalent payloads for unscoped and project-scoped routes', async () => { + const homedirSpy = spyOn(os, 'homedir').mockReturnValue(path.join(tempDir, 'home')); + try { + const projectDir = path.join(tempDir, 'project-one'); + mkdirSync(path.join(projectDir, '.agentv'), { recursive: true }); + const project = addProject(projectDir); + const traceArtifactPath = 'demo/test-greeting/outputs/trace.json'; + const runId = writeLocalTraceRun( + projectDir, + 'project-trace', + '2026-03-25T09-30-00-000Z', + traceArtifactPath, + `${JSON.stringify(traceSessionEnvelope({ spanName: 'project root' }))}\n`, + ); + + const app = createApp([], projectDir, projectDir, undefined, { studioDir }); + const unscoped = await app.request( + `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/trace-session`, + ); + const scoped = await app.request( + `/api/projects/${project.id}/runs/${encodeURIComponent(runId)}/evals/test-greeting/trace-session`, + ); + + expect(unscoped.status).toBe(200); + expect(scoped.status).toBe(200); + expect(await scoped.json()).toEqual(await unscoped.json()); + } finally { + homedirSpy.mockRestore(); + } + }); + + it('returns a typed missing state without breaking run detail', async () => { + const runId = writeLocalRunArtifact( + tempDir, + 'missing-trace', + '2026-03-25T10-30-00-000Z', + RESULT_A, + ); + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const traceRes = await app.request( + `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/trace-session`, + ); + expect(traceRes.status).toBe(200); + const traceData = (await traceRes.json()) as { + schema_version: string; + status: string; + message: string; + }; + expect(traceData.schema_version).toBe('agentv.dashboard.trace_artifact.v1'); + expect(traceData.status).toBe('missing'); + expect(traceData.message).toContain('outputs/trace.json'); + + const detailRes = await app.request(`/api/runs/${encodeURIComponent(runId)}`); + expect(detailRes.status).toBe(200); + const detailData = (await detailRes.json()) as { results: Array<{ testId: string }> }; + expect(detailData.results[0]?.testId).toBe('test-greeting'); + }); + + it('returns a typed dangling state when the trace pointer cannot be read', async () => { + const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'dangling-trace'); + const runId = 'dangling-trace::2026-03-25T10-45-00-000Z'; + const timestampDir = path.join(runsDir, '2026-03-25T10-45-00-000Z'); + const artifactPath = 'demo/test-greeting/outputs/trace.json'; + + mkdirSync(timestampDir, { recursive: true }); + writeFileSync( + path.join(timestampDir, 'index.jsonl'), + toJsonl({ + ...RESULT_A, + experiment: 'dangling-trace', + trace_path: artifactPath, + }), + ); + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const res = await app.request( + `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/trace-session`, + ); + + expect(res.status).toBe(200); + const data = (await res.json()) as { + status: string; + trace_path: string; + message: string; + }; + expect(data.status).toBe('dangling'); + expect(data.trace_path).toBe(artifactPath); + expect(data.message).toContain('not available'); + }); + + it('returns unsupported for trace artifacts that still need normalization', async () => { + const traceArtifactPath = 'demo/test-greeting/outputs/trace.json'; + const runId = writeLocalTraceRun( + tempDir, + 'unsupported-trace', + '2026-03-25T10-50-00-000Z', + traceArtifactPath, + `${JSON.stringify({ resourceSpans: [] })}\n`, + ); + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const res = await app.request( + `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/trace-session`, + ); + + expect(res.status).toBe(200); + const data = (await res.json()) as { + status: string; + trace_path: string; + message: string; + }; + expect(data.status).toBe('unsupported'); + expect(data.trace_path).toBe(traceArtifactPath); + expect(data.message).toContain('trace normalization pipeline'); + }); + + it('rejects trace artifact paths that escape the run workspace', async () => { + const secret = 'outside trace secret'; + writeFileSync(path.join(tempDir, 'outside-trace.json'), secret); + const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'escaped-trace'); + const runId = 'escaped-trace::2026-03-25T11-00-00-000Z'; + const timestampDir = path.join(runsDir, '2026-03-25T11-00-00-000Z'); + const artifactPath = '../../../../../outside-trace.json'; + + mkdirSync(timestampDir, { recursive: true }); + writeFileSync( + path.join(timestampDir, 'index.jsonl'), + toJsonl({ + ...RESULT_A, + experiment: 'escaped-trace', + trace_path: artifactPath, + }), + ); + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const res = await app.request( + `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/trace-session`, + ); + + expect(res.status).toBe(403); + const text = await res.text(); + expect(text).not.toContain(secret); + const data = JSON.parse(text) as { status: string; trace_path: string }; + expect(data.status).toBe('rejected'); + expect(data.trace_path).toBe(artifactPath); + }); + }); + describe('GET /api/runs/:filename/evals/:evalId/transcript', () => { it('loads canonical transcript JSONL lazily from the manifest pointer', async () => { const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'with-transcript'); @@ -3226,11 +3508,14 @@ describe('serve app', () => { role: 'assistant', content: 'sidecar transcript body', })}\n`; - const traceJson = `${JSON.stringify({ - schema_version: 'agentv.trace.v1', - test_id: 'test-greeting', - spans: [{ name: 'root' }], - })}\n`; + const traceJson = `${JSON.stringify( + traceSessionEnvelope({ + runId, + testId: 'test-greeting', + target: 'gpt-4o', + spanName: 'remote root', + }), + )}\n`; git(`git switch --quiet --orphan ${resultsBranch}`, seedDir); git('git rm -rf --quiet . 2>/dev/null || true', seedDir); @@ -3334,6 +3619,21 @@ describe('serve app', () => { expect(artifactRefLookup()).toBe('present'); expect(existsSync(path.join(cloneDir, ...transcriptKey.split('/')))).toBe(false); + const traceSessionRes = await app.request( + `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/trace-session`, + ); + expect(traceSessionRes.status).toBe(200); + const traceSessionData = (await traceSessionRes.json()) as { + status: string; + trace_path: string; + trace_session: { spans: Array<{ name: string }>; run_id?: string }; + }; + expect(traceSessionData.status).toBe('ok'); + expect(traceSessionData.trace_path).toBe(traceArtifactPath); + expect(traceSessionData.trace_session.run_id).toBe(runId); + expect(traceSessionData.trace_session.spans[0]?.name).toBe('remote root'); + expect(existsSync(path.join(cloneDir, ...traceKey.split('/')))).toBe(false); + const traceRes = await app.request( `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/files/${traceArtifactPath}?raw=1`, ); @@ -3482,12 +3782,14 @@ describe('serve app', () => { expect(data.answer_content).toBeUndefined(); }); - it('does not read transcript bodies for list, detail, or aggregate routes', async () => { + it('does not read transcript or trace bodies for list, detail, or aggregate routes', async () => { const timestamp = '2026-03-25T14-00-00-000Z'; const transcriptArtifactPath = 'demo/test-greeting/outputs/transcript.jsonl'; + const traceArtifactPath = 'demo/test-greeting/outputs/trace.json'; const runId = writeLocalRunArtifact(tempDir, 'lazy-guard', timestamp, { ...RESULT_A, transcript_path: transcriptArtifactPath, + trace_path: traceArtifactPath, }); const timestampDir = path.join( tempDir, @@ -3498,6 +3800,11 @@ describe('serve app', () => { timestamp, ); mkdirSync(path.join(timestampDir, transcriptArtifactPath), { recursive: true }); + mkdirSync(path.dirname(path.join(timestampDir, traceArtifactPath)), { recursive: true }); + writeFileSync( + path.join(timestampDir, traceArtifactPath), + 'malformed trace body that list routes must not parse', + ); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); diff --git a/apps/dashboard/package.json b/apps/dashboard/package.json index 34bdbacfb..281e0ced8 100644 --- a/apps/dashboard/package.json +++ b/apps/dashboard/package.json @@ -10,6 +10,7 @@ "test": "bun test" }, "dependencies": { + "@agentv/core": "workspace:*", "@monaco-editor/react": "^4.7.0", "@tanstack/react-query": "^5.75.5", "@tanstack/react-router": "^1.120.3", diff --git a/apps/dashboard/src/lib/trace-read-model.ts b/apps/dashboard/src/lib/trace-read-model.ts index ec34982a4..866eba697 100644 --- a/apps/dashboard/src/lib/trace-read-model.ts +++ b/apps/dashboard/src/lib/trace-read-model.ts @@ -1,860 +1,11 @@ -import type { - ExternalTraceMetadata, - TraceSessionArtifactLink, - TraceSessionConversionWarning, - TraceSessionEvent, - TraceSessionEventKind, - TraceSessionResponse, - TraceSessionScore, - TraceSessionSource, - TraceSessionSourceRef, - TraceSessionSpan, - TraceSessionTokenUsage, -} from './types'; - -export const TRACE_SESSION_SCHEMA_VERSION = 'agentv.dashboard.trace_session.v1' as const; - -export interface TraceSessionProjectionOptions { - runId?: string; - artifactPath?: string; -} - -export interface TraceSpanNode { - id: string; - spanId: string; - parentSpanId?: string | null; - span: TraceSessionSpan; - children: TraceSpanNode[]; - diagnostics?: TraceSpanTreeDiagnostic[]; -} - -export type TraceSpanTreeDiagnosticCode = - | 'cycle' - | 'duplicate_span_id' - | 'missing_parent' - | 'missing_span_id' - | 'self_parent'; - -export interface TraceSpanTreeDiagnostic { - code: TraceSpanTreeDiagnosticCode; - message: string; - span_id?: string; - node_id?: string; - parent_span_id?: string; -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function asRecord(value: unknown): Record | undefined { - return isRecord(value) ? value : undefined; -} - -function asArray(value: unknown): unknown[] { - return Array.isArray(value) ? value : []; -} - -function stringValue(value: unknown): string | undefined { - return typeof value === 'string' && value.length > 0 ? value : undefined; -} - -function finiteNumber(value: unknown): number | undefined { - return typeof value === 'number' && Number.isFinite(value) ? value : undefined; -} - -function finiteInteger(value: unknown): number | undefined { - return typeof value === 'number' && Number.isInteger(value) && value >= 0 ? value : undefined; -} - -function boolValue(value: unknown): boolean | undefined { - return typeof value === 'boolean' ? value : undefined; -} - -function dropUndefined>(value: T): T { - return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined)) as T; -} - -function compactRecord(value: Record): Record | undefined { - const compacted = dropUndefined(value); - return Object.keys(compacted).length > 0 ? compacted : undefined; -} - -function nonEmptyArray(value: readonly T[] | undefined): readonly T[] | undefined { - return value && value.length > 0 ? value : undefined; -} - -function unixNanoToIso(value: string | undefined): string | undefined { - if (!value) { - return undefined; - } - try { - return new Date(Number(BigInt(value) / 1_000_000n)).toISOString(); - } catch { - return undefined; - } -} - -function durationMsFromNanos( - start: string | undefined, - end: string | undefined, -): number | undefined { - if (!start || !end) { - return undefined; - } - try { - const startNanos = BigInt(start); - const endNanos = BigInt(end); - if (endNanos < startNanos) { - return undefined; - } - return Number(endNanos - startNanos) / 1_000_000; - } catch { - return undefined; - } -} - -function numberFromAttributes( - attributes: Record, - keys: readonly string[], -): number | undefined { - for (const key of keys) { - const value = finiteNumber(attributes[key]); - if (value !== undefined) { - return value; - } - } - return undefined; -} - -function tokenUsageFromAttributes( - attributes: Record | undefined, -): TraceSessionTokenUsage | undefined { - if (!attributes) { - return undefined; - } - - const nested = asRecord(attributes.token_usage); - const usage = compactRecord({ - input: - finiteNumber(nested?.input) ?? - numberFromAttributes(attributes, [ - 'gen_ai.usage.input_tokens', - 'llm.token_count.prompt', - 'input_tokens', - ]), - output: - finiteNumber(nested?.output) ?? - numberFromAttributes(attributes, [ - 'gen_ai.usage.output_tokens', - 'llm.token_count.completion', - 'output_tokens', - ]), - reasoning: - finiteNumber(nested?.reasoning) ?? - numberFromAttributes(attributes, [ - 'gen_ai.usage.reasoning.output_tokens', - 'reasoning_tokens', - ]), - cached: - finiteNumber(nested?.cached) ?? - numberFromAttributes(attributes, ['gen_ai.usage.cache_read.input_tokens', 'cached_tokens']), - total: finiteNumber(nested?.total) ?? numberFromAttributes(attributes, ['total_tokens']), - }); - - return usage as TraceSessionTokenUsage | undefined; -} - -function isExternalTraceKey(key: string): boolean { - return ( - key === 'external_trace' || - key.startsWith('external_trace_') || - key.startsWith('external_trace.') - ); -} - -function isCredentialLikeKey(key: string): boolean { - const normalized = key.toLowerCase(); - if ( - normalized === 'token_usage' || - normalized.endsWith('_tokens') || - normalized.endsWith('.tokens') || - normalized.includes('usage.') - ) { - return false; - } - return /(^|[._-])(api[._-]?key|authorization|bearer|password|secret|private[._-]?key|access[._-]?token|auth[._-]?token|client[._-]?secret|id[._-]?token|refresh[._-]?token|session[._-]?token|token)($|[._-])/.test( - normalized, - ); -} - -function sanitizeAttributeMap( - value: Record | undefined, -): Record | undefined { - if (!value) { - return undefined; - } - const entries = Object.entries(value).flatMap(([key, entry]) => { - if (isExternalTraceKey(key) || isCredentialLikeKey(key)) { - return []; - } - if (isRecord(entry)) { - const nested = sanitizeAttributeMap(entry); - return nested ? [[key, nested] as const] : []; - } - return [[key, entry] as const]; - }); - return entries.length > 0 ? Object.fromEntries(entries) : undefined; -} - -function spanStatusFromValue(value: unknown): TraceSessionSpan['status'] { - const record = asRecord(value); - if (!record) { - return undefined; - } - return compactRecord({ - code: - stringValue(record.code) ?? - (typeof record.code === 'number' ? String(record.code) : undefined), - message: stringValue(record.message), - }) as TraceSessionSpan['status']; -} - -function eventKind( - name: string, - attributes: Record | undefined, -): TraceSessionEventKind { - const lowerName = name.toLowerCase(); - if ( - lowerName.includes('score') || - finiteNumber(attributes?.score) !== undefined || - finiteNumber(attributes?.['agentv.score']) !== undefined || - finiteNumber(attributes?.['agentv.grader.score']) !== undefined - ) { - return 'score'; - } - if ( - lowerName.includes('annotation') || - stringValue(attributes?.text) !== undefined || - stringValue(attributes?.annotation) !== undefined || - stringValue(attributes?.['agentv.annotation.text']) !== undefined - ) { - return 'annotation'; - } - if (lowerName === 'exception') { - return 'exception'; - } - return 'event'; -} - -function scoreFromEvent(attributes: Record | undefined): number | undefined { - if (!attributes) { - return undefined; - } - return ( - finiteNumber(attributes.score) ?? - finiteNumber(attributes['agentv.score']) ?? - finiteNumber(attributes['agentv.grader.score']) - ); -} - -function textFromEvent(attributes: Record | undefined): string | undefined { - if (!attributes) { - return undefined; - } - return ( - stringValue(attributes.text) ?? - stringValue(attributes.annotation) ?? - stringValue(attributes['agentv.annotation.text']) ?? - stringValue(attributes['exception.message']) - ); -} - -function passedFromEvent(attributes: Record | undefined): boolean | undefined { - if (!attributes) { - return undefined; - } - return boolValue(attributes.passed) ?? boolValue(attributes['agentv.annotation.passed']); -} - -function eventId( - spanId: string, - index: number, - attributes: Record | undefined, -): string { - return ( - stringValue(attributes?.event_id) ?? - stringValue(attributes?.['agentv.event_id']) ?? - `${spanId}:event:${index}` - ); -} - -function projectSpanEvent( - spanId: string, - event: unknown, - index: number, -): TraceSessionEvent | undefined { - const record = asRecord(event); - if (!record) { - return undefined; - } - const name = stringValue(record.name); - if (!name) { - return undefined; - } - - const attributes = asRecord(record.attributes); - const safeAttributes = sanitizeAttributeMap(attributes); - return dropUndefined({ - event_id: eventId(spanId, index, attributes), - span_id: spanId, - name, - kind: eventKind(name, attributes), - time_unix_nano: stringValue(record.time_unix_nano), - timestamp: unixNanoToIso(stringValue(record.time_unix_nano)), - score: scoreFromEvent(attributes), - text: textFromEvent(attributes), - passed: passedFromEvent(attributes), - attributes: safeAttributes, - }); -} - -function projectSpan(span: unknown, index: number): TraceSessionSpan | undefined { - const record = asRecord(span); - if (!record) { - return undefined; - } - - const spanId = stringValue(record.span_id) ?? `span-${index}`; - const traceId = stringValue(record.trace_id); - const parentSpanId = record.parent_span_id === null ? null : stringValue(record.parent_span_id); - const attributes = asRecord(record.attributes); - const safeAttributes = sanitizeAttributeMap(attributes); - const startTimeUnixNano = stringValue(record.start_time_unix_nano); - const endTimeUnixNano = stringValue(record.end_time_unix_nano); - const events = asArray(record.events) - .map((event, eventIndex) => projectSpanEvent(spanId, event, eventIndex)) - .filter((event): event is TraceSessionEvent => event !== undefined); - - return dropUndefined({ - id: spanId, - trace_id: traceId, - span_id: spanId, - parent_span_id: parentSpanId, - name: stringValue(record.name) ?? spanId, - kind: stringValue(record.kind), - status: spanStatusFromValue(record.status), - start_time_unix_nano: startTimeUnixNano, - end_time_unix_nano: endTimeUnixNano, - start_time: unixNanoToIso(startTimeUnixNano), - end_time: unixNanoToIso(endTimeUnixNano), - duration_ms: durationMsFromNanos(startTimeUnixNano, endTimeUnixNano), - token_usage: tokenUsageFromAttributes(attributes), - attributes: safeAttributes, - events: events.length > 0 ? events : undefined, - }); -} - -function projectScores(scores: unknown): TraceSessionScore[] | undefined { - const projected: TraceSessionScore[] = []; - - for (const score of asArray(scores)) { - const record = asRecord(score); - const name = stringValue(record?.name); - const value = finiteNumber(record?.score); - if (!record || !name || value === undefined) { - continue; - } - projected.push( - dropUndefined({ - name, - type: stringValue(record.type), - score: value, - weight: finiteNumber(record.weight), - verdict: stringValue(record.verdict), - source: stringValue(record.source), - evaluated_at: stringValue(record.evaluated_at), - target_span_id: stringValue(record.target_span_id), - evidence: asRecord(record.evidence), - }) as TraceSessionScore, - ); - } - - return projected.length > 0 ? projected : undefined; -} - -const EXTERNAL_TRACE_KEYS = [ - 'provider', - 'source', - 'endpoint', - 'profile', - 'project', - 'project_id', - 'session_id', - 'session_node_id', - 'trace_id', - 'trace_node_id', - 'span_id', - 'span_node_id', - 'traceparent', - 'tracestate', - 'ui_url', - 'run_id', - 'test_id', - 'target', -] as const; - -function sanitizeUrl(value: unknown): string | undefined { - const raw = stringValue(value); - if (!raw) { - return undefined; - } - try { - const url = new URL(raw); - if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) { - return undefined; - } - url.search = ''; - url.hash = ''; - return url.toString(); - } catch { - return undefined; - } -} - -function sanitizeExternalTrace(value: unknown): ExternalTraceMetadata | undefined { - const record = asRecord(value); - if (!record) { - return undefined; - } - - const sanitized = compactRecord({ - provider: stringValue(record.provider), - source: stringValue(record.source), - endpoint: sanitizeUrl(record.endpoint), - profile: stringValue(record.profile), - project: stringValue(record.project), - project_id: stringValue(record.project_id) ?? stringValue(record.projectId), - session_id: stringValue(record.session_id) ?? stringValue(record.session), - session_node_id: - stringValue(record.session_node_id) ?? - stringValue(record.session_node) ?? - stringValue(record.node_id), - trace_id: stringValue(record.trace_id) ?? stringValue(record.trace), - trace_node_id: stringValue(record.trace_node_id) ?? stringValue(record.trace_node), - span_id: stringValue(record.span_id) ?? stringValue(record.span), - span_node_id: stringValue(record.span_node_id) ?? stringValue(record.span_node), - traceparent: stringValue(record.traceparent), - tracestate: stringValue(record.tracestate), - ui_url: sanitizeUrl(record.ui_url ?? record.url ?? record.href), - run_id: stringValue(record.run_id), - test_id: stringValue(record.test_id), - target: stringValue(record.target), - }) as ExternalTraceMetadata | undefined; - - return sanitized && EXTERNAL_TRACE_KEYS.some((key) => sanitized[key] !== undefined) - ? sanitized - : undefined; -} - -function externalTraceFromFlatMetadata( - metadata: Record | undefined, -): ExternalTraceMetadata | undefined { - if (!metadata) { - return undefined; - } - return sanitizeExternalTrace({ - provider: metadata.external_trace_provider ?? metadata['external_trace.provider'], - source: metadata.external_trace_source ?? metadata['external_trace.source'], - endpoint: metadata.external_trace_endpoint ?? metadata['external_trace.endpoint'], - profile: metadata.external_trace_profile ?? metadata['external_trace.profile'], - project: metadata.external_trace_project ?? metadata['external_trace.project'], - project_id: metadata.external_trace_project_id ?? metadata['external_trace.project_id'], - session_id: - metadata.external_trace_session_id ?? - metadata.external_trace_session ?? - metadata['external_trace.session_id'] ?? - metadata['external_trace.session'], - session_node_id: - metadata.external_trace_session_node_id ?? - metadata.external_trace_node_id ?? - metadata['external_trace.session_node_id'] ?? - metadata['external_trace.node_id'], - trace_id: - metadata.external_trace_trace_id ?? - metadata.external_trace_trace ?? - metadata['external_trace.trace_id'] ?? - metadata['external_trace.trace'], - trace_node_id: - metadata.external_trace_trace_node_id ?? metadata['external_trace.trace_node_id'], - span_id: - metadata.external_trace_span_id ?? - metadata.external_trace_span ?? - metadata['external_trace.span_id'] ?? - metadata['external_trace.span'], - span_node_id: metadata.external_trace_span_node_id ?? metadata['external_trace.span_node_id'], - traceparent: metadata.external_trace_traceparent ?? metadata['external_trace.traceparent'], - tracestate: metadata.external_trace_tracestate ?? metadata['external_trace.tracestate'], - ui_url: - metadata.external_trace_ui_url ?? - metadata.external_trace_url ?? - metadata['external_trace.ui_url'] ?? - metadata['external_trace.url'], - run_id: metadata.external_trace_run_id ?? metadata['external_trace.run_id'], - test_id: metadata.external_trace_test_id ?? metadata['external_trace.test_id'], - target: metadata.external_trace_target ?? metadata['external_trace.target'], - }); -} - -function sanitizeMetadataValue(value: unknown): unknown | undefined { - if (Array.isArray(value)) { - const sanitized = value - .map(sanitizeMetadataValue) - .filter((entry): entry is unknown => entry !== undefined); - return sanitized.length > 0 ? sanitized : undefined; - } - if (isRecord(value)) { - return sanitizeMetadata(value); - } - return value; -} - -function sanitizeMetadata( - value: Record | undefined, -): Record | undefined { - if (!value) { - return undefined; - } - const entries = Object.entries(value).flatMap(([key, entry]) => { - if (isExternalTraceKey(key) || isCredentialLikeKey(key)) { - return []; - } - const sanitized = sanitizeMetadataValue(entry); - return sanitized !== undefined ? [[key, sanitized] as const] : []; - }); - return entries.length > 0 ? Object.fromEntries(entries) : undefined; -} - -function sourceFromEnvelope( - source: Record | undefined, - artifactPath: string | undefined, -): TraceSessionSource | undefined { - if (!source && !artifactPath) { - return undefined; - } - return compactRecord({ - kind: stringValue(source?.kind), - path: stringValue(source?.path), - provider: stringValue(source?.provider), - format: stringValue(source?.format), - version: stringValue(source?.version), - artifact_path: artifactPath, - metadata: sanitizeMetadata(asRecord(source?.metadata)), - }) as TraceSessionSource | undefined; -} - -function safeArtifactPath(value: unknown): string | undefined { - const raw = stringValue(value); - if (!raw || raw.includes('\0')) { - return undefined; - } - - const normalized = raw.replace(/\\/g, '/'); - if (normalized.startsWith('/') || normalized.startsWith('//')) { - return undefined; - } - if (/^[a-z][a-z0-9+.-]*:/i.test(normalized)) { - return undefined; - } - if (normalized.split('/').includes('..')) { - return undefined; - } - return normalized; -} - -function projectArtifactLinks(artifacts: unknown): TraceSessionArtifactLink[] | undefined { - const record = asRecord(artifacts); - if (!record) { - return undefined; - } - - const links = Object.entries(record) - .flatMap(([name, value]) => { - if (!stringValue(name) || isCredentialLikeKey(name)) { - return []; - } - const artifactPath = safeArtifactPath(value); - return artifactPath ? [{ name, path: artifactPath }] : []; - }) - .sort((first, second) => first.name.localeCompare(second.name)); - - return links.length > 0 ? links : undefined; -} - -function projectSourceRef(value: unknown): TraceSessionSourceRef | undefined { - const record = asRecord(value); - if (!record) { - return undefined; - } - return compactRecord({ - event_id: stringValue(record.event_id), - message_id: stringValue(record.message_id), - span_id: stringValue(record.span_id), - trace_id: stringValue(record.trace_id), - raw_kind: stringValue(record.raw_kind), - path: safeArtifactPath(record.path), - line: finiteInteger(record.line), - metadata: sanitizeMetadata(asRecord(record.metadata)), - }) as TraceSessionSourceRef | undefined; -} - -function projectConversionWarnings(warnings: unknown): TraceSessionConversionWarning[] | undefined { - const projected: TraceSessionConversionWarning[] = []; - - for (const warning of asArray(warnings)) { - const record = asRecord(warning); - const code = stringValue(record?.code); - const message = stringValue(record?.message); - if (!record || !code || !message) { - continue; - } - projected.push( - dropUndefined({ - code, - severity: stringValue(record.severity), - span_id: stringValue(record.span_id), - source_ref: projectSourceRef(record.source_ref), - message, - details: sanitizeMetadata(asRecord(record.details)), - }) as TraceSessionConversionWarning, - ); - } - - return projected.length > 0 ? projected : undefined; -} - -function externalTraceFromEnvelope( - envelope: Record, -): ExternalTraceMetadata | undefined { - const source = asRecord(envelope.source); - const sourceMetadata = asRecord(source?.metadata); - const trace = asRecord(envelope.trace); - const rootSpanId = stringValue(trace?.root_span_id); - const rootSpan = asArray(trace?.spans) - .map(asRecord) - .find((span) => stringValue(span?.span_id) === rootSpanId); - const rootAttributes = asRecord(rootSpan?.attributes); - - return ( - sanitizeExternalTrace(envelope.external_trace) ?? - sanitizeExternalTrace(sourceMetadata?.external_trace) ?? - externalTraceFromFlatMetadata(sourceMetadata) ?? - externalTraceFromFlatMetadata(rootAttributes) - ); -} - -export function traceEnvelopeToTraceSessionResponse( - input: unknown, - options: TraceSessionProjectionOptions = {}, -): TraceSessionResponse { - const envelope = asRecord(input) ?? {}; - const evaluation = asRecord(envelope.eval); - const trace = asRecord(envelope.trace); - const spans = asArray(trace?.spans) - .map(projectSpan) - .filter((span): span is TraceSessionSpan => span !== undefined); - const events = spans.flatMap((span) => span.events ?? []); - - return dropUndefined({ - schema_version: TRACE_SESSION_SCHEMA_VERSION, - artifact_id: stringValue(envelope.artifact_id), - created_at: stringValue(envelope.created_at), - run_id: options.runId ?? stringValue(evaluation?.run_id), - test_id: stringValue(evaluation?.test_id), - suite: stringValue(evaluation?.suite), - target: stringValue(evaluation?.target), - trace_id: stringValue(trace?.trace_id), - root_span_id: stringValue(trace?.root_span_id), - source: sourceFromEnvelope(asRecord(envelope.source), options.artifactPath), - external_trace: externalTraceFromEnvelope(envelope), - artifact_links: projectArtifactLinks(envelope.artifacts), - conversion_warnings: projectConversionWarnings(envelope.conversion_warnings), - spans, - events, - scores: projectScores(envelope.scores), - }); -} - -export function buildTraceSpanTree(spans: readonly TraceSessionSpan[]): TraceSpanNode[] { - const nodes: TraceSpanNode[] = []; - const firstNodeBySpanId = new Map(); - const spanIdCounts = new Map(); - - spans.forEach((span, index) => { - const rawSpanId = stringValue(span.span_id); - const spanId = rawSpanId ?? `missing-span-${index}`; - const occurrence = (spanIdCounts.get(spanId) ?? 0) + 1; - spanIdCounts.set(spanId, occurrence); - - const node: TraceSpanNode = { - id: occurrence === 1 ? spanId : `${spanId}#${occurrence}`, - spanId, - parentSpanId: span.parent_span_id, - span, - children: [], - diagnostics: rawSpanId - ? undefined - : [ - { - code: 'missing_span_id', - message: 'Span was missing span_id and was assigned a stable node id.', - node_id: spanId, - }, - ], - }; - - if (occurrence > 1) { - addNodeDiagnostic(node, { - code: 'duplicate_span_id', - message: 'Duplicate span_id was preserved with a collision-free node id.', - span_id: spanId, - node_id: node.id, - }); - } - if (!firstNodeBySpanId.has(spanId)) { - firstNodeBySpanId.set(spanId, node); - } - nodes.push(node); - }); - - const parentByNodeId = new Map(); - for (const node of nodes) { - const parentSpanId = - typeof node.parentSpanId === 'string' && node.parentSpanId.length > 0 - ? node.parentSpanId - : undefined; - if (!parentSpanId) { - continue; - } - if (parentSpanId === node.spanId) { - addNodeDiagnostic(node, { - code: 'self_parent', - message: 'Span parent_span_id points to itself; span was promoted to a root.', - span_id: node.spanId, - node_id: node.id, - parent_span_id: parentSpanId, - }); - continue; - } - const parent = firstNodeBySpanId.get(parentSpanId); - if (!parent) { - addNodeDiagnostic(node, { - code: 'missing_parent', - message: 'Span parent_span_id was not present in this trace; span was promoted to a root.', - span_id: node.spanId, - node_id: node.id, - parent_span_id: parentSpanId, - }); - continue; - } - parentByNodeId.set(node.id, parent); - } - - const cyclicNodes: TraceSpanNode[] = []; - for (const node of nodes) { - if (hasAncestorCycle(node, parentByNodeId)) { - cyclicNodes.push(node); - } - } - for (const node of cyclicNodes) { - parentByNodeId.delete(node.id); - addNodeDiagnostic(node, { - code: 'cycle', - message: 'Span parent chain contains a cycle; span was promoted to a root.', - span_id: node.spanId, - node_id: node.id, - parent_span_id: typeof node.parentSpanId === 'string' ? node.parentSpanId : undefined, - }); - } - - const roots: TraceSpanNode[] = []; - for (const node of nodes) { - const parent = parentByNodeId.get(node.id); - if (parent) { - parent.children.push(node); - } else { - roots.push(node); - } - } - - sortTraceSpanNodes(roots); - return roots; -} - -function addNodeDiagnostic(node: TraceSpanNode, diagnostic: TraceSpanTreeDiagnostic): void { - node.diagnostics = [...(node.diagnostics ?? []), diagnostic]; -} - -function hasAncestorCycle( - node: TraceSpanNode, - parentByNodeId: ReadonlyMap, -): boolean { - const seen = new Set(); - let cursor = parentByNodeId.get(node.id); - while (cursor) { - if (cursor.id === node.id || seen.has(cursor.id)) { - return true; - } - seen.add(cursor.id); - cursor = parentByNodeId.get(cursor.id); - } - return false; -} - -function compareUnixNanoValue(first: string | undefined, second: string | undefined): number { - if (first === second) { - return 0; - } - if (!first) { - return 1; - } - if (!second) { - return -1; - } - try { - const firstValue = BigInt(first); - const secondValue = BigInt(second); - return firstValue < secondValue ? -1 : firstValue > secondValue ? 1 : 0; - } catch { - return first.localeCompare(second); - } -} - -function compareTraceSpanNodes(first: TraceSpanNode, second: TraceSpanNode): number { - const byStart = compareUnixNanoValue( - first.span.start_time_unix_nano, - second.span.start_time_unix_nano, - ); - if (byStart !== 0) { - return byStart; - } - if (first.spanId === second.parentSpanId) { - return -1; - } - if (second.spanId === first.parentSpanId) { - return 1; - } - const bySpanId = first.spanId.localeCompare(second.spanId); - return bySpanId !== 0 ? bySpanId : first.id.localeCompare(second.id); -} - -function sortTraceSpanNodes(nodes: TraceSpanNode[]): void { - nodes.sort(compareTraceSpanNodes); - for (const node of nodes) { - node.children.sort(compareTraceSpanNodes); - if (node.children.length > 0) { - sortTraceSpanNodes(node.children); - } - node.diagnostics = nonEmptyArray(node.diagnostics) as TraceSpanTreeDiagnostic[] | undefined; - } -} +export { + TRACE_SESSION_SCHEMA_VERSION, + buildTraceSpanTree, + traceEnvelopeToTraceSessionResponse, +} from '@agentv/core'; +export type { + TraceSessionProjectionOptions, + TraceSpanNode, + TraceSpanTreeDiagnostic, + TraceSpanTreeDiagnosticCode, +} from '@agentv/core'; diff --git a/apps/dashboard/src/lib/types.ts b/apps/dashboard/src/lib/types.ts index a6efab3ac..ad69a3549 100644 --- a/apps/dashboard/src/lib/types.ts +++ b/apps/dashboard/src/lib/types.ts @@ -1,3 +1,18 @@ +import type { + ExternalTraceMetadataWire as CoreExternalTraceMetadata, + TraceSessionArtifactLink as CoreTraceSessionArtifactLink, + TraceSessionConversionWarning as CoreTraceSessionConversionWarning, + TraceSessionEvent as CoreTraceSessionEvent, + TraceSessionEventKind as CoreTraceSessionEventKind, + TraceSessionResponse as CoreTraceSessionResponse, + TraceSessionScore as CoreTraceSessionScore, + TraceSessionSource as CoreTraceSessionSource, + TraceSessionSourceRef as CoreTraceSessionSourceRef, + TraceSessionSpan as CoreTraceSessionSpan, + TraceSessionSpanStatus as CoreTraceSessionSpanStatus, + TraceSessionTokenUsage as CoreTraceSessionTokenUsage, +} from '@agentv/core'; + /** * TypeScript types for the AgentV Dashboard API responses. * @@ -135,30 +150,7 @@ export interface SourceTraceability { referenced_files?: SourceReferencedFile[]; } -export interface ExternalTraceMetadata { - /** - * Optional external viewer reference only. AgentV run artifacts remain the - * canonical source of truth for Dashboard trace/session details. - */ - provider?: string; - source?: string; - endpoint?: string; - profile?: string; - project?: string; - project_id?: string; - session_id?: string; - session_node_id?: string; - trace_id?: string; - trace_node_id?: string; - span_id?: string; - span_node_id?: string; - traceparent?: string; - tracestate?: string; - ui_url?: string; - run_id?: string; - test_id?: string; - target?: string; -} +export type ExternalTraceMetadata = CoreExternalTraceMetadata; export interface PhoenixLinkedSessionTokenUsage { input?: number; @@ -251,117 +243,17 @@ export interface PhoenixLinkedSessionResponse { annotations?: PhoenixLinkedSessionAnnotation[]; } -export interface TraceSessionTokenUsage { - input?: number; - output?: number; - reasoning?: number; - cached?: number; - total?: number; -} - -export interface TraceSessionSpanStatus { - code?: string; - message?: string; -} - -export type TraceSessionEventKind = 'annotation' | 'exception' | 'event' | 'score'; - -export interface TraceSessionEvent { - event_id: string; - span_id: string; - name: string; - kind: TraceSessionEventKind; - time_unix_nano?: string; - timestamp?: string; - score?: number; - text?: string; - passed?: boolean; - attributes?: Record; -} - -export interface TraceSessionSpan { - id: string; - trace_id?: string; - span_id: string; - parent_span_id?: string | null; - name: string; - kind?: string; - status?: TraceSessionSpanStatus; - start_time_unix_nano?: string; - end_time_unix_nano?: string; - start_time?: string; - end_time?: string; - duration_ms?: number; - token_usage?: TraceSessionTokenUsage; - attributes?: Record; - events?: TraceSessionEvent[]; -} - -export interface TraceSessionScore { - name: string; - type?: string; - score: number; - weight?: number; - verdict?: string; - source?: string; - evaluated_at?: string; - target_span_id?: string; - evidence?: Record; -} - -export interface TraceSessionSource { - kind?: string; - path?: string; - provider?: string; - format?: string; - version?: string; - artifact_path?: string; - metadata?: Record; -} - -export interface TraceSessionArtifactLink { - name: string; - path: string; -} - -export interface TraceSessionSourceRef { - event_id?: string; - message_id?: string; - span_id?: string; - trace_id?: string; - raw_kind?: string; - path?: string; - line?: number; - metadata?: Record; -} - -export interface TraceSessionConversionWarning { - code: string; - severity?: 'info' | 'warning' | 'error' | string; - span_id?: string; - source_ref?: TraceSessionSourceRef; - message: string; - details?: Record; -} - -export interface TraceSessionResponse { - schema_version: 'agentv.dashboard.trace_session.v1'; - artifact_id?: string; - created_at?: string; - run_id?: string; - test_id?: string; - suite?: string; - target?: string; - trace_id?: string; - root_span_id?: string; - source?: TraceSessionSource; - external_trace?: ExternalTraceMetadata; - artifact_links?: TraceSessionArtifactLink[]; - conversion_warnings?: TraceSessionConversionWarning[]; - spans: TraceSessionSpan[]; - events: TraceSessionEvent[]; - scores?: TraceSessionScore[]; -} +export type TraceSessionTokenUsage = CoreTraceSessionTokenUsage; +export type TraceSessionSpanStatus = CoreTraceSessionSpanStatus; +export type TraceSessionEventKind = CoreTraceSessionEventKind; +export type TraceSessionEvent = CoreTraceSessionEvent; +export type TraceSessionSpan = CoreTraceSessionSpan; +export type TraceSessionScore = CoreTraceSessionScore; +export type TraceSessionSource = CoreTraceSessionSource; +export type TraceSessionArtifactLink = CoreTraceSessionArtifactLink; +export type TraceSessionSourceRef = CoreTraceSessionSourceRef; +export type TraceSessionConversionWarning = CoreTraceSessionConversionWarning; +export type TraceSessionResponse = CoreTraceSessionResponse; export interface EvalResult { testId: string; diff --git a/bun.lock b/bun.lock index 9c7492892..4d8cdd29d 100644 --- a/bun.lock +++ b/bun.lock @@ -20,7 +20,7 @@ }, "apps/cli": { "name": "agentv", - "version": "4.41.1-next.1", + "version": "4.42.4", "bin": { "agentv": "./dist/cli.js", }, @@ -56,6 +56,7 @@ "name": "@agentv/dashboard", "version": "0.0.1", "dependencies": { + "@agentv/core": "workspace:*", "@monaco-editor/react": "^4.7.0", "@tanstack/react-query": "^5.75.5", "@tanstack/react-router": "^1.120.3", @@ -84,7 +85,7 @@ }, "packages/core": { "name": "@agentv/core", - "version": "4.41.1-next.1", + "version": "4.42.4", "dependencies": { "@agentclientprotocol/sdk": "^0.14.1", "@earendil-works/pi-ai": "^0.74.0", @@ -119,7 +120,7 @@ }, "packages/eval": { "name": "@agentv/eval", - "version": "4.41.1-next.1", + "version": "4.42.4", "dependencies": { "@agentv/sdk": "workspace:*", }, @@ -140,7 +141,7 @@ }, "packages/sdk": { "name": "@agentv/sdk", - "version": "4.41.1-next.1", + "version": "4.42.4", "dependencies": { "yaml": "^2.8.3", "zod": "^3.23.8", diff --git a/packages/core/src/evaluation/dashboard-trace-read-model.ts b/packages/core/src/evaluation/dashboard-trace-read-model.ts new file mode 100644 index 000000000..b6b2ae147 --- /dev/null +++ b/packages/core/src/evaluation/dashboard-trace-read-model.ts @@ -0,0 +1,970 @@ +/** + * Dashboard trace/session wire read model. + * + * This module projects already-normalized `agentv.trace.v1` envelopes into the + * stable snake_case payload consumed by the local Dashboard and served by the + * CLI Dashboard API. It intentionally does not normalize raw OTLP/provider + * traces; that stays in the trace normalization pipeline before artifacts reach + * this projector. + */ + +import type { ExternalTraceMetadataWire } from './external-trace.js'; + +export interface TraceSessionTokenUsage { + input?: number; + output?: number; + reasoning?: number; + cached?: number; + total?: number; +} + +export interface TraceSessionSpanStatus { + code?: string; + message?: string; +} + +export type TraceSessionEventKind = 'annotation' | 'exception' | 'event' | 'score'; + +export interface TraceSessionEvent { + event_id: string; + span_id: string; + name: string; + kind: TraceSessionEventKind; + time_unix_nano?: string; + timestamp?: string; + score?: number; + text?: string; + passed?: boolean; + attributes?: Record; +} + +export interface TraceSessionSpan { + id: string; + trace_id?: string; + span_id: string; + parent_span_id?: string | null; + name: string; + kind?: string; + status?: TraceSessionSpanStatus; + start_time_unix_nano?: string; + end_time_unix_nano?: string; + start_time?: string; + end_time?: string; + duration_ms?: number; + token_usage?: TraceSessionTokenUsage; + attributes?: Record; + events?: TraceSessionEvent[]; +} + +export interface TraceSessionScore { + name: string; + type?: string; + score: number; + weight?: number; + verdict?: string; + source?: string; + evaluated_at?: string; + target_span_id?: string; + evidence?: Record; +} + +export interface TraceSessionSource { + kind?: string; + path?: string; + provider?: string; + format?: string; + version?: string; + artifact_path?: string; + metadata?: Record; +} + +export interface TraceSessionArtifactLink { + name: string; + path: string; +} + +export interface TraceSessionSourceRef { + event_id?: string; + message_id?: string; + span_id?: string; + trace_id?: string; + raw_kind?: string; + path?: string; + line?: number; + metadata?: Record; +} + +export interface TraceSessionConversionWarning { + code: string; + severity?: 'info' | 'warning' | 'error' | string; + span_id?: string; + source_ref?: TraceSessionSourceRef; + message: string; + details?: Record; +} + +export interface TraceSessionResponse { + schema_version: 'agentv.dashboard.trace_session.v1'; + artifact_id?: string; + created_at?: string; + run_id?: string; + test_id?: string; + suite?: string; + target?: string; + trace_id?: string; + root_span_id?: string; + source?: TraceSessionSource; + external_trace?: ExternalTraceMetadataWire; + artifact_links?: TraceSessionArtifactLink[]; + conversion_warnings?: TraceSessionConversionWarning[]; + spans: TraceSessionSpan[]; + events: TraceSessionEvent[]; + scores?: TraceSessionScore[]; +} + +export const TRACE_SESSION_SCHEMA_VERSION = 'agentv.dashboard.trace_session.v1' as const; + +export interface TraceSessionProjectionOptions { + runId?: string; + artifactPath?: string; +} + +export interface TraceSpanNode { + id: string; + spanId: string; + parentSpanId?: string | null; + span: TraceSessionSpan; + children: TraceSpanNode[]; + diagnostics?: TraceSpanTreeDiagnostic[]; +} + +export type TraceSpanTreeDiagnosticCode = + | 'cycle' + | 'duplicate_span_id' + | 'missing_parent' + | 'missing_span_id' + | 'self_parent'; + +export interface TraceSpanTreeDiagnostic { + code: TraceSpanTreeDiagnosticCode; + message: string; + span_id?: string; + node_id?: string; + parent_span_id?: string; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function asRecord(value: unknown): Record | undefined { + return isRecord(value) ? value : undefined; +} + +function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function stringValue(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function finiteNumber(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +function finiteInteger(value: unknown): number | undefined { + return typeof value === 'number' && Number.isInteger(value) && value >= 0 ? value : undefined; +} + +function boolValue(value: unknown): boolean | undefined { + return typeof value === 'boolean' ? value : undefined; +} + +function dropUndefined>(value: T): T { + return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined)) as T; +} + +function compactRecord(value: Record): Record | undefined { + const compacted = dropUndefined(value); + return Object.keys(compacted).length > 0 ? compacted : undefined; +} + +function nonEmptyArray(value: readonly T[] | undefined): readonly T[] | undefined { + return value && value.length > 0 ? value : undefined; +} + +function unixNanoToIso(value: string | undefined): string | undefined { + if (!value) { + return undefined; + } + try { + return new Date(Number(BigInt(value) / 1_000_000n)).toISOString(); + } catch { + return undefined; + } +} + +function durationMsFromNanos( + start: string | undefined, + end: string | undefined, +): number | undefined { + if (!start || !end) { + return undefined; + } + try { + const startNanos = BigInt(start); + const endNanos = BigInt(end); + if (endNanos < startNanos) { + return undefined; + } + return Number(endNanos - startNanos) / 1_000_000; + } catch { + return undefined; + } +} + +function numberFromAttributes( + attributes: Record, + keys: readonly string[], +): number | undefined { + for (const key of keys) { + const value = finiteNumber(attributes[key]); + if (value !== undefined) { + return value; + } + } + return undefined; +} + +function tokenUsageFromAttributes( + attributes: Record | undefined, +): TraceSessionTokenUsage | undefined { + if (!attributes) { + return undefined; + } + + const nested = asRecord(attributes.token_usage); + const usage = compactRecord({ + input: + finiteNumber(nested?.input) ?? + numberFromAttributes(attributes, [ + 'gen_ai.usage.input_tokens', + 'llm.token_count.prompt', + 'input_tokens', + ]), + output: + finiteNumber(nested?.output) ?? + numberFromAttributes(attributes, [ + 'gen_ai.usage.output_tokens', + 'llm.token_count.completion', + 'output_tokens', + ]), + reasoning: + finiteNumber(nested?.reasoning) ?? + numberFromAttributes(attributes, [ + 'gen_ai.usage.reasoning.output_tokens', + 'reasoning_tokens', + ]), + cached: + finiteNumber(nested?.cached) ?? + numberFromAttributes(attributes, ['gen_ai.usage.cache_read.input_tokens', 'cached_tokens']), + total: finiteNumber(nested?.total) ?? numberFromAttributes(attributes, ['total_tokens']), + }); + + return usage as TraceSessionTokenUsage | undefined; +} + +function isExternalTraceKey(key: string): boolean { + return ( + key === 'external_trace' || + key.startsWith('external_trace_') || + key.startsWith('external_trace.') + ); +} + +function isCredentialLikeKey(key: string): boolean { + const normalized = key.toLowerCase(); + if ( + normalized === 'token_usage' || + normalized.endsWith('_tokens') || + normalized.endsWith('.tokens') || + normalized.includes('usage.') + ) { + return false; + } + return /(^|[._-])(api[._-]?key|authorization|bearer|password|secret|private[._-]?key|access[._-]?token|auth[._-]?token|client[._-]?secret|id[._-]?token|refresh[._-]?token|session[._-]?token|token)($|[._-])/.test( + normalized, + ); +} + +function sanitizeAttributeMap( + value: Record | undefined, +): Record | undefined { + if (!value) { + return undefined; + } + const entries = Object.entries(value).flatMap(([key, entry]) => { + if (isExternalTraceKey(key) || isCredentialLikeKey(key)) { + return []; + } + if (isRecord(entry)) { + const nested = sanitizeAttributeMap(entry); + return nested ? [[key, nested] as const] : []; + } + return [[key, entry] as const]; + }); + return entries.length > 0 ? Object.fromEntries(entries) : undefined; +} + +function spanStatusFromValue(value: unknown): TraceSessionSpan['status'] { + const record = asRecord(value); + if (!record) { + return undefined; + } + return compactRecord({ + code: + stringValue(record.code) ?? + (typeof record.code === 'number' ? String(record.code) : undefined), + message: stringValue(record.message), + }) as TraceSessionSpan['status']; +} + +function eventKind( + name: string, + attributes: Record | undefined, +): TraceSessionEventKind { + const lowerName = name.toLowerCase(); + if ( + lowerName.includes('score') || + finiteNumber(attributes?.score) !== undefined || + finiteNumber(attributes?.['agentv.score']) !== undefined || + finiteNumber(attributes?.['agentv.grader.score']) !== undefined + ) { + return 'score'; + } + if ( + lowerName.includes('annotation') || + stringValue(attributes?.text) !== undefined || + stringValue(attributes?.annotation) !== undefined || + stringValue(attributes?.['agentv.annotation.text']) !== undefined + ) { + return 'annotation'; + } + if (lowerName === 'exception') { + return 'exception'; + } + return 'event'; +} + +function scoreFromEvent(attributes: Record | undefined): number | undefined { + if (!attributes) { + return undefined; + } + return ( + finiteNumber(attributes.score) ?? + finiteNumber(attributes['agentv.score']) ?? + finiteNumber(attributes['agentv.grader.score']) + ); +} + +function textFromEvent(attributes: Record | undefined): string | undefined { + if (!attributes) { + return undefined; + } + return ( + stringValue(attributes.text) ?? + stringValue(attributes.annotation) ?? + stringValue(attributes['agentv.annotation.text']) ?? + stringValue(attributes['exception.message']) + ); +} + +function passedFromEvent(attributes: Record | undefined): boolean | undefined { + if (!attributes) { + return undefined; + } + return boolValue(attributes.passed) ?? boolValue(attributes['agentv.annotation.passed']); +} + +function eventId( + spanId: string, + index: number, + attributes: Record | undefined, +): string { + return ( + stringValue(attributes?.event_id) ?? + stringValue(attributes?.['agentv.event_id']) ?? + `${spanId}:event:${index}` + ); +} + +function projectSpanEvent( + spanId: string, + event: unknown, + index: number, +): TraceSessionEvent | undefined { + const record = asRecord(event); + if (!record) { + return undefined; + } + const name = stringValue(record.name); + if (!name) { + return undefined; + } + + const attributes = asRecord(record.attributes); + const safeAttributes = sanitizeAttributeMap(attributes); + return dropUndefined({ + event_id: eventId(spanId, index, attributes), + span_id: spanId, + name, + kind: eventKind(name, attributes), + time_unix_nano: stringValue(record.time_unix_nano), + timestamp: unixNanoToIso(stringValue(record.time_unix_nano)), + score: scoreFromEvent(attributes), + text: textFromEvent(attributes), + passed: passedFromEvent(attributes), + attributes: safeAttributes, + }); +} + +function projectSpan(span: unknown, index: number): TraceSessionSpan | undefined { + const record = asRecord(span); + if (!record) { + return undefined; + } + + const spanId = stringValue(record.span_id) ?? `span-${index}`; + const traceId = stringValue(record.trace_id); + const parentSpanId = record.parent_span_id === null ? null : stringValue(record.parent_span_id); + const attributes = asRecord(record.attributes); + const safeAttributes = sanitizeAttributeMap(attributes); + const startTimeUnixNano = stringValue(record.start_time_unix_nano); + const endTimeUnixNano = stringValue(record.end_time_unix_nano); + const events = asArray(record.events) + .map((event, eventIndex) => projectSpanEvent(spanId, event, eventIndex)) + .filter((event): event is TraceSessionEvent => event !== undefined); + + return dropUndefined({ + id: spanId, + trace_id: traceId, + span_id: spanId, + parent_span_id: parentSpanId, + name: stringValue(record.name) ?? spanId, + kind: stringValue(record.kind), + status: spanStatusFromValue(record.status), + start_time_unix_nano: startTimeUnixNano, + end_time_unix_nano: endTimeUnixNano, + start_time: unixNanoToIso(startTimeUnixNano), + end_time: unixNanoToIso(endTimeUnixNano), + duration_ms: durationMsFromNanos(startTimeUnixNano, endTimeUnixNano), + token_usage: tokenUsageFromAttributes(attributes), + attributes: safeAttributes, + events: events.length > 0 ? events : undefined, + }); +} + +function projectScores(scores: unknown): TraceSessionScore[] | undefined { + const projected: TraceSessionScore[] = []; + + for (const score of asArray(scores)) { + const record = asRecord(score); + const name = stringValue(record?.name); + const value = finiteNumber(record?.score); + if (!record || !name || value === undefined) { + continue; + } + projected.push( + dropUndefined({ + name, + type: stringValue(record.type), + score: value, + weight: finiteNumber(record.weight), + verdict: stringValue(record.verdict), + source: stringValue(record.source), + evaluated_at: stringValue(record.evaluated_at), + target_span_id: stringValue(record.target_span_id), + evidence: asRecord(record.evidence), + }) as TraceSessionScore, + ); + } + + return projected.length > 0 ? projected : undefined; +} + +const EXTERNAL_TRACE_KEYS = [ + 'provider', + 'source', + 'endpoint', + 'profile', + 'project', + 'project_id', + 'session_id', + 'session_node_id', + 'trace_id', + 'trace_node_id', + 'span_id', + 'span_node_id', + 'traceparent', + 'tracestate', + 'ui_url', + 'run_id', + 'test_id', + 'target', +] as const; + +function sanitizeUrl(value: unknown): string | undefined { + const raw = stringValue(value); + if (!raw) { + return undefined; + } + try { + const url = new URL(raw); + if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) { + return undefined; + } + url.search = ''; + url.hash = ''; + return url.toString(); + } catch { + return undefined; + } +} + +function sanitizeExternalTrace(value: unknown): ExternalTraceMetadataWire | undefined { + const record = asRecord(value); + if (!record) { + return undefined; + } + + const sanitized = compactRecord({ + provider: stringValue(record.provider), + source: stringValue(record.source), + endpoint: sanitizeUrl(record.endpoint), + profile: stringValue(record.profile), + project: stringValue(record.project), + project_id: stringValue(record.project_id) ?? stringValue(record.projectId), + session_id: stringValue(record.session_id) ?? stringValue(record.session), + session_node_id: + stringValue(record.session_node_id) ?? + stringValue(record.session_node) ?? + stringValue(record.node_id), + trace_id: stringValue(record.trace_id) ?? stringValue(record.trace), + trace_node_id: stringValue(record.trace_node_id) ?? stringValue(record.trace_node), + span_id: stringValue(record.span_id) ?? stringValue(record.span), + span_node_id: stringValue(record.span_node_id) ?? stringValue(record.span_node), + traceparent: stringValue(record.traceparent), + tracestate: stringValue(record.tracestate), + ui_url: sanitizeUrl(record.ui_url ?? record.url ?? record.href), + run_id: stringValue(record.run_id), + test_id: stringValue(record.test_id), + target: stringValue(record.target), + }) as ExternalTraceMetadataWire | undefined; + + return sanitized && EXTERNAL_TRACE_KEYS.some((key) => sanitized[key] !== undefined) + ? sanitized + : undefined; +} + +function externalTraceFromFlatMetadata( + metadata: Record | undefined, +): ExternalTraceMetadataWire | undefined { + if (!metadata) { + return undefined; + } + return sanitizeExternalTrace({ + provider: metadata.external_trace_provider ?? metadata['external_trace.provider'], + source: metadata.external_trace_source ?? metadata['external_trace.source'], + endpoint: metadata.external_trace_endpoint ?? metadata['external_trace.endpoint'], + profile: metadata.external_trace_profile ?? metadata['external_trace.profile'], + project: metadata.external_trace_project ?? metadata['external_trace.project'], + project_id: metadata.external_trace_project_id ?? metadata['external_trace.project_id'], + session_id: + metadata.external_trace_session_id ?? + metadata.external_trace_session ?? + metadata['external_trace.session_id'] ?? + metadata['external_trace.session'], + session_node_id: + metadata.external_trace_session_node_id ?? + metadata.external_trace_node_id ?? + metadata['external_trace.session_node_id'] ?? + metadata['external_trace.node_id'], + trace_id: + metadata.external_trace_trace_id ?? + metadata.external_trace_trace ?? + metadata['external_trace.trace_id'] ?? + metadata['external_trace.trace'], + trace_node_id: + metadata.external_trace_trace_node_id ?? metadata['external_trace.trace_node_id'], + span_id: + metadata.external_trace_span_id ?? + metadata.external_trace_span ?? + metadata['external_trace.span_id'] ?? + metadata['external_trace.span'], + span_node_id: metadata.external_trace_span_node_id ?? metadata['external_trace.span_node_id'], + traceparent: metadata.external_trace_traceparent ?? metadata['external_trace.traceparent'], + tracestate: metadata.external_trace_tracestate ?? metadata['external_trace.tracestate'], + ui_url: + metadata.external_trace_ui_url ?? + metadata.external_trace_url ?? + metadata['external_trace.ui_url'] ?? + metadata['external_trace.url'], + run_id: metadata.external_trace_run_id ?? metadata['external_trace.run_id'], + test_id: metadata.external_trace_test_id ?? metadata['external_trace.test_id'], + target: metadata.external_trace_target ?? metadata['external_trace.target'], + }); +} + +function sanitizeMetadataValue(value: unknown): unknown | undefined { + if (Array.isArray(value)) { + const sanitized = value + .map(sanitizeMetadataValue) + .filter((entry): entry is unknown => entry !== undefined); + return sanitized.length > 0 ? sanitized : undefined; + } + if (isRecord(value)) { + return sanitizeMetadata(value); + } + return value; +} + +function sanitizeMetadata( + value: Record | undefined, +): Record | undefined { + if (!value) { + return undefined; + } + const entries = Object.entries(value).flatMap(([key, entry]) => { + if (isExternalTraceKey(key) || isCredentialLikeKey(key)) { + return []; + } + const sanitized = sanitizeMetadataValue(entry); + return sanitized !== undefined ? [[key, sanitized] as const] : []; + }); + return entries.length > 0 ? Object.fromEntries(entries) : undefined; +} + +function sourceFromEnvelope( + source: Record | undefined, + artifactPath: string | undefined, +): TraceSessionSource | undefined { + if (!source && !artifactPath) { + return undefined; + } + return compactRecord({ + kind: stringValue(source?.kind), + path: stringValue(source?.path), + provider: stringValue(source?.provider), + format: stringValue(source?.format), + version: stringValue(source?.version), + artifact_path: artifactPath, + metadata: sanitizeMetadata(asRecord(source?.metadata)), + }) as TraceSessionSource | undefined; +} + +function safeArtifactPath(value: unknown): string | undefined { + const raw = stringValue(value); + if (!raw || raw.includes('\0')) { + return undefined; + } + + const normalized = raw.replace(/\\/g, '/'); + if (normalized.startsWith('/') || normalized.startsWith('//')) { + return undefined; + } + if (/^[a-z][a-z0-9+.-]*:/i.test(normalized)) { + return undefined; + } + if (normalized.split('/').includes('..')) { + return undefined; + } + return normalized; +} + +function projectArtifactLinks(artifacts: unknown): TraceSessionArtifactLink[] | undefined { + const record = asRecord(artifacts); + if (!record) { + return undefined; + } + + const links = Object.entries(record) + .flatMap(([name, value]) => { + if (!stringValue(name) || isCredentialLikeKey(name)) { + return []; + } + const artifactPath = safeArtifactPath(value); + return artifactPath ? [{ name, path: artifactPath }] : []; + }) + .sort((first, second) => first.name.localeCompare(second.name)); + + return links.length > 0 ? links : undefined; +} + +function projectSourceRef(value: unknown): TraceSessionSourceRef | undefined { + const record = asRecord(value); + if (!record) { + return undefined; + } + return compactRecord({ + event_id: stringValue(record.event_id), + message_id: stringValue(record.message_id), + span_id: stringValue(record.span_id), + trace_id: stringValue(record.trace_id), + raw_kind: stringValue(record.raw_kind), + path: safeArtifactPath(record.path), + line: finiteInteger(record.line), + metadata: sanitizeMetadata(asRecord(record.metadata)), + }) as TraceSessionSourceRef | undefined; +} + +function projectConversionWarnings(warnings: unknown): TraceSessionConversionWarning[] | undefined { + const projected: TraceSessionConversionWarning[] = []; + + for (const warning of asArray(warnings)) { + const record = asRecord(warning); + const code = stringValue(record?.code); + const message = stringValue(record?.message); + if (!record || !code || !message) { + continue; + } + projected.push( + dropUndefined({ + code, + severity: stringValue(record.severity), + span_id: stringValue(record.span_id), + source_ref: projectSourceRef(record.source_ref), + message, + details: sanitizeMetadata(asRecord(record.details)), + }) as TraceSessionConversionWarning, + ); + } + + return projected.length > 0 ? projected : undefined; +} + +function externalTraceFromEnvelope( + envelope: Record, +): ExternalTraceMetadataWire | undefined { + const source = asRecord(envelope.source); + const sourceMetadata = asRecord(source?.metadata); + const trace = asRecord(envelope.trace); + const rootSpanId = stringValue(trace?.root_span_id); + const rootSpan = asArray(trace?.spans) + .map(asRecord) + .find((span) => stringValue(span?.span_id) === rootSpanId); + const rootAttributes = asRecord(rootSpan?.attributes); + + return ( + sanitizeExternalTrace(envelope.external_trace) ?? + sanitizeExternalTrace(sourceMetadata?.external_trace) ?? + externalTraceFromFlatMetadata(sourceMetadata) ?? + externalTraceFromFlatMetadata(rootAttributes) + ); +} + +export function traceEnvelopeToTraceSessionResponse( + input: unknown, + options: TraceSessionProjectionOptions = {}, +): TraceSessionResponse { + const envelope = asRecord(input) ?? {}; + const evaluation = asRecord(envelope.eval); + const trace = asRecord(envelope.trace); + const spans = asArray(trace?.spans) + .map(projectSpan) + .filter((span): span is TraceSessionSpan => span !== undefined); + const events = spans.flatMap((span) => span.events ?? []); + + return dropUndefined({ + schema_version: TRACE_SESSION_SCHEMA_VERSION, + artifact_id: stringValue(envelope.artifact_id), + created_at: stringValue(envelope.created_at), + run_id: options.runId ?? stringValue(evaluation?.run_id), + test_id: stringValue(evaluation?.test_id), + suite: stringValue(evaluation?.suite), + target: stringValue(evaluation?.target), + trace_id: stringValue(trace?.trace_id), + root_span_id: stringValue(trace?.root_span_id), + source: sourceFromEnvelope(asRecord(envelope.source), options.artifactPath), + external_trace: externalTraceFromEnvelope(envelope), + artifact_links: projectArtifactLinks(envelope.artifacts), + conversion_warnings: projectConversionWarnings(envelope.conversion_warnings), + spans, + events, + scores: projectScores(envelope.scores), + }); +} + +export function buildTraceSpanTree(spans: readonly TraceSessionSpan[]): TraceSpanNode[] { + const nodes: TraceSpanNode[] = []; + const firstNodeBySpanId = new Map(); + const spanIdCounts = new Map(); + + spans.forEach((span, index) => { + const rawSpanId = stringValue(span.span_id); + const spanId = rawSpanId ?? `missing-span-${index}`; + const occurrence = (spanIdCounts.get(spanId) ?? 0) + 1; + spanIdCounts.set(spanId, occurrence); + + const node: TraceSpanNode = { + id: occurrence === 1 ? spanId : `${spanId}#${occurrence}`, + spanId, + parentSpanId: span.parent_span_id, + span, + children: [], + diagnostics: rawSpanId + ? undefined + : [ + { + code: 'missing_span_id', + message: 'Span was missing span_id and was assigned a stable node id.', + node_id: spanId, + }, + ], + }; + + if (occurrence > 1) { + addNodeDiagnostic(node, { + code: 'duplicate_span_id', + message: 'Duplicate span_id was preserved with a collision-free node id.', + span_id: spanId, + node_id: node.id, + }); + } + if (!firstNodeBySpanId.has(spanId)) { + firstNodeBySpanId.set(spanId, node); + } + nodes.push(node); + }); + + const parentByNodeId = new Map(); + for (const node of nodes) { + const parentSpanId = + typeof node.parentSpanId === 'string' && node.parentSpanId.length > 0 + ? node.parentSpanId + : undefined; + if (!parentSpanId) { + continue; + } + if (parentSpanId === node.spanId) { + addNodeDiagnostic(node, { + code: 'self_parent', + message: 'Span parent_span_id points to itself; span was promoted to a root.', + span_id: node.spanId, + node_id: node.id, + parent_span_id: parentSpanId, + }); + continue; + } + const parent = firstNodeBySpanId.get(parentSpanId); + if (!parent) { + addNodeDiagnostic(node, { + code: 'missing_parent', + message: 'Span parent_span_id was not present in this trace; span was promoted to a root.', + span_id: node.spanId, + node_id: node.id, + parent_span_id: parentSpanId, + }); + continue; + } + parentByNodeId.set(node.id, parent); + } + + const cyclicNodes: TraceSpanNode[] = []; + for (const node of nodes) { + if (hasAncestorCycle(node, parentByNodeId)) { + cyclicNodes.push(node); + } + } + for (const node of cyclicNodes) { + parentByNodeId.delete(node.id); + addNodeDiagnostic(node, { + code: 'cycle', + message: 'Span parent chain contains a cycle; span was promoted to a root.', + span_id: node.spanId, + node_id: node.id, + parent_span_id: typeof node.parentSpanId === 'string' ? node.parentSpanId : undefined, + }); + } + + const roots: TraceSpanNode[] = []; + for (const node of nodes) { + const parent = parentByNodeId.get(node.id); + if (parent) { + parent.children.push(node); + } else { + roots.push(node); + } + } + + sortTraceSpanNodes(roots); + return roots; +} + +function addNodeDiagnostic(node: TraceSpanNode, diagnostic: TraceSpanTreeDiagnostic): void { + node.diagnostics = [...(node.diagnostics ?? []), diagnostic]; +} + +function hasAncestorCycle( + node: TraceSpanNode, + parentByNodeId: ReadonlyMap, +): boolean { + const seen = new Set(); + let cursor = parentByNodeId.get(node.id); + while (cursor) { + if (cursor.id === node.id || seen.has(cursor.id)) { + return true; + } + seen.add(cursor.id); + cursor = parentByNodeId.get(cursor.id); + } + return false; +} + +function compareUnixNanoValue(first: string | undefined, second: string | undefined): number { + if (first === second) { + return 0; + } + if (!first) { + return 1; + } + if (!second) { + return -1; + } + try { + const firstValue = BigInt(first); + const secondValue = BigInt(second); + return firstValue < secondValue ? -1 : firstValue > secondValue ? 1 : 0; + } catch { + return first.localeCompare(second); + } +} + +function compareTraceSpanNodes(first: TraceSpanNode, second: TraceSpanNode): number { + const byStart = compareUnixNanoValue( + first.span.start_time_unix_nano, + second.span.start_time_unix_nano, + ); + if (byStart !== 0) { + return byStart; + } + if (first.spanId === second.parentSpanId) { + return -1; + } + if (second.spanId === first.parentSpanId) { + return 1; + } + const bySpanId = first.spanId.localeCompare(second.spanId); + return bySpanId !== 0 ? bySpanId : first.id.localeCompare(second.id); +} + +function sortTraceSpanNodes(nodes: TraceSpanNode[]): void { + nodes.sort(compareTraceSpanNodes); + for (const node of nodes) { + node.children.sort(compareTraceSpanNodes); + if (node.children.length > 0) { + sortTraceSpanNodes(node.children); + } + node.diagnostics = nonEmptyArray(node.diagnostics) as TraceSpanTreeDiagnostic[] | undefined; + } +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ff73abe87..074d3505c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2,6 +2,7 @@ export * from './evaluation/content.js'; export * from './evaluation/types.js'; export * from './evaluation/trace.js'; export * from './evaluation/trace-envelope.js'; +export * from './evaluation/dashboard-trace-read-model.js'; export * from './evaluation/external-trace.js'; export * from './evaluation/projection-identity.js'; export * from './evaluation/replay-fixtures.js';