From 22f7c3db329ba38d34963f7c89f21ee4414ca210 Mon Sep 17 00:00:00 2001 From: Muhammad Ali Hashar Date: Sun, 25 Oct 2026 15:07:34 -0400 Subject: [PATCH 1/2] refactor: move the harness text dispatch out of the wasm module The two functions that parse or render a harness's native text by HarnessId lived privately in src/wasm.rs, so only the WASM bindings could use them. Move them to a new always-compiled txcript::dispatch module and make wasm.rs call them. No behaviour change. This lets other code (such as a conversion report) render a transcript into any harness's format and read it back without going through WASM. Co-Authored-By: Claude Sonnet 5.5 --- src/dispatch.rs | 77 +++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 3 ++ src/wasm.rs | 59 ++----------------------------------- 3 files changed, 82 insertions(+), 57 deletions(-) create mode 100644 src/dispatch.rs diff --git a/src/dispatch.rs b/src/dispatch.rs new file mode 100644 index 0000000..512558c --- /dev/null +++ b/src/dispatch.rs @@ -0,0 +1,77 @@ +//! Text-level codec dispatch: parse a harness's native session text into the +//! canonical model, or render the canonical model back to native text, keyed +//! by a runtime [`HarnessId`]. +//! +//! No filesystem, no `SQLite`, no subprocess: this is the codec layer alone +//! (see [`TextCodec`]), so it is available on every target, WASM included. The +//! WASM bindings and [`fidelity`](crate::fidelity) both route through it. + +use crate::harness::{ + amp, antigravity, campfire, chatgpt, claude_chat, claude_code, codex, cowork, cowork_remote, + cursor, cursor_desktop, fx, grok, grok_bot, hermes, opencode, pi, simple, +}; +use crate::transcript::{Codec, Common, HarnessId, TextCodec, Transcript}; + +/// Parse `harness`'s native session text into the canonical model. +/// +/// # Errors +/// When the text is not that harness's session format. +pub fn parse_to_common(harness: HarnessId, text: &str) -> crate::Result> { + fn go(text: &str) -> crate::Result> { + H::to_common(&H::from_text(text)?) + } + match harness { + HarnessId::ClaudeCode => go::(text), + HarnessId::ClaudeChat => go::(text), + HarnessId::CoworkRemote => go::(text), + HarnessId::ChatGpt => go::(text), + HarnessId::Codex => go::(text), + HarnessId::OpenCode => go::(text), + HarnessId::Pi => go::(text), + HarnessId::Campfire => go::(text), + HarnessId::Cursor => go::(text), + HarnessId::CursorDesktop => go::(text), + HarnessId::Grok => go::(text), + HarnessId::GrokBot => go::(text), + HarnessId::Fx => go::(text), + HarnessId::Hermes => go::(text), + HarnessId::Amp => go::(text), + HarnessId::Antigravity => go::(text), + HarnessId::Simple => go::(text), + HarnessId::Cowork => go::(text), + } +} + +/// Render the canonical model as `harness`'s native session text. +/// +/// # Errors +/// When the harness cannot represent the transcript (the live read-only +/// sources, for one). +pub fn render_from_common( + harness: HarnessId, + common: &Transcript, +) -> crate::Result { + fn go(common: &Transcript) -> crate::Result { + H::to_text(&H::from_common(common)?) + } + match harness { + HarnessId::ClaudeCode => go::(common), + HarnessId::ClaudeChat => go::(common), + HarnessId::CoworkRemote => go::(common), + HarnessId::ChatGpt => go::(common), + HarnessId::Codex => go::(common), + HarnessId::OpenCode => go::(common), + HarnessId::Pi => go::(common), + HarnessId::Campfire => go::(common), + HarnessId::Cursor => go::(common), + HarnessId::CursorDesktop => go::(common), + HarnessId::Grok => go::(common), + HarnessId::GrokBot => go::(common), + HarnessId::Fx => go::(common), + HarnessId::Hermes => go::(common), + HarnessId::Amp => go::(common), + HarnessId::Antigravity => go::(common), + HarnessId::Simple => go::(common), + HarnessId::Cowork => go::(common), + } +} diff --git a/src/lib.rs b/src/lib.rs index a340523..3dd58da 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,8 +17,11 @@ //! - [`Transcript`], [`Harness`], [`Codec`], [`Store`] — the generic type and //! the traits over it. //! - [`harness`] — one module per implemented harness. +//! - [`dispatch`] — parse or render a harness's native text, chosen by +//! [`HarnessId`] at run time. pub mod common; +pub mod dispatch; pub mod error; pub mod harness; #[cfg(not(target_arch = "wasm32"))] diff --git a/src/wasm.rs b/src/wasm.rs index e1bbb6f..2417b07 100644 --- a/src/wasm.rs +++ b/src/wasm.rs @@ -13,11 +13,8 @@ use serde::{Deserialize, Serialize}; use wasm_bindgen::prelude::*; use crate::common; -use crate::harness::{ - amp, antigravity, campfire, chatgpt, claude_chat, claude_code, codex, cowork, cowork_remote, - cursor, cursor_desktop, fx, grok, grok_bot, hermes, opencode, pi, simple, -}; -use crate::transcript::{Codec, Common, HarnessId, TextCodec, Transcript}; +use crate::dispatch::{parse_to_common, render_from_common}; +use crate::transcript::{HarnessId, Transcript}; /// Continue/convert a session from one harness's native text into another's. /// @@ -194,58 +191,6 @@ struct CommonJson { messages: Vec, } -fn parse_to_common(harness: HarnessId, text: &str) -> crate::Result> { - fn go(text: &str) -> crate::Result> { - H::to_common(&H::from_text(text)?) - } - match harness { - HarnessId::ClaudeCode => go::(text), - HarnessId::ClaudeChat => go::(text), - HarnessId::CoworkRemote => go::(text), - HarnessId::ChatGpt => go::(text), - HarnessId::Codex => go::(text), - HarnessId::OpenCode => go::(text), - HarnessId::Pi => go::(text), - HarnessId::Campfire => go::(text), - HarnessId::Cursor => go::(text), - HarnessId::CursorDesktop => go::(text), - HarnessId::Grok => go::(text), - HarnessId::GrokBot => go::(text), - HarnessId::Fx => go::(text), - HarnessId::Hermes => go::(text), - HarnessId::Amp => go::(text), - HarnessId::Antigravity => go::(text), - HarnessId::Simple => go::(text), - HarnessId::Cowork => go::(text), - } -} - -fn render_from_common(harness: HarnessId, common: &Transcript) -> crate::Result { - fn go(common: &Transcript) -> crate::Result { - H::to_text(&H::from_common(common)?) - } - match harness { - HarnessId::ClaudeCode => go::(common), - HarnessId::ClaudeChat => go::(common), - HarnessId::CoworkRemote => go::(common), - HarnessId::ChatGpt => go::(common), - HarnessId::Codex => go::(common), - HarnessId::OpenCode => go::(common), - HarnessId::Pi => go::(common), - HarnessId::Campfire => go::(common), - HarnessId::Cursor => go::(common), - HarnessId::CursorDesktop => go::(common), - HarnessId::Grok => go::(common), - HarnessId::GrokBot => go::(common), - HarnessId::Fx => go::(common), - HarnessId::Hermes => go::(common), - HarnessId::Amp => go::(common), - HarnessId::Antigravity => go::(common), - HarnessId::Simple => go::(common), - HarnessId::Cowork => go::(common), - } -} - fn parse_harness(name: &str) -> Result { name.parse().map_err(js) } From 0c7d08a50cb02b50d137594992d787d88318d154 Mon Sep 17 00:00:00 2001 From: Muhammad Ali Hashar Date: Sun, 25 Oct 2026 15:08:47 -0400 Subject: [PATCH 2/2] feat(fidelity): measure what converting a session into a harness loses Add txcript::fidelity. assess(target, source) renders a transcript into a harness's native text, reads it back, and compares the two, so it reports exactly what that codec keeps, changes, loses, or makes up. check_all runs it for every harness that can render. Losses are grouped as content (text, tool calls and results, images), accounting (usage, stop reasons, per-message model, thinking signatures) and metadata (title, model, branch, cwd). Blocks are compared as a flat stream, not message by message, because targets split and merge turns. JSON re-spacing and regenerated ids are not counted as loss. The result is serialisable. The measurement is codec-level and does not include what a store adds, such as artifact files. Tests: unit tests for each category, one golden table of what every harness loses on a rich session (also published in docs/formats), a proptest that no target panics and that Simple reports nothing, and the docs table is checked against the codecs so it cannot drift. Co-Authored-By: Claude Sonnet 5.5 --- docs/formats/README.md | 20 + src/fidelity.rs | 2602 +++++++++++++++++++++++++++++++++ src/lib.rs | 3 + tests/README.md | 7 +- tests/integration/fidelity.rs | 741 ++++++++++ tests/integration/main.rs | 5 +- 6 files changed, 3374 insertions(+), 4 deletions(-) create mode 100644 src/fidelity.rs create mode 100644 tests/integration/fidelity.rs diff --git a/docs/formats/README.md b/docs/formats/README.md index 633fdde..f3d1585 100644 --- a/docs/formats/README.md +++ b/docs/formats/README.md @@ -57,3 +57,23 @@ than none. | [pi.md](pi.md) | pi | `src/harness/pi.rs` | | [campfire.md](campfire.md) | Campfire (embeds pi) | `src/harness/campfire.rs` | | [simple.md](simple.md) | Simple (txcript's own interchange format) | `src/harness/simple.rs` | + +## Conversion fidelity + +What converting *into* each harness loses, measured by `txcript::fidelity` on one synthetic session that has every kind of block: signed and unsigned thinking, a Bash call with a description, an edit, an MCP call with a structured result, an error result, a slash command, an image, token usage, stop reasons, per-message models, a title, a git branch. Each cell names the categories that were lost or changed (`ok`: nothing). Items the target only made up, and message timestamps, are not listed. + +| Target | Content | Accounting | Metadata | +| --- | --- | --- | --- | +| claude_code | ok | changed: stop_reason | ok | +| codex | lost: tool_args, is_error; changed: tool_name | lost: thinking_signature, usage, stop_reason | lost: title, model | +| opencode | lost: tool_call, tool_result | lost: thinking_signature; changed: stop_reason | lost: git_branch | +| pi | lost: tool_call | lost: thinking_signature | lost: title, model, git_branch | +| campfire | lost: tool_call | lost: thinking_signature | lost: title, model, git_branch | +| cursor | lost: tool_call | lost: usage, stop_reason | lost: git_branch | +| cursor_desktop | lost: tool_call, tool_args, tool_result, image | lost: usage, stop_reason | lost: git_branch | +| grok | lost: tool_call | lost: thinking_signature, usage, stop_reason | ok | +| fx | lost: tool_call, tool_args | lost: usage, stop_reason | lost: git_branch | +| antigravity | lost: tool_call, tool_args, tool_result | lost: message_model | lost: model; changed: title | +| cowork | ok | changed: stop_reason | ok | + +The `tool_call` and `tool_result` losses are the slash command, which several formats have no way to record; `tool_name` is it written as `_release`. Hermes, Amp, Simple, and `grok_bot` are measured too (see the golden table in `tests/integration/fidelity.rs`) but are not listed here: `continue` cannot write the first three, and `grok_bot` mints a live agent. The measurement is codec-level: it does not include what a store adds, such as artifact files. It is a regression oracle as well as documentation: `tests/integration/fidelity.rs` fails when the table drifts from the codecs, and prints the rows to paste. diff --git a/src/fidelity.rs b/src/fidelity.rs new file mode 100644 index 0000000..fa41d5f --- /dev/null +++ b/src/fidelity.rs @@ -0,0 +1,2602 @@ +//! Conversion fidelity: what a conversion into a harness would lose, before +//! doing it. +//! +//! [`assess`] renders a canonical transcript as the target harness's native +//! text, parses that text back, and compares the two. Everything the target's +//! codec fails to persist shows up as a [`Loss`]. The result is a +//! *codec-level* measurement: it covers exactly what the codec writes into the +//! harness's session text, and nothing a [`Store`](crate::Store) adds or +//! changes around it (artifacts materialized as files, a fresh id and +//! timestamp, a re-homed working directory, the `OpenCode` import binary, the +//! Cursor `SQLite` database). There is no I/O, so it runs everywhere the +//! codecs do, WASM included. +//! +//! # How the comparison works +//! +//! Targets split and merge messages, so messages are never compared one to +//! one. Each transcript is flattened to a stream of typed items (text, +//! thinking, tool calls, tool results, images, artifacts) and the streams are +//! matched by content. What matches is preserved; leftovers that resemble +//! each other are [`LossKind::Changed`] (a renamed tool, an edited text); the +//! rest are [`LossKind::Lost`] when only the source has them and +//! [`LossKind::Synthesized`] when only the round trip does. Per-message +//! accounting (usage, stop reason, model, timestamp) and session metadata are +//! compared the same way, field by field. +//! +//! Normalization keeps formatting out of the verdict: text is trimmed, JSON +//! (tool inputs, structured or JSON-looking tool results) compares by value so +//! key order and whitespace never count, a list of text parts equals its +//! joined text, and regenerated ids are ignored. +//! +//! # Severity +//! +//! [`Severity`] ranks the worst thing a conversion does. Only +//! [`LossKind::Lost`] and [`LossKind::Changed`] items raise it; synthesized +//! items are listed but never fail a check. +//! +//! - [`Severity::Content`]: text, thinking, tool calls and arguments, results, +//! images, artifacts, error flags, block order. +//! - [`Severity::Accounting`]: usage, stop reasons, per-message models, +//! thinking signatures. +//! - [`Severity::Metadata`]: title, model, git branch, working directory, +//! timestamps, lineage. + +use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::hash::{DefaultHasher, Hash, Hasher}; + +use serde::Serialize; +use serde_json::Value; + +use crate::common::{Artifact, ArtifactSource, Block, Role, StopReason, ToolOutput, Usage}; +use crate::dispatch::{parse_to_common, render_from_common}; +use crate::{Common, HarnessId, Result, Transcript}; + +/// The harnesses a canonical transcript can be rendered into: every codec +/// except the live read-only sources (Claude Chat, cloud Cowork, `ChatGPT`). +/// +/// Whether a harness can also be *written* to disk (`continue --with`) is a +/// store question this module does not answer: Simple, Hermes, and Amp have +/// codecs but no store to write into. +pub const TARGETS: [HarnessId; 15] = [ + HarnessId::ClaudeCode, + HarnessId::Codex, + HarnessId::OpenCode, + HarnessId::Pi, + HarnessId::Campfire, + HarnessId::Cursor, + HarnessId::CursorDesktop, + HarnessId::Grok, + HarnessId::GrokBot, + HarnessId::Fx, + HarnessId::Hermes, + HarnessId::Amp, + HarnessId::Antigravity, + HarnessId::Simple, + HarnessId::Cowork, +]; + +/// Longest example kept in a [`Loss`], in characters. +const EXAMPLE_CAP: usize = 80; +/// Longest example kept under [`Options::detail`]. +const DETAIL_CAP: usize = 240; +/// How many look-alike candidates one leftover is compared against. +const PAIR_CANDIDATES: usize = 64; + +/// How bad a conversion is, by the worst class of thing it loses or changes. +/// Ordered: `None < Metadata < Accounting < Content`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Severity { + /// Nothing is lost or changed. + None, + /// Only session metadata differs. + Metadata, + /// Token accounting, stop reasons, or per-message models differ. + Accounting, + /// The conversation itself differs. + Content, +} + +impl Severity { + /// The stable lowercase name, as used in JSON and `--fail-on`. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Severity::None => "none", + Severity::Metadata => "metadata", + Severity::Accounting => "accounting", + Severity::Content => "content", + } + } +} + +/// What happened to an item on the way through the target. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum LossKind { + /// Present in the source, absent after the round trip. + Lost, + /// Present in both, but different. + Changed, + /// Absent in the source, present after the round trip. + Synthesized, +} + +/// What part of a transcript an item is about. Declared content first, then +/// accounting, then metadata; items sort in this order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Category { + /// Message text. + Text, + /// Thinking text. + Thinking, + /// A tool call that does not survive at all. + ToolCall, + /// A tool call that survives under another name. + ToolName, + /// A tool call that survives with different arguments. + ToolArgs, + /// A tool result. + ToolResult, + /// An inline image. + Image, + /// A named file. + Artifact, + /// A tool result's error flag. + IsError, + /// Blocks that survive but in a different order. + Order, + /// The opaque token that lets a harness replay thinking. + ThinkingSignature, + /// Per-message token accounting. + Usage, + /// Why a turn ended. + StopReason, + /// The model that produced a message. + MessageModel, + /// The session title. + Title, + /// The session's primary model. + Model, + /// The git branch. + GitBranch, + /// The working directory. + Cwd, + /// Session or message timestamps. + Timestamp, + /// How the session relates to another. + Lineage, +} + +impl Category { + /// The severity class this category belongs to. + #[must_use] + pub const fn severity(self) -> Severity { + match self { + Category::Text + | Category::Thinking + | Category::ToolCall + | Category::ToolName + | Category::ToolArgs + | Category::ToolResult + | Category::Image + | Category::Artifact + | Category::IsError + | Category::Order => Severity::Content, + Category::ThinkingSignature + | Category::Usage + | Category::StopReason + | Category::MessageModel => Severity::Accounting, + Category::Title + | Category::Model + | Category::GitBranch + | Category::Cwd + | Category::Timestamp + | Category::Lineage => Severity::Metadata, + } + } +} + +/// One kind of difference, with how many times it happened. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Loss { + /// Lost, changed, or synthesized. + pub kind: LossKind, + /// What it is about. + pub category: Category, + /// A finer label within the category: `Bash.description` for a dropped + /// tool argument, `/release -> _release` for a renamed tool, the usage + /// field that was dropped. + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, + /// How many items. + pub count: usize, + /// Index in the source of the first affected message. `None` for + /// synthesized items, which are not in the source. + #[serde(skip_serializing_if = "Option::is_none")] + pub first_at: Option, + /// A short sample: at most 80 characters, control characters replaced by + /// spaces, and never a tool result body or image data (unless + /// [`Options::detail`]). + #[serde(skip_serializing_if = "Option::is_none")] + pub example: Option, +} + +/// What converting a transcript into one harness costs. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Report { + /// The harness converted into. + pub target: HarnessId, + /// The worst class of loss; see the module docs. + pub severity: Severity, + /// Messages in the source. + pub messages_before: usize, + /// Messages after the round trip. Targets split and merge turns, so this + /// differing is not itself a loss. + pub messages_after: usize, + /// The differences, sorted by category, kind, then path. + pub items: Vec, + /// Things worth knowing that are not differences, such as an empty + /// session. + pub warnings: Vec, +} + +impl Report { + /// Total number of items of `kind` across every category. + #[must_use] + pub fn total(&self, kind: LossKind) -> usize { + self.items + .iter() + .filter(|item| item.kind == kind) + .map(|item| item.count) + .sum() + } +} + +/// Knobs for [`assess_with`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Options { + /// Keep longer examples, and a preview of tool results. Off by default so + /// a report stays safe to paste: counts and short samples only. + pub detail: bool, +} + +/// Assess converting `source` into `target`. +/// +/// # Errors +/// When the target's codec cannot render the transcript or cannot read its +/// own output back. +pub fn assess(target: HarnessId, source: &Transcript) -> Result { + assess_with(target, source, Options::default()) +} + +/// [`assess`] with options. +/// +/// # Errors +/// As [`assess`]. +pub fn assess_with( + target: HarnessId, + source: &Transcript, + options: Options, +) -> Result { + // Stores write artifacts out as files before the codec sees them; do the + // same on paper so the codec gets the shape it gets in real use. + let prepared = with_artifact_paths(source); + let text = render_from_common(target, &prepared)?; + let round_trip = parse_to_common(target, &text)?; + Ok(compare(target, source, &round_trip, options)) +} + +/// Whether an artifact's content travels in the transcript rather than in a +/// file. +fn held_inline(artifact: &Artifact) -> bool { + !matches!(artifact.source, ArtifactSource::Path { .. }) +} + +/// `source` with every inline artifact replaced by a file path, as the store +/// layer does before writing. Without such artifacts, `source` itself. +fn with_artifact_paths(source: &Transcript) -> Cow<'_, Transcript> { + let any = source + .body + .iter() + .flat_map(|m| &m.content) + .any(|b| matches!(b, Block::Artifact { artifact } if held_inline(artifact))); + if !any { + return Cow::Borrowed(source); + } + let mut prepared = source.clone(); + for (at, message) in prepared.body.iter_mut().enumerate() { + for (n, block) in message.content.iter_mut().enumerate() { + if let Block::Artifact { artifact } = block + && held_inline(artifact) + { + let media_type = match &artifact.source { + ArtifactSource::Text { media_type, .. } + | ArtifactSource::Base64 { media_type, .. } + | ArtifactSource::Path { media_type, .. } => media_type.clone(), + }; + let name = artifact.name.replace(['/', '\\'], "_"); + artifact.source = ArtifactSource::Path { + path: format!("/txcript-artifacts/{at}-{n}/{name}"), + media_type, + }; + } + } + } + Cow::Owned(prepared) +} + +/// Assess `source` against every harness in [`TARGETS`]. A target whose codec +/// fails is reported as an error under its own entry and never stops the +/// others. +#[must_use] +pub fn check_all(source: &Transcript) -> Vec<(HarnessId, Result)> { + TARGETS + .into_iter() + .map(|target| (target, assess(target, source))) + .collect() +} + +// ── comparison ───────────────────────────────────────────────────────── + +fn compare( + target: HarnessId, + source: &Transcript, + round_trip: &Transcript, + options: Options, +) -> Report { + let mut acc = Acc::new(options); + let src = flatten(source); + let rt = flatten(round_trip); + compare_content(&src, &rt, &mut acc); + compare_accounting(source, round_trip, &mut acc); + compare_meta(source, round_trip, &mut acc); + + let items: Vec = acc.items.into_values().collect(); + let severity = items + .iter() + .filter(|item| item.kind != LossKind::Synthesized) + .map(|item| item.category.severity()) + .max() + .unwrap_or(Severity::None); + let mut warnings = Vec::new(); + if source.body.is_empty() { + warnings.push("the session has 0 messages; there is nothing to lose".to_string()); + } + Report { + target, + severity, + messages_before: source.body.len(), + messages_after: round_trip.body.len(), + items, + warnings, + } +} + +/// Collects items, merging repeats of the same kind of difference. +struct Acc { + items: BTreeMap<(Category, LossKind, Option), Loss>, + cap: usize, + detail: bool, +} + +impl Acc { + fn new(options: Options) -> Acc { + Acc { + items: BTreeMap::new(), + cap: if options.detail { + DETAIL_CAP + } else { + EXAMPLE_CAP + }, + detail: options.detail, + } + } + + fn add( + &mut self, + kind: LossKind, + category: Category, + path: Option, + at: Option, + example: impl FnOnce() -> Option, + ) { + self.add_n(kind, category, path, at, 1, example); + } + + fn add_n( + &mut self, + kind: LossKind, + category: Category, + path: Option, + at: Option, + count: usize, + example: impl FnOnce() -> Option, + ) { + if count == 0 { + return; + } + let cap = self.cap; + let entry = self + .items + .entry((category, kind, path.clone())) + .or_insert_with(|| Loss { + kind, + category, + path, + count: 0, + first_at: at, + example: example().map(|e| clip(&e, cap)), + }); + entry.count += count; + entry.first_at = match (entry.first_at, at) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }; + } +} + +/// `s` on one line, at most `cap` characters, with an ellipsis when cut. +fn clip(s: &str, cap: usize) -> String { + let mut out = String::new(); + let mut chars = s + .trim() + .chars() + .map(|c| if c.is_control() { ' ' } else { c }); + for _ in 0..cap.saturating_sub(1) { + match chars.next() { + Some(c) => out.push(c), + None => return out, + } + } + match (chars.next(), chars.next()) { + (None, _) => {} + (Some(c), None) => out.push(c), + (Some(_), Some(_)) => out.push('…'), + } + out +} + +/// `before -> after`, each side cut to the stretch around where the two start +/// to differ, with line breaks and tabs shown rather than collapsed, so a +/// change in trailing whitespace is visible. +fn arrow(before: &str, after: &str, cap: usize) -> String { + let half = cap.saturating_sub(4) / 2; + let fits = |s: &str| s.chars().take(half + 1).count() <= half; + let skip = if fits(before) && fits(after) { + 0 + } else { + let shared = before + .chars() + .zip(after.chars()) + .take_while(|(x, y)| x == y) + .count(); + shared.saturating_sub(8) + }; + let side = |s: &str| { + let mut out: Vec = Vec::new(); + if skip > 0 { + out.push('…'); + } + for c in s.chars().skip(skip) { + match c { + '\n' => out.extend(['\\', 'n']), + '\t' => out.extend(['\\', 't']), + '\r' => out.extend(['\\', 'r']), + c if c.is_control() => out.push(' '), + c => out.push(c), + } + if out.len() > half { + out.truncate(half.saturating_sub(1)); + out.push('…'); + break; + } + } + out.into_iter().collect::() + }; + format!("{} -> {}", side(before), side(after)) +} + +// ── content stream ───────────────────────────────────────────────────── + +/// A tool result reduced to what it says. +enum Proj<'a> { + Text(Cow<'a, str>), + Json(Cow<'a, Value>), +} + +impl<'a> Proj<'a> { + fn into_text(self) -> Cow<'a, str> { + match self { + Proj::Text(s) => s, + Proj::Json(v) => Cow::Owned(v.to_string()), + } + } +} + +enum Body<'a> { + Text { + role: Role, + text: &'a str, + }, + Thinking { + text: &'a str, + signature: Option<&'a str>, + encrypted: Option<&'a str>, + }, + Call { + name: String, + input: Value, + }, + Result { + output: &'a ToolOutput, + is_error: bool, + }, + Image { + media_type: &'a str, + }, + Artifact { + name: &'a str, + }, +} + +/// One block of a flattened transcript. +struct Item<'a> { + /// Identity of the block's content, ids and formatting excluded. + key: u64, + /// The message the block came from. + at: usize, + body: Body<'a>, +} + +fn hasher(tag: u8) -> DefaultHasher { + let mut h = DefaultHasher::new(); + tag.hash(&mut h); + h +} + +/// Hash a JSON value by content: object keys in sorted order, so neither key +/// order nor whitespace in the source text affects it. +fn hash_value(v: &Value, h: &mut DefaultHasher) { + match v { + Value::Null => 0u8.hash(h), + Value::Bool(b) => (1u8, b).hash(h), + Value::Number(n) => { + 2u8.hash(h); + if let Some(u) = n.as_u64() { + (0u8, u).hash(h); + } else if let Some(i) = n.as_i64() { + (1u8, i).hash(h); + } else { + (2u8, n.as_f64().map(f64::to_bits)).hash(h); + } + } + Value::String(s) => (3u8, s).hash(h), + Value::Array(items) => { + (4u8, items.len()).hash(h); + for item in items { + hash_value(item, h); + } + } + Value::Object(map) => { + (5u8, map.len()).hash(h); + let mut keys: Vec<&String> = map.keys().collect(); + keys.sort(); + for key in keys { + key.hash(h); + hash_value(&map[key], h); + } + } + } +} + +/// The text of a result that is a list of plain text parts. +fn text_parts(parts: &[Value]) -> Option { + let mut texts = Vec::with_capacity(parts.len()); + for part in parts { + let map = part.as_object()?; + if map.get("type")?.as_str()? != "text" || map.keys().any(|k| k != "type" && k != "text") { + return None; + } + texts.push(map.get("text")?.as_str()?); + } + (!texts.is_empty()).then(|| texts.join("\n")) +} + +fn project(output: &ToolOutput) -> Proj<'_> { + /// Text that is JSON compares as that JSON, so re-spacing it is not a + /// change. `depth` bounds how many layers of JSON-in-a-string are peeled. + fn of_text(t: Cow<'_, str>, depth: u8) -> Proj<'_> { + if depth > 0 + && t.starts_with(['{', '[']) + && let Ok(v) = serde_json::from_str::(&t) + { + return of_value(Cow::Owned(v), depth - 1); + } + Proj::Text(t) + } + fn of_value(v: Cow<'_, Value>, depth: u8) -> Proj<'_> { + match &*v { + Value::String(s) => of_text(Cow::Owned(s.trim().to_string()), depth), + Value::Array(parts) => match text_parts(parts) { + Some(joined) => of_text(Cow::Owned(joined.trim().to_string()), depth), + None => Proj::Json(v), + }, + _ => Proj::Json(v), + } + } + match output { + ToolOutput::Text(s) => of_text(Cow::Borrowed(s.trim()), 3), + ToolOutput::Json(v) => of_value(Cow::Borrowed(v), 3), + } +} + +fn flatten(t: &Transcript) -> Vec> { + let mut out = Vec::new(); + for (at, message) in t.body.iter().enumerate() { + for block in &message.content { + let item = match block { + Block::Text { text } => { + let text = text.trim(); + if text.is_empty() { + continue; + } + let mut h = hasher(0); + (message.role == Role::User, text).hash(&mut h); + Item { + key: h.finish(), + at, + body: Body::Text { + role: message.role, + text, + }, + } + } + Block::Thinking { + text, + signature, + encrypted, + } => { + let text = text.trim(); + if text.is_empty() { + continue; + } + let mut h = hasher(1); + text.hash(&mut h); + Item { + key: h.finish(), + at, + body: Body::Thinking { + text, + signature: signature.as_deref(), + encrypted: encrypted.as_deref(), + }, + } + } + Block::ToolUse { tool, .. } => { + let (name, input) = tool.to_canonical(); + let mut h = hasher(2); + name.hash(&mut h); + hash_value(&input, &mut h); + Item { + key: h.finish(), + at, + body: Body::Call { name, input }, + } + } + Block::ToolResult { + content, is_error, .. + } => { + // Only the hash is kept: a result can be megabytes, and the + // projection is rebuilt for the few that need comparing. + let mut h = hasher(3); + match &project(content) { + Proj::Text(s) => (0u8, s.as_ref()).hash(&mut h), + Proj::Json(v) => { + 1u8.hash(&mut h); + hash_value(v, &mut h); + } + } + Item { + key: h.finish(), + at, + body: Body::Result { + output: content, + is_error: *is_error, + }, + } + } + Block::Image { source } => { + let mut h = hasher(4); + (&source.media_type, &source.data).hash(&mut h); + Item { + key: h.finish(), + at, + body: Body::Image { + media_type: &source.media_type, + }, + } + } + Block::Artifact { artifact } => Item { + key: artifact_key(artifact), + at, + body: Body::Artifact { + name: &artifact.name, + }, + }, + }; + out.push(item); + } + } + out +} + +/// An artifact's identity is its name: the store writes its content to a file +/// of its own, which the codec never carries, so content is not compared. +fn artifact_key(artifact: &Artifact) -> u64 { + let mut h = hasher(5); + artifact.name.hash(&mut h); + h.finish() +} + +/// Match equal keys one to one, in order. Returns the matched index pairs, +/// then the unmatched indices of each side. +fn match_keys(src: &[u64], rt: &[u64]) -> (Vec<(usize, usize)>, Vec, Vec) { + let mut index: HashMap> = HashMap::new(); + for (j, key) in rt.iter().enumerate() { + index.entry(*key).or_default().push_back(j); + } + let mut used = vec![false; rt.len()]; + let mut pairs = Vec::new(); + let mut src_left = Vec::new(); + for (i, key) in src.iter().enumerate() { + match index.get_mut(key).and_then(VecDeque::pop_front) { + Some(j) => { + used[j] = true; + pairs.push((i, j)); + } + None => src_left.push(i), + } + } + let rt_left = (0..rt.len()).filter(|j| !used[*j]).collect(); + (pairs, src_left, rt_left) +} + +/// The source positions of matched items that left the longest in-order run +/// behind: the blocks that moved. `pairs` is sorted by source position. +fn moved(pairs: &[(usize, usize)]) -> Vec { + let mut tails: Vec = Vec::new(); + let mut prev = vec![None; pairs.len()]; + for (k, &(_, j)) in pairs.iter().enumerate() { + let pos = tails.partition_point(|&t| pairs[t].1 < j); + prev[k] = pos.checked_sub(1).map(|p| tails[p]); + if pos == tails.len() { + tails.push(k); + } else { + tails[pos] = k; + } + } + let mut keep = vec![false; pairs.len()]; + let mut cursor = tails.last().copied(); + while let Some(k) = cursor { + keep[k] = true; + cursor = prev[k]; + } + pairs + .iter() + .zip(keep) + .filter(|(_, kept)| !kept) + .map(|(pair, _)| pair.0) + .collect() +} + +/// Whitespace-insensitive equality, without allocating. +fn same_words(a: &str, b: &str) -> bool { + a.split_whitespace().eq(b.split_whitespace()) +} + +/// The first few characters of `s` with whitespace collapsed: two texts can +/// only resemble each other when these agree. +fn lead(s: &str) -> u64 { + let mut h = DefaultHasher::new(); + let mut taken = 0; + for word in s.split_whitespace() { + for c in word.chars() { + if taken >= 6 { + return h.finish(); + } + c.hash(&mut h); + taken += 1; + } + ' '.hash(&mut h); + taken += 1; + } + h.finish() +} + +/// Whether two leftover texts are probably the same text, edited: equal but +/// for whitespace, or opening with the same eight characters. +fn related(a: &str, b: &str) -> bool { + same_words(a, b) || a.chars().zip(b.chars()).take_while(|(p, q)| p == q).count() >= 8 +} + +/// A leftover tool call's identity for pairing: its name and each argument. +fn call_keys(name: &str, input: &Value) -> Vec<(u8, u64)> { + let mut by_name = hasher(10); + name.hash(&mut by_name); + let mut by_input = hasher(11); + hash_value(input, &mut by_input); + vec![(0, by_name.finish()), (1, by_input.finish())] +} + +/// Whether two leftover calls are the same call, altered. +fn related_calls(a: (&str, &Value), b: (&str, &Value)) -> bool { + if a.1 == b.1 { + return true; + } + if a.0 != b.0 { + return false; + } + match (a.1.as_object(), b.1.as_object()) { + (Some(x), Some(y)) => x.iter().any(|(k, v)| y.get(k) == Some(v)), + _ => false, + } +} + +/// Pair leftover source items with leftover round-trip items that resemble +/// them. Candidates come from `keys` buckets; `related` has the last word. +/// Returns the pairs and what stayed unpaired on each side. +fn pair_up( + src_left: &[usize], + rt_left: &[usize], + src_keys: &dyn Fn(usize) -> Vec<(u8, u64)>, + rt_keys: &dyn Fn(usize) -> Vec<(u8, u64)>, + related: &dyn Fn(usize, usize) -> bool, +) -> (Vec<(usize, usize)>, Vec, Vec) { + let mut buckets: HashMap<(u8, u64), Vec> = HashMap::new(); + for &j in rt_left { + for key in rt_keys(j) { + buckets.entry(key).or_default().push(j); + } + } + let mut used: HashSet = HashSet::new(); + let mut cursors: HashMap<(u8, u64), usize> = HashMap::new(); + let mut pairs = Vec::new(); + let mut src_unpaired = Vec::new(); + for &i in src_left { + let mut found = None; + for key in src_keys(i) { + let Some(bucket) = buckets.get(&key) else { + continue; + }; + let cursor = cursors.entry(key).or_insert(0); + while *cursor < bucket.len() && used.contains(&bucket[*cursor]) { + *cursor += 1; + } + found = bucket[*cursor..] + .iter() + .filter(|j| !used.contains(j)) + .take(PAIR_CANDIDATES) + .find(|&&j| related(i, j)) + .copied(); + if found.is_some() { + break; + } + } + match found { + Some(j) => { + used.insert(j); + pairs.push((i, j)); + } + None => src_unpaired.push(i), + } + } + let rt_unpaired = rt_left + .iter() + .copied() + .filter(|j| !used.contains(j)) + .collect(); + (pairs, src_unpaired, rt_unpaired) +} + +fn category_of(body: &Body<'_>) -> Category { + match body { + Body::Text { .. } => Category::Text, + Body::Thinking { .. } => Category::Thinking, + Body::Call { .. } => Category::ToolCall, + Body::Result { .. } => Category::ToolResult, + Body::Image { .. } => Category::Image, + Body::Artifact { .. } => Category::Artifact, + } +} + +/// A short description of an item for an example. +fn describe(body: &Body<'_>, detail: bool) -> Option { + match body { + Body::Text { text, .. } | Body::Thinking { text, .. } => Some((*text).to_string()), + Body::Call { name, input } => Some(call_label(name, input)), + Body::Result { output, .. } => detail.then(|| project(output).into_text().into_owned()), + Body::Image { media_type } => Some((*media_type).to_string()), + Body::Artifact { name } => Some((*name).to_string()), + } +} + +/// `Bash: cargo test`: the tool and its most identifying argument. +fn call_label(name: &str, input: &Value) -> String { + const PRIMARY: [&str; 6] = ["command", "file_path", "path", "url", "pattern", "query"]; + let arg = input.as_object().and_then(|map| { + PRIMARY + .iter() + .find_map(|key| map.get(*key).and_then(Value::as_str)) + }); + match arg { + Some(arg) => format!("{name}: {arg}"), + None => name.to_string(), + } +} + +/// One round of leftover pairing for a kind of block: the pairs, and what +/// stayed unpaired on each side. +type Pairing = (Vec<(usize, usize)>, Vec, Vec); + +fn leftover_texts(src: &[Item<'_>], rt: &[Item<'_>], s: &[usize], r: &[usize]) -> Pairing { + fn text<'a>(items: &[Item<'a>], n: usize) -> &'a str { + match &items[n].body { + Body::Text { text, .. } | Body::Thinking { text, .. } => text, + _ => "", + } + } + let keys = |items: &[Item<'_>], n: usize| { + let tag = u8::from(matches!(items[n].body, Body::Thinking { .. })); + vec![(tag, lead(text(items, n)))] + }; + pair_up(s, r, &|i| keys(src, i), &|j| keys(rt, j), &|i, j| { + // A text never pairs with a thinking block. + keys(src, i) == keys(rt, j) && related(text(src, i), text(rt, j)) + }) +} + +fn leftover_calls(src: &[Item<'_>], rt: &[Item<'_>], s: &[usize], r: &[usize]) -> Pairing { + fn call<'i>(items: &'i [Item<'_>], n: usize) -> Option<(&'i str, &'i Value)> { + match &items[n].body { + Body::Call { name, input } => Some((name.as_str(), input)), + _ => None, + } + } + let keys = |items: &[Item<'_>], n: usize| { + call(items, n).map_or_else(Vec::new, |(name, input)| call_keys(name, input)) + }; + pair_up( + s, + r, + &|i| keys(src, i), + &|j| keys(rt, j), + &|i, j| match (call(src, i), call(rt, j)) { + (Some(a), Some(b)) => related_calls(a, b), + _ => false, + }, + ) +} + +fn leftover_results(src: &[Item<'_>], rt: &[Item<'_>], s: &[usize], r: &[usize]) -> Pairing { + /// What each leftover result says, worked out once. + fn texts<'a>(items: &[Item<'a>], idx: &[usize]) -> HashMap> { + idx.iter() + .filter_map(|&n| match &items[n].body { + Body::Result { output, .. } => Some((n, project(output).into_text())), + _ => None, + }) + .collect() + } + let (said_src, said_rt) = (texts(src, s), texts(rt, r)); + let key = |said: &HashMap>, n: usize| { + vec![(2, lead(said.get(&n).map_or("", AsRef::as_ref)))] + }; + pair_up( + s, + r, + &|i| key(&said_src, i), + &|j| key(&said_rt, j), + &|i, j| match (said_src.get(&i), said_rt.get(&j)) { + (Some(a), Some(b)) => related(a, b), + _ => false, + }, + ) +} + +/// Pair leftovers by a coarse label alone (an image's media type, an +/// artifact's name): same label, same item. +fn leftover_labeled( + src: &[Item<'_>], + rt: &[Item<'_>], + s: &[usize], + r: &[usize], + label: for<'a> fn(&'a Body<'_>) -> Option<&'a str>, +) -> Pairing { + let keys = |items: &[Item<'_>], n: usize| { + label(&items[n].body).map_or_else(Vec::new, |l| { + let mut h = hasher(14); + l.hash(&mut h); + vec![(5, h.finish())] + }) + }; + pair_up(s, r, &|i| keys(src, i), &|j| keys(rt, j), &|_, _| true) +} + +/// Blocks that matched by content may still differ in what rides along: +/// a thinking block's replay token, a result's error flag. +fn compare_matched(src: &[Item<'_>], rt: &[Item<'_>], matched: &[(usize, usize)], acc: &mut Acc) { + for &(i, j) in matched { + match (&src[i].body, &rt[j].body) { + (Body::Thinking { .. }, Body::Thinking { .. }) => { + compare_token(&src[i], &rt[j], acc); + } + (Body::Result { is_error: was, .. }, Body::Result { is_error: now, .. }) + if was != now => + { + if *was { + acc.add( + LossKind::Lost, + Category::IsError, + None, + Some(src[i].at), + || None, + ); + } else { + acc.add(LossKind::Synthesized, Category::IsError, None, None, || { + None + }); + } + } + _ => {} + } + } +} + +fn compare_content(src: &[Item<'_>], rt: &[Item<'_>], acc: &mut Acc) { + let src_keys: Vec = src.iter().map(|i| i.key).collect(); + let rt_keys: Vec = rt.iter().map(|i| i.key).collect(); + let (matched, src_left, rt_left) = match_keys(&src_keys, &rt_keys); + let (detail, cap) = (acc.detail, acc.cap); + + compare_matched(src, rt, &matched, acc); + + let out_of_order = moved(&matched); + if let Some(&first) = out_of_order.first() { + acc.add_n( + LossKind::Changed, + Category::Order, + None, + Some(src[first].at), + out_of_order.len(), + || None, + ); + } + + // Leftovers that resemble each other are one item changed; the rest are + // lost (source only) or synthesized (round trip only). + let split = |items: &[Item<'_>], idx: &[usize], category: Category| -> Vec { + idx.iter() + .copied() + .filter(|&n| category_of(&items[n].body) == category) + .collect() + }; + let by = |category: Category| { + ( + split(src, &src_left, category), + split(rt, &rt_left, category), + ) + }; + let mut unpaired_src: Vec = Vec::new(); + let mut unpaired_rt: Vec = Vec::new(); + let mut settle = |(pairs, su, ru): Pairing, on_pair: &mut dyn FnMut(usize, usize)| { + for (i, j) in pairs { + on_pair(i, j); + } + unpaired_src.extend(su); + unpaired_rt.extend(ru); + }; + + // Text and thinking pair separately from each other. + for category in [Category::Text, Category::Thinking] { + let (s, r) = by(category); + settle(leftover_texts(src, rt, &s, &r), &mut |i, j| { + changed_text(&src[i], &rt[j], acc); + }); + } + let (s, r) = by(Category::ToolCall); + settle(leftover_calls(src, rt, &s, &r), &mut |i, j| { + changed_call(&src[i], &rt[j], acc); + }); + let (s, r) = by(Category::ToolResult); + settle(leftover_results(src, rt, &s, &r), &mut |i, j| { + acc.add( + LossKind::Changed, + Category::ToolResult, + None, + Some(src[i].at), + || { + detail.then(|| { + arrow( + &describe_full(&src[i].body), + &describe_full(&rt[j].body), + cap, + ) + }) + }, + ); + }); + let (s, r) = by(Category::Image); + settle( + leftover_labeled(src, rt, &s, &r, |b| match b { + Body::Image { media_type } => Some(media_type), + _ => None, + }), + &mut |i, _| { + acc.add( + LossKind::Changed, + Category::Image, + None, + Some(src[i].at), + || describe(&src[i].body, false), + ); + }, + ); + let (s, r) = by(Category::Artifact); + settle( + leftover_labeled(src, rt, &s, &r, |b| match b { + Body::Artifact { name } => Some(name), + _ => None, + }), + &mut |i, _| { + acc.add( + LossKind::Changed, + Category::Artifact, + None, + Some(src[i].at), + || describe(&src[i].body, false), + ); + }, + ); + + emit_unpaired(src, rt, unpaired_src, unpaired_rt, acc); +} + +/// What no counterpart was found for: lost from the source, synthesized in +/// the round trip. +fn emit_unpaired( + src: &[Item<'_>], + rt: &[Item<'_>], + mut unpaired_src: Vec, + mut unpaired_rt: Vec, + acc: &mut Acc, +) { + let detail = acc.detail; + unpaired_src.sort_unstable(); + unpaired_rt.sort_unstable(); + for i in unpaired_src { + let item = &src[i]; + acc.add( + LossKind::Lost, + category_of(&item.body), + None, + Some(item.at), + || describe(&item.body, detail), + ); + } + for j in unpaired_rt { + let item = &rt[j]; + acc.add( + LossKind::Synthesized, + category_of(&item.body), + None, + None, + || describe(&item.body, detail), + ); + } +} + +fn describe_full(body: &Body<'_>) -> String { + describe(body, true).unwrap_or_default() +} + +/// A matched thinking block whose replay token did not survive intact. +fn compare_token(src: &Item<'_>, rt: &Item<'_>, acc: &mut Acc) { + let ( + Body::Thinking { + signature: s_sig, + encrypted: s_enc, + .. + }, + Body::Thinking { + signature: r_sig, + encrypted: r_enc, + .. + }, + ) = (&src.body, &rt.body) + else { + return; + }; + if (s_sig, s_enc) == (r_sig, r_enc) { + return; + } + let had = s_sig.is_some() || s_enc.is_some(); + let has = r_sig.is_some() || r_enc.is_some(); + let (kind, at) = match (had, has) { + (true, false) => (LossKind::Lost, Some(src.at)), + (false, true) => (LossKind::Synthesized, None), + _ => (LossKind::Changed, Some(src.at)), + }; + acc.add(kind, Category::ThinkingSignature, None, at, || None); +} + +/// Two leftover texts that resemble each other. +fn changed_text(src: &Item<'_>, rt: &Item<'_>, acc: &mut Acc) { + let cap = acc.cap; + match (&src.body, &rt.body) { + ( + Body::Text { + role: was, + text: before, + }, + Body::Text { + role: now, + text: after, + }, + ) => { + if was != now && same_words(before, after) { + let name = |r: &Role| { + if *r == Role::User { + "user" + } else { + "assistant" + } + }; + acc.add( + LossKind::Changed, + Category::Text, + Some("role".to_string()), + Some(src.at), + || Some(format!("{} -> {}", name(was), name(now))), + ); + } else { + acc.add( + LossKind::Changed, + Category::Text, + None, + Some(src.at), + || Some(arrow(before, after, cap)), + ); + } + } + (Body::Thinking { text: before, .. }, Body::Thinking { text: after, .. }) => { + acc.add( + LossKind::Changed, + Category::Thinking, + None, + Some(src.at), + || Some(arrow(before, after, cap)), + ); + compare_token(src, rt, acc); + } + _ => {} + } +} + +fn show(v: &Value) -> String { + v.as_str().map_or_else(|| v.to_string(), str::to_string) +} + +/// Two leftover calls that are the same call, altered: a new name, or +/// arguments that were dropped or rewritten. +fn changed_call(src: &Item<'_>, rt: &Item<'_>, acc: &mut Acc) { + let ( + Body::Call { + name: s_name, + input: s_in, + }, + Body::Call { + name: r_name, + input: r_in, + }, + ) = (&src.body, &rt.body) + else { + return; + }; + let at = Some(src.at); + let cap = acc.cap; + if s_name != r_name { + acc.add( + LossKind::Changed, + Category::ToolName, + Some(format!("{s_name} -> {r_name}")), + at, + || None, + ); + } + match (s_in.as_object(), r_in.as_object()) { + (Some(before), Some(after)) => { + fn sorted( + map: &serde_json::Map, + keep: impl Fn(&String) -> bool, + ) -> Vec<&String> { + let mut keys: Vec<&String> = map.keys().filter(|k| keep(k)).collect(); + keys.sort(); + keys + } + for key in sorted(before, |k| after.contains_key(k)) { + if after[key] != before[key] { + acc.add( + LossKind::Changed, + Category::ToolArgs, + Some(format!("{s_name}.{key}")), + at, + || Some(arrow(&show(&before[key]), &show(&after[key]), cap)), + ); + } + } + let mut added = sorted(after, |k| !before.contains_key(k)); + for key in sorted(before, |k| !after.contains_key(k)) { + // The same value under another key is a rename, not a loss. + if let Some(pos) = added.iter().position(|k| after[*k] == before[key]) { + let new = added.remove(pos); + acc.add( + LossKind::Changed, + Category::ToolArgs, + Some(format!("{s_name}.{key} -> {new}")), + at, + || Some(show(&before[key])), + ); + } else { + acc.add( + LossKind::Lost, + Category::ToolArgs, + Some(format!("{s_name}.{key}")), + at, + || Some(show(&before[key])), + ); + } + } + for key in added { + acc.add( + LossKind::Synthesized, + Category::ToolArgs, + Some(format!("{s_name}.{key}")), + None, + || Some(show(&after[key])), + ); + } + } + _ if s_in != r_in => acc.add( + LossKind::Changed, + Category::ToolArgs, + Some(s_name.clone()), + at, + || Some(arrow(&show(s_in), &show(r_in), cap)), + ), + _ => {} + } +} + +// ── accounting ───────────────────────────────────────────────────────── + +fn str_key(s: &str) -> u64 { + let mut h = DefaultHasher::new(); + s.hash(&mut h); + h.finish() +} + +fn stop_name(reason: &StopReason) -> String { + match reason { + StopReason::EndTurn => "end_turn".to_string(), + StopReason::ToolUse => "tool_use".to_string(), + StopReason::MaxTokens => "max_tokens".to_string(), + StopReason::StopSequence => "stop_sequence".to_string(), + StopReason::Aborted => "aborted".to_string(), + StopReason::Error => "error".to_string(), + StopReason::Other(other) => format!("other({other})"), + } +} + +/// Compare per-message values that targets may move between messages: equal +/// values cancel out; when the same number are left on each side they are +/// the same slots changed, otherwise the surplus is lost or synthesized. +fn compare_values(category: Category, src: &[(usize, String)], rt: &[String], acc: &mut Acc) { + let s_keys: Vec = src.iter().map(|(_, v)| str_key(v)).collect(); + let r_keys: Vec = rt.iter().map(|v| str_key(v)).collect(); + let (_, src_left, rt_left) = match_keys(&s_keys, &r_keys); + let cap = acc.cap; + if !src_left.is_empty() && src_left.len() == rt_left.len() { + for (&i, &j) in src_left.iter().zip(&rt_left) { + acc.add(LossKind::Changed, category, None, Some(src[i].0), || { + Some(arrow(&src[i].1, &rt[j], cap)) + }); + } + return; + } + for &i in &src_left { + acc.add(LossKind::Lost, category, None, Some(src[i].0), || { + Some(src[i].1.clone()) + }); + } + for &j in &rt_left { + acc.add(LossKind::Synthesized, category, None, None, || { + Some(rt[j].clone()) + }); + } +} + +/// A usage record's fields. An absent cache count and a zero one both mean +/// "nothing cached", so they compare equal. +fn usage_fields(u: &Usage) -> [(&'static str, u64); 4] { + [ + ("input_tokens", u.input_tokens), + ("output_tokens", u.output_tokens), + ( + "cache_read_input_tokens", + u.cache_read_input_tokens.unwrap_or(0), + ), + ( + "cache_creation_input_tokens", + u.cache_creation_input_tokens.unwrap_or(0), + ), + ] +} + +fn compare_usage(src: &[(usize, Usage)], rt: &[Usage], acc: &mut Acc) { + let key = |u: &Usage| { + let mut h = DefaultHasher::new(); + usage_fields(u).hash(&mut h); + h.finish() + }; + // The same turn with its cache counts dropped still has the same input + // and output counts. + let core = |u: &Usage| { + let mut h = DefaultHasher::new(); + (u.input_tokens, u.output_tokens).hash(&mut h); + h.finish() + }; + let s_keys: Vec = src.iter().map(|(_, u)| key(u)).collect(); + let r_keys: Vec = rt.iter().map(key).collect(); + let (_, src_left, rt_left) = match_keys(&s_keys, &r_keys); + let s_cores: Vec = src_left.iter().map(|&i| core(&src[i].1)).collect(); + let r_cores: Vec = rt_left.iter().map(|&j| core(&rt[j])).collect(); + let (pairs, s_gone, r_new) = match_keys(&s_cores, &r_cores); + let mut fieldwise = |i: usize, j: usize| { + let at = Some(src[i].0); + for ((name, was), (_, now)) in usage_fields(&src[i].1) + .into_iter() + .zip(usage_fields(&rt[j])) + { + let path = Some(name.to_string()); + match (was, now) { + (a, b) if a == b => {} + (_, 0) => acc.add(LossKind::Lost, Category::Usage, path, at, || None), + (0, _) => acc.add(LossKind::Synthesized, Category::Usage, path, None, || None), + (a, b) => acc.add(LossKind::Changed, Category::Usage, path, at, || { + Some(format!("{a} -> {b}")) + }), + } + } + }; + for (a, b) in pairs { + fieldwise(src_left[a], rt_left[b]); + } + // Nothing alike but as many records on each side: the same turns, + // counted differently. + if !s_gone.is_empty() && s_gone.len() == r_new.len() { + for (&a, &b) in s_gone.iter().zip(&r_new) { + fieldwise(src_left[a], rt_left[b]); + } + return; + } + for a in s_gone { + acc.add( + LossKind::Lost, + Category::Usage, + None, + Some(src[src_left[a]].0), + || None, + ); + } + for _ in r_new { + acc.add(LossKind::Synthesized, Category::Usage, None, None, || None); + } +} + +fn compare_accounting(src: &Transcript, rt: &Transcript, acc: &mut Acc) { + let usage: Vec<(usize, Usage)> = src + .body + .iter() + .enumerate() + .filter_map(|(at, m)| m.usage.map(|u| (at, u))) + .collect(); + let usage_rt: Vec = rt.body.iter().filter_map(|m| m.usage).collect(); + compare_usage(&usage, &usage_rt, acc); + + let stops = |t: &Transcript| -> Vec<(usize, String)> { + t.body + .iter() + .enumerate() + .filter_map(|(at, m)| m.stop_reason.as_ref().map(|r| (at, stop_name(r)))) + .collect() + }; + let rt_stops: Vec = stops(rt).into_iter().map(|(_, v)| v).collect(); + compare_values(Category::StopReason, &stops(src), &rt_stops, acc); + + let models = |t: &Transcript| -> Vec<(usize, String)> { + t.body + .iter() + .enumerate() + .filter_map(|(at, m)| { + let model = m.model.as_deref().map(str::trim).filter(|m| !m.is_empty()); + model.map(|model| (at, model.to_string())) + }) + .collect() + }; + let rt_models: Vec = models(rt).into_iter().map(|(_, v)| v).collect(); + compare_values(Category::MessageModel, &models(src), &rt_models, acc); + + // Targets that do not keep message times leave the source's seconds + // missing from the round trip. + let kept: HashSet = rt.body.iter().map(|m| m.timestamp.timestamp()).collect(); + for (at, m) in src.body.iter().enumerate() { + if !kept.contains(&m.timestamp.timestamp()) { + acc.add( + LossKind::Lost, + Category::Timestamp, + Some("message".to_string()), + Some(at), + || Some(m.timestamp.to_rfc3339()), + ); + } + } +} + +// ── metadata ─────────────────────────────────────────────────────────── + +fn compare_field(category: Category, was: Option<&str>, now: Option<&str>, acc: &mut Acc) { + // An empty value is no value. + let (was, now) = (was.filter(|s| !s.is_empty()), now.filter(|s| !s.is_empty())); + let cap = acc.cap; + match (was, now) { + (Some(a), None) => acc.add(LossKind::Lost, category, None, None, || Some(a.to_string())), + (Some(a), Some(b)) if a != b => { + acc.add(LossKind::Changed, category, None, None, || { + Some(arrow(a, b, cap)) + }); + } + (None, Some(b)) => acc.add(LossKind::Synthesized, category, None, None, || { + Some(b.to_string()) + }), + _ => {} + } +} + +fn compare_meta(src: &Transcript, rt: &Transcript, acc: &mut Acc) { + let (s, r) = (&src.meta, &rt.meta); + compare_field( + Category::Title, + s.title.as_deref().map(str::trim), + r.title.as_deref().map(str::trim), + acc, + ); + compare_field(Category::Model, s.model.as_deref(), r.model.as_deref(), acc); + compare_field( + Category::GitBranch, + s.git_branch.as_deref(), + r.git_branch.as_deref(), + acc, + ); + compare_field(Category::Cwd, s.cwd.as_deref(), r.cwd.as_deref(), acc); + let lineage = + |l: &crate::common::Lineage| format!("{:?} of {}", l.relation, l.parent).to_lowercase(); + compare_field( + Category::Lineage, + s.lineage.as_ref().map(lineage).as_deref(), + r.lineage.as_ref().map(lineage).as_deref(), + acc, + ); +} + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] + + use chrono::{DateTime, Utc}; + use serde_json::json; + + use super::*; + use crate::common::{ImageSource, Lineage, Message, Meta, Relation, Tool}; + + fn at(secs: i64) -> DateTime { + DateTime::from_timestamp(1_790_000_000 + secs, 0).unwrap() + } + + fn meta() -> Meta { + Meta { + id: "t".into(), + timestamp: at(0), + cwd: None, + git_branch: None, + title: None, + cli_version: None, + model: None, + lineage: None, + } + } + + fn msg(role: Role, content: Vec, secs: i64) -> Message { + Message { + role, + content, + timestamp: at(secs), + model: None, + stop_reason: None, + usage: None, + } + } + + fn text(s: &str) -> Block { + Block::Text { text: s.into() } + } + + fn call(id: &str, name: &str, input: Value) -> Block { + Block::ToolUse { + id: id.into(), + tool: Tool::from_canonical(name, input), + } + } + + fn result(id: &str, out: ToolOutput) -> Block { + Block::ToolResult { + tool_use_id: id.into(), + content: out, + is_error: false, + } + } + + fn thinking(s: &str, signature: Option<&str>) -> Block { + Block::Thinking { + text: s.into(), + signature: signature.map(Into::into), + encrypted: None, + } + } + + fn convo(messages: Vec) -> Transcript { + Transcript::new(meta(), messages) + } + + fn user(blocks: Vec) -> Message { + msg(Role::User, blocks, 1) + } + + fn assistant(blocks: Vec) -> Message { + msg(Role::Assistant, blocks, 2) + } + + fn run(src: &Transcript, rt: &Transcript) -> Report { + compare(HarnessId::Simple, src, rt, Options::default()) + } + + /// The `(kind, category, path, count)` of every item. + fn shape(report: &Report) -> Vec<(LossKind, Category, Option<&str>, usize)> { + report + .items + .iter() + .map(|i| (i.kind, i.category, i.path.as_deref(), i.count)) + .collect() + } + + /// Assert that a comparison found nothing. + fn assert_clean(report: &Report) { + assert_eq!(report.items.len(), 0, "{:?}", report.items); + } + + fn base() -> Transcript { + convo(vec![ + user(vec![text("fix it")]), + assistant(vec![ + thinking("hmm", Some("sig")), + text("on it"), + call( + "c1", + "Bash", + json!({"command": "ls", "description": "list"}), + ), + ]), + user(vec![result("c1", ToolOutput::Text("a\nb".into()))]), + ]) + } + + #[test] + fn identical_transcripts_have_no_items() { + let src = base(); + let report = run(&src, &src.clone()); + assert!(report.items.is_empty(), "{report:?}"); + assert_eq!(report.severity, Severity::None); + assert_eq!(report.warnings.len(), 0); + assert_eq!(report.messages_before, report.messages_after); + } + + #[test] + fn lost_text_is_content() { + let src = convo(vec![user(vec![text("one"), text("two")])]); + let rt = convo(vec![user(vec![text("one")])]); + let report = run(&src, &rt); + assert_eq!(shape(&report), [(LossKind::Lost, Category::Text, None, 1)]); + assert_eq!(report.items[0].example.as_deref(), Some("two")); + assert_eq!(report.items[0].first_at, Some(0)); + assert_eq!(report.severity, Severity::Content); + } + + #[test] + fn synthesized_text_is_listed_but_does_not_raise_severity() { + let src = convo(vec![user(vec![text("one")])]); + let rt = convo(vec![user(vec![text("one"), text("extra")])]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [(LossKind::Synthesized, Category::Text, None, 1)] + ); + assert_eq!(report.items[0].first_at, None); + assert_eq!(report.severity, Severity::None); + } + + #[test] + fn edited_text_is_changed_once_not_lost_and_synthesized() { + let src = convo(vec![user(vec![text("please fix the parser bug today")])]); + let rt = convo(vec![user(vec![text("please fix the parser bug")])]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [(LossKind::Changed, Category::Text, None, 1)] + ); + assert_eq!( + report.items[0].example.as_deref(), + Some("please fix the parser bug today -> please fix the parser bug") + ); + } + + #[test] + fn unrelated_replacement_text_is_lost_plus_synthesized() { + let src = convo(vec![user(vec![text("alpha beta gamma")])]); + let rt = convo(vec![user(vec![text("omega psi chi")])]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [ + (LossKind::Lost, Category::Text, None, 1), + (LossKind::Synthesized, Category::Text, None, 1) + ] + ); + } + + #[test] + fn surrounding_whitespace_is_not_a_change() { + let src = convo(vec![user(vec![text(" hello\n")])]); + let rt = convo(vec![user(vec![text("hello")])]); + assert_clean(&run(&src, &rt)); + } + + #[test] + fn empty_text_blocks_are_ignored() { + let src = convo(vec![user(vec![text("hello")])]); + let rt = convo(vec![user(vec![text("hello"), text(" ")])]); + assert_clean(&run(&src, &rt)); + } + + #[test] + fn a_text_changing_role_is_a_changed_text() { + let src = convo(vec![user(vec![text("same words")])]); + let rt = convo(vec![msg(Role::Assistant, vec![text("same words")], 1)]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [(LossKind::Changed, Category::Text, Some("role"), 1)] + ); + assert_eq!( + report.items[0].example.as_deref(), + Some("user -> assistant") + ); + } + + #[test] + fn thinking_lost_and_changed() { + let src = convo(vec![assistant(vec![thinking( + "deep thoughts about it", + None, + )])]); + let lost = run(&src, &convo(vec![assistant(vec![])])); + assert_eq!( + shape(&lost), + [(LossKind::Lost, Category::Thinking, None, 1)] + ); + let changed = run( + &src, + &convo(vec![assistant(vec![thinking("deep thoughts about", None)])]), + ); + assert_eq!( + shape(&changed), + [(LossKind::Changed, Category::Thinking, None, 1)] + ); + } + + #[test] + fn thinking_signature_lost_changed_synthesized() { + let signed = convo(vec![assistant(vec![thinking("hmm", Some("a"))])]); + let bare = convo(vec![assistant(vec![thinking("hmm", None)])]); + let other = convo(vec![assistant(vec![thinking("hmm", Some("b"))])]); + assert_eq!( + shape(&run(&signed, &bare)), + [(LossKind::Lost, Category::ThinkingSignature, None, 1)] + ); + assert_eq!( + shape(&run(&signed, &other)), + [(LossKind::Changed, Category::ThinkingSignature, None, 1)] + ); + assert_eq!( + shape(&run(&bare, &signed)), + [(LossKind::Synthesized, Category::ThinkingSignature, None, 1)] + ); + assert_eq!(run(&signed, &bare).severity, Severity::Accounting); + } + + #[test] + fn a_dropped_tool_call_is_lost_with_its_label() { + let src = base(); + let mut rt = base(); + rt.body[1].content.pop(); + let report = run(&src, &rt); + assert!( + report.items.iter().any(|i| i.kind == LossKind::Lost + && i.category == Category::ToolCall + && i.example.as_deref() == Some("Bash: ls")), + "{report:?}" + ); + } + + #[test] + fn a_renamed_tool_is_changed_by_name_only() { + let src = convo(vec![user(vec![call( + "c1", + "/release", + json!({"args": "patch"}), + )])]); + let rt = convo(vec![user(vec![call( + "c1", + "_release", + json!({"args": "patch"}), + )])]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [( + LossKind::Changed, + Category::ToolName, + Some("/release -> _release"), + 1 + )] + ); + } + + #[test] + fn dropped_rewritten_and_added_tool_arguments() { + let src = convo(vec![assistant(vec![call( + "c1", + "Bash", + json!({"command": "ls", "description": "list", "timeout_ms": 5}), + )])]); + let rt = convo(vec![assistant(vec![call( + "c1", + "Bash", + json!({"command": "ls", "timeout_ms": 9, "workdir": "/x"}), + )])]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [ + ( + LossKind::Lost, + Category::ToolArgs, + Some("Bash.description"), + 1 + ), + ( + LossKind::Changed, + Category::ToolArgs, + Some("Bash.timeout_ms"), + 1 + ), + ( + LossKind::Synthesized, + Category::ToolArgs, + Some("Bash.workdir"), + 1 + ), + ] + ); + assert_eq!(report.items[0].example.as_deref(), Some("list")); + assert_eq!(report.items[1].example.as_deref(), Some("5 -> 9")); + } + + #[test] + fn a_call_with_nothing_in_common_is_lost_not_changed() { + let src = convo(vec![assistant(vec![call( + "c1", + "Bash", + json!({"command": "ls"}), + )])]); + let rt = convo(vec![assistant(vec![call( + "c1", + "Bash", + json!({"command": "pwd"}), + )])]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [ + (LossKind::Lost, Category::ToolCall, None, 1), + (LossKind::Synthesized, Category::ToolCall, None, 1) + ] + ); + } + + #[test] + fn json_formatting_is_not_loss() { + // Key order and whitespace in a result held as text, or in a + // structured result, never count. + let src = convo(vec![ + assistant(vec![call( + "c1", + "mcp__x__y", + json!({"b": 1, "a": {"y": [1, 2], "x": null}}), + )]), + user(vec![result( + "c1", + ToolOutput::Text("{\"a\": 1,\n \"b\": [1,\n 2]}".into()), + )]), + ]); + let rt = convo(vec![ + assistant(vec![call( + "c9", + "mcp__x__y", + json!({"a": {"x": null, "y": [1, 2]}, "b": 1}), + )]), + user(vec![result( + "c9", + ToolOutput::Json(json!({"b": [1, 2], "a": 1})), + )]), + ]); + let report = run(&src, &rt); + assert!(report.items.is_empty(), "{report:?}"); + } + + #[test] + fn a_list_of_text_parts_equals_its_text() { + let src = convo(vec![user(vec![result( + "c1", + ToolOutput::Json(json!([{"type": "text", "text": "hello"}])), + )])]); + let rt = convo(vec![user(vec![result( + "c1", + ToolOutput::Text("hello".into()), + )])]); + assert_clean(&run(&src, &rt)); + } + + #[test] + fn a_changed_result_hides_its_body_unless_detail() { + let src = convo(vec![user(vec![result( + "c1", + ToolOutput::Text("secret output line one and more".into()), + )])]); + let rt = convo(vec![user(vec![result( + "c1", + ToolOutput::Text("secret output line one".into()), + )])]); + let plain = run(&src, &rt); + assert_eq!( + shape(&plain), + [(LossKind::Changed, Category::ToolResult, None, 1)] + ); + assert_eq!(plain.items[0].example, None); + let detailed = compare(HarnessId::Simple, &src, &rt, Options { detail: true }); + assert!( + detailed.items[0] + .example + .as_deref() + .is_some_and(|e| e.contains("secret")) + ); + } + + #[test] + fn a_lost_result_never_shows_its_body() { + let src = convo(vec![user(vec![result( + "c1", + ToolOutput::Text("password=hunter2".into()), + )])]); + let report = run(&src, &convo(vec![user(vec![])])); + assert_eq!( + shape(&report), + [(LossKind::Lost, Category::ToolResult, None, 1)] + ); + assert_eq!(report.items[0].example, None); + } + + #[test] + fn error_flag_lost_and_synthesized() { + let flagged = |is_error| { + convo(vec![user(vec![Block::ToolResult { + tool_use_id: "c1".into(), + content: ToolOutput::Text("boom".into()), + is_error, + }])]) + }; + assert_eq!( + shape(&run(&flagged(true), &flagged(false))), + [(LossKind::Lost, Category::IsError, None, 1)] + ); + assert_eq!( + shape(&run(&flagged(false), &flagged(true))), + [(LossKind::Synthesized, Category::IsError, None, 1)] + ); + } + + #[test] + fn images_and_artifacts() { + let image = |data: &str| Block::Image { + source: ImageSource { + source_type: "base64".into(), + media_type: "image/png".into(), + data: data.into(), + }, + }; + let art = |text: &str| Block::Artifact { + artifact: Artifact { + id: "a".into(), + name: "report.md".into(), + source: ArtifactSource::Text { + text: text.into(), + media_type: None, + }, + }, + }; + let src = convo(vec![user(vec![image("AAAA"), art("# one")])]); + let gone = run(&src, &convo(vec![user(vec![])])); + assert_eq!( + shape(&gone), + [ + (LossKind::Lost, Category::Image, None, 1), + (LossKind::Lost, Category::Artifact, None, 1) + ] + ); + assert_eq!(gone.items[0].example.as_deref(), Some("image/png")); + assert_eq!(gone.items[1].example.as_deref(), Some("report.md")); + // A re-encoded image is a change; an artifact is its name, since its + // content lives in a file of the store's making. + let edited = run(&src, &convo(vec![user(vec![image("BBBB"), art("# two")])])); + assert_eq!( + shape(&edited), + [(LossKind::Changed, Category::Image, None, 1)] + ); + } + + #[test] + fn an_artifact_that_comes_back_as_text_is_lost() { + // Harnesses without a file carrier keep only a mention in the text; + // claude_code keeps the artifact itself. + let src = convo(vec![assistant(vec![Block::Artifact { + artifact: Artifact { + id: "a".into(), + name: "report.md".into(), + source: ArtifactSource::Text { + text: "# one".into(), + media_type: None, + }, + }, + }])]); + let kept = assess(HarnessId::ClaudeCode, &src).unwrap(); + assert!( + kept.items.iter().all(|i| i.category != Category::Artifact), + "{:?}", + kept.items + ); + let mention = assess(HarnessId::Cursor, &src).unwrap(); + assert!( + mention + .items + .iter() + .any(|i| i.kind == LossKind::Lost && i.category == Category::Artifact), + "{:?}", + mention.items + ); + } + + #[test] + fn split_and_merged_messages_are_not_loss() { + let blocks = vec![ + text("first"), + call("c1", "Bash", json!({"command": "ls"})), + text("last"), + ]; + let merged = convo(vec![assistant(blocks.clone())]); + let split = convo(blocks.into_iter().map(|b| assistant(vec![b])).collect()); + assert_clean(&run(&merged, &split)); + let back = run(&split, &merged); + assert_clean(&back); + assert_eq!(back.messages_before, 3); + assert_eq!(back.messages_after, 1); + } + + #[test] + fn reordered_blocks_are_a_content_change() { + let src = convo(vec![assistant(vec![ + text("one"), + text("two"), + text("three"), + ])]); + let rt = convo(vec![assistant(vec![ + text("three"), + text("one"), + text("two"), + ])]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [(LossKind::Changed, Category::Order, None, 1)] + ); + assert_eq!(report.severity, Severity::Content); + } + + fn with_usage(n: u64, cache: Option) -> Transcript { + let mut t = convo(vec![assistant(vec![text("hi")])]); + t.body[0].usage = Some(Usage { + input_tokens: n, + output_tokens: 5, + cache_read_input_tokens: cache, + cache_creation_input_tokens: None, + }); + t + } + + #[test] + fn usage_lost_changed_field_by_field_and_synthesized() { + let plain = convo(vec![assistant(vec![text("hi")])]); + assert_eq!( + shape(&run(&with_usage(10, None), &plain)), + [(LossKind::Lost, Category::Usage, None, 1)] + ); + assert_eq!( + shape(&run(&plain, &with_usage(10, None))), + [(LossKind::Synthesized, Category::Usage, None, 1)] + ); + assert_eq!( + shape(&run(&with_usage(10, Some(7)), &with_usage(10, None))), + [( + LossKind::Lost, + Category::Usage, + Some("cache_read_input_tokens"), + 1 + )] + ); + assert_eq!( + shape(&run(&with_usage(10, None), &with_usage(12, None))), + [(LossKind::Changed, Category::Usage, Some("input_tokens"), 1)] + ); + } + + #[test] + fn an_absent_cache_count_equals_zero() { + assert_clean(&run(&with_usage(10, None), &with_usage(10, Some(0)))); + assert_clean(&run(&with_usage(10, Some(0)), &with_usage(10, None))); + } + + fn with_stop(reason: Option) -> Transcript { + let mut t = convo(vec![assistant(vec![text("hi")])]); + t.body[0].stop_reason = reason; + t + } + + #[test] + fn stop_reason_lost_changed_synthesized() { + let ended = with_stop(Some(StopReason::EndTurn)); + let none = with_stop(None); + assert_eq!( + shape(&run(&ended, &none)), + [(LossKind::Lost, Category::StopReason, None, 1)] + ); + assert_eq!( + shape(&run(&none, &ended)), + [(LossKind::Synthesized, Category::StopReason, None, 1)] + ); + // Same slot, other representation: one change, not a loss plus a + // synthesis. + let aborted = with_stop(Some(StopReason::Aborted)); + let other = with_stop(Some(StopReason::Other("aborted".into()))); + let report = run(&aborted, &other); + assert_eq!( + shape(&report), + [(LossKind::Changed, Category::StopReason, None, 1)] + ); + assert_eq!( + report.items[0].example.as_deref(), + Some("aborted -> other(aborted)") + ); + assert_eq!(report.severity, Severity::Accounting); + } + + #[test] + fn unequal_stop_reason_leftovers_are_not_paired() { + let mut src = convo(vec![assistant(vec![text("a")]), assistant(vec![text("b")])]); + src.body[0].stop_reason = Some(StopReason::EndTurn); + src.body[1].stop_reason = Some(StopReason::Aborted); + let mut rt = src.clone(); + rt.body[0].stop_reason = Some(StopReason::ToolUse); + rt.body[1].stop_reason = None; + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [ + (LossKind::Lost, Category::StopReason, None, 2), + (LossKind::Synthesized, Category::StopReason, None, 1) + ] + ); + } + + #[test] + fn message_model_lost_changed_synthesized() { + let model = |m: Option<&str>| { + let mut t = convo(vec![assistant(vec![text("hi")])]); + t.body[0].model = m.map(Into::into); + t + }; + assert_eq!( + shape(&run(&model(Some("m1")), &model(None))), + [(LossKind::Lost, Category::MessageModel, None, 1)] + ); + assert_eq!( + shape(&run(&model(Some("m1")), &model(Some("m2")))), + [(LossKind::Changed, Category::MessageModel, None, 1)] + ); + assert_eq!( + shape(&run(&model(None), &model(Some("m1")))), + [(LossKind::Synthesized, Category::MessageModel, None, 1)] + ); + } + + #[test] + fn message_times_are_compared_by_second() { + let src = convo(vec![ + msg(Role::User, vec![text("a")], 1), + msg(Role::User, vec![text("b")], 2), + ]); + let mut rt = src.clone(); + rt.body[1].timestamp = at(50); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [(LossKind::Lost, Category::Timestamp, Some("message"), 1)] + ); + assert_eq!(report.items[0].first_at, Some(1)); + assert_eq!(report.severity, Severity::Metadata); + // Sub-second precision is not a loss. + let mut fine = src.clone(); + fine.body[0].timestamp += chrono::Duration::milliseconds(400); + assert_clean(&run(&src, &fine)); + } + + #[test] + fn session_metadata_lost_changed_synthesized() { + let mut src = base(); + src.meta.title = Some("A title".into()); + src.meta.model = Some("m1".into()); + src.meta.git_branch = Some("main".into()); + src.meta.cwd = Some("/work".into()); + src.meta.lineage = Some(Lineage { + parent: "p1".into(), + relation: Relation::Fork, + }); + let mut rt = base(); + rt.meta.title = Some("Another".into()); + rt.meta.model = None; + rt.meta.cwd = Some("/work".into()); + rt.meta.git_branch = None; + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [ + (LossKind::Changed, Category::Title, None, 1), + (LossKind::Lost, Category::Model, None, 1), + (LossKind::Lost, Category::GitBranch, None, 1), + (LossKind::Lost, Category::Lineage, None, 1), + ] + ); + assert_eq!(report.items[3].example.as_deref(), Some("fork of p1")); + assert_eq!(report.severity, Severity::Metadata); + + let mut gained = base(); + gained.meta.cwd = Some("/new".into()); + assert_eq!( + shape(&run(&base(), &gained)), + [(LossKind::Synthesized, Category::Cwd, None, 1)] + ); + // Titles compare trimmed. + let mut padded = base(); + padded.meta.title = Some("T ".into()); + let mut bare = base(); + bare.meta.title = Some("T".into()); + assert_clean(&run(&padded, &bare)); + } + + #[test] + fn the_session_start_is_not_reported() { + let src = base(); + let mut rt = base(); + rt.meta.timestamp = at(3600); + assert_clean(&run(&src, &rt)); + } + + #[test] + fn an_empty_session_warns() { + let src = convo(vec![]); + let report = run(&src, &src.clone()); + assert_clean(&report); + assert_eq!(report.severity, Severity::None); + assert_eq!(report.warnings.len(), 1); + assert!(report.warnings[0].contains("0 messages")); + } + + #[test] + fn severity_is_ordered_and_takes_the_worst_item() { + assert!(Severity::None < Severity::Metadata); + assert!(Severity::Metadata < Severity::Accounting); + assert!(Severity::Accounting < Severity::Content); + + let mut src = base(); + src.meta.title = Some("t".into()); + src.body[1].stop_reason = Some(StopReason::EndTurn); + let mut rt = src.clone(); + rt.meta.title = None; + assert_eq!(run(&src, &rt).severity, Severity::Metadata); + rt.body[1].stop_reason = None; + assert_eq!(run(&src, &rt).severity, Severity::Accounting); + rt.body[0].content.clear(); + assert_eq!(run(&src, &rt).severity, Severity::Content); + } + + #[test] + fn categories_map_to_their_severity_class() { + assert_eq!(Category::Text.severity(), Severity::Content); + assert_eq!(Category::IsError.severity(), Severity::Content); + assert_eq!(Category::Order.severity(), Severity::Content); + assert_eq!(Category::Usage.severity(), Severity::Accounting); + assert_eq!(Category::ThinkingSignature.severity(), Severity::Accounting); + assert_eq!(Category::Lineage.severity(), Severity::Metadata); + assert_eq!(Category::Timestamp.severity(), Severity::Metadata); + } + + #[test] + fn examples_are_capped_and_single_line() { + let long = format!("start\n\u{1b}[31m{}", "x".repeat(500)); + let src = convo(vec![user(vec![text(&long)])]); + let report = run(&src, &convo(vec![user(vec![])])); + let example = report.items[0].example.as_deref().unwrap(); + assert_eq!(example.chars().count(), EXAMPLE_CAP); + assert!(example.ends_with('…')); + assert!(!example.chars().any(char::is_control), "{example:?}"); + let detailed = compare( + HarnessId::Simple, + &src, + &convo(vec![user(vec![])]), + Options { detail: true }, + ); + let longer = detailed.items[0].example.as_deref().unwrap(); + assert_eq!(longer.chars().count(), DETAIL_CAP); + } + + #[test] + fn clip_handles_edges() { + assert_eq!(clip("abc", 80), "abc"); + assert_eq!(clip(" abc ", 80), "abc"); + assert_eq!(clip("abcd", 4), "abcd"); + assert_eq!(clip("abcde", 4), "abc…"); + assert_eq!(clip("", 4), ""); + assert_eq!(clip("日本語のコメントも直して", 5), "日本語の…"); + } + + #[test] + fn comparison_is_deterministic() { + let src = base(); + let mut rt = base(); + rt.body[1].content.truncate(1); + rt.meta.title = Some("x".into()); + let first = run(&src, &rt); + for _ in 0..20 { + assert_eq!(run(&src, &rt), first); + } + } + + #[test] + fn duplicate_blocks_match_one_to_one() { + let src = convo(vec![user(vec![text("ok"), text("ok"), text("ok")])]); + let rt = convo(vec![user(vec![text("ok")])]); + let report = run(&src, &rt); + assert_eq!(shape(&report), [(LossKind::Lost, Category::Text, None, 2)]); + } + + #[test] + fn assess_round_trips_through_the_real_codec() { + let src = base(); + let report = assess(HarnessId::Simple, &src).unwrap(); + assert!(report.items.is_empty(), "{report:?}"); + assert_eq!(report.target, HarnessId::Simple); + let serialized = serde_json::to_value(&report).unwrap(); + assert_eq!(serialized["target"], "simple"); + assert_eq!(serialized["severity"], "none"); + assert_eq!(serialized["items"], json!([])); + } + + #[test] + fn check_all_covers_every_target_and_survives_an_empty_session() { + let results = check_all(&convo(vec![])); + assert_eq!(results.len(), TARGETS.len()); + for (target, report) in results { + let report = report.unwrap_or_else(|e| panic!("{target}: {e}")); + assert_eq!(report.messages_before, 0); + assert_eq!(report.warnings.len(), 1); + } + } + + #[test] + fn read_only_sources_are_not_targets() { + for harness in [ + HarnessId::ClaudeChat, + HarnessId::CoworkRemote, + HarnessId::ChatGpt, + ] { + assert!(!TARGETS.contains(&harness)); + } + for harness in HarnessId::ALL { + let readonly = matches!( + harness, + HarnessId::ClaudeChat | HarnessId::CoworkRemote | HarnessId::ChatGpt + ); + assert_eq!(TARGETS.contains(&harness), !readonly, "{harness}"); + } + } + + #[test] + fn re_spaced_json_inside_text_parts_is_not_a_change() { + // Found by hand: a target re-serialized the JSON a result carried as + // the text of a text part, compactly; that is formatting, not loss. + let parts = |text: &str| ToolOutput::Json(json!([{"type": "text", "text": text}])); + let src = convo(vec![user(vec![result( + "c1", + parts("{\"a\": 1,\n \"b\": [1, 2]}"), + )])]); + let rt = convo(vec![user(vec![result( + "c1", + parts("{\"a\":1,\"b\":[1,2]}"), + )])]); + assert_clean(&run(&src, &rt)); + // A real difference inside it still counts. + let other = convo(vec![user(vec![result( + "c1", + parts("{\"a\":2,\"b\":[1,2]}"), + )])]); + assert_eq!( + shape(&run(&src, &other)), + [ + (LossKind::Lost, Category::ToolResult, None, 1), + (LossKind::Synthesized, Category::ToolResult, None, 1) + ] + ); + } + + #[test] + fn nested_stringified_json_is_bounded() { + let mut text = "{\"k\": 1}".to_string(); + // Each layer doubles the escaping, so a dozen is already deep. + for _ in 0..12 { + text = serde_json::to_string(&json!([text])).unwrap(); + } + let src = convo(vec![user(vec![result("c1", ToolOutput::Text(text))])]); + assert_clean(&run(&src, &src.clone())); + } + + #[test] + fn empty_models_and_titles_are_no_value() { + // Found by hand: pi writes an empty model on every turn it has none + // for, which is not a per-message model. + let mut src = convo(vec![assistant(vec![text("hi")])]); + let mut rt = src.clone(); + rt.body[0].model = Some(String::new()); + rt.meta.title = Some(String::new()); + assert_clean(&run(&src, &rt)); + src.body[0].model = Some(" ".into()); + assert_clean(&run(&src, &rt)); + } + + #[test] + fn a_cache_count_dropped_from_fewer_records_is_both_a_field_and_a_record_loss() { + // Found by hand: cursor_desktop drops cache counts and merges turns. + let record = |input: u64, cache: Option| Usage { + input_tokens: input, + output_tokens: 5, + cache_read_input_tokens: cache, + cache_creation_input_tokens: None, + }; + let mut src = convo(vec![assistant(vec![text("a")]); 3]); + for (n, m) in src.body.iter_mut().enumerate() { + m.usage = Some(record(10 + n as u64, Some(7))); + } + let mut rt = convo(vec![assistant(vec![text("a")]); 2]); + rt.body[0].usage = Some(record(10, None)); + rt.body[1].usage = Some(record(11, None)); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [ + (LossKind::Lost, Category::Text, None, 1), + (LossKind::Lost, Category::Usage, None, 1), + ( + LossKind::Lost, + Category::Usage, + Some("cache_read_input_tokens"), + 2 + ), + ] + ); + } + + #[test] + fn arrow_shows_whitespace_and_cuts_around_the_difference() { + assert_eq!(arrow("# notes\n", "# notes", 80), "# notes\\n -> # notes"); + let head = "x".repeat(300); + let a = format!("{head} alpha {}", "y".repeat(300)); + let b = format!("{head} beta {}", "y".repeat(300)); + let shown = arrow(&a, &b, 80); + assert!(shown.chars().count() <= 80, "{shown}"); + assert!(shown.contains("alpha") && shown.contains("beta"), "{shown}"); + assert!(shown.starts_with('…'), "{shown}"); + } + + #[test] + fn many_look_alike_leftovers_are_bounded_and_counted() { + // Results that open alike but are different never pair; each is + // compared against a bounded number of candidates. + let results = |tail: &str| -> Vec { + (0..2000) + .map(|n| { + user(vec![result( + "c", + ToolOutput::Text(format!("output{tail}{n}")), + )]) + }) + .collect() + }; + let report = run(&convo(results("-a-")), &convo(results("-b-"))); + assert_eq!( + shape(&report), + [ + (LossKind::Lost, Category::ToolResult, None, 2000), + (LossKind::Synthesized, Category::ToolResult, None, 2000), + ] + ); + } + + #[test] + fn loss_serializes_without_empty_fields() { + let src = convo(vec![user(vec![result("c1", ToolOutput::Text("x".into()))])]); + let report = run(&src, &convo(vec![user(vec![])])); + let value = serde_json::to_value(&report.items[0]).unwrap(); + assert_eq!( + value, + json!({"kind": "lost", "category": "tool_result", "count": 1, "first_at": 0}) + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index 3dd58da..8fea8c5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,10 +19,13 @@ //! - [`harness`] — one module per implemented harness. //! - [`dispatch`] — parse or render a harness's native text, chosen by //! [`HarnessId`] at run time. +//! - [`fidelity`] — what converting into a harness loses, measured before +//! anything is written. pub mod common; pub mod dispatch; pub mod error; +pub mod fidelity; pub mod harness; #[cfg(not(target_arch = "wasm32"))] pub mod local; diff --git a/tests/README.md b/tests/README.md index cb06c99..73c03b5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -35,8 +35,11 @@ three contracts every harness must honor: extracted correctly. Plus cross-cutting modules: `cross_harness` (a conversation survives a hop -through every harness), `path_safety` (adversarial ids, symlinks, and -references cannot escape a store root), `store_delete`, `search`, and +through every harness), `fidelity` (what converting into each harness loses, +pinned in one golden table that also feeds `docs/formats/README.md`; update +both deliberately when a codec change moves it), `path_safety` (adversarial +ids, symlinks, and references cannot escape a store root), `store_delete`, +`search`, and `properties` — proptest sweeps asserting the codec fixpoint over generated conversations for every harness that writes (all but the pull-only Claude Chat and ChatGPT) at once. The generator in `properties` diff --git a/tests/integration/fidelity.rs b/tests/integration/fidelity.rs new file mode 100644 index 0000000..161d658 --- /dev/null +++ b/tests/integration/fidelity.rs @@ -0,0 +1,741 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] + +//! Conversion fidelity (`txcript::fidelity`) measured against every harness +//! codec: a rich canonical session assessed per target with a golden result, +//! determinism, and no-panic sweeps over generated transcripts. +//! +//! The golden table below is the oracle for codec fidelity. A codec change +//! that loses less (or more) shows up as a line to add, change, or delete +//! here, which is the point: update it deliberately, in the same commit as the +//! codec change, and update the table in `docs/formats/README.md` that +//! `docs_table_matches_the_measurement` checks. Only losses and changes are +//! listed (synthesized items and per-message timestamps are not stable enough +//! to pin) and each line is one harness, kind, category and count on the +//! `rich` session. + +use chrono::{DateTime, Utc}; +use proptest::prelude::*; +use serde_json::json; +use txcript::common::{ + Block, ImageSource, Message, Meta, Role, StopReason, Tool, ToolOutput, Usage, +}; +use txcript::fidelity::{self, Category, LossKind, Severity}; +use txcript::{Common, HarnessId, Transcript}; + +fn ts(secs: i64) -> DateTime { + DateTime::from_timestamp(1_790_000_000 + secs, 0).unwrap() +} + +// One fixture with every kind of block; length grows with the kinds covered. +#[allow(clippy::too_many_lines)] +fn rich() -> Transcript { + let meta = Meta { + id: "fid-1".into(), + timestamp: ts(0), + cwd: Some("/work/repo".into()), + git_branch: Some("feat/clock".into()), + title: Some("Fix the flaky clock test".into()), + cli_version: Some("1.2.3".into()), + model: Some("claude-opus-4-8".into()), + lineage: None, + }; + let model = || Some("claude-opus-4-8".to_string()); + let usage = |n: u64| { + Some(Usage { + input_tokens: 1000 + n, + output_tokens: 100 + n, + cache_read_input_tokens: Some(900), + cache_creation_input_tokens: None, + }) + }; + let user = |blocks: Vec, t: i64| Message { + role: Role::User, + content: blocks, + timestamp: ts(t), + model: None, + stop_reason: None, + usage: None, + }; + let assistant = |blocks: Vec, t: i64, stop: StopReason| Message { + role: Role::Assistant, + content: blocks, + timestamp: ts(t), + model: model(), + stop_reason: Some(stop), + usage: usage(u64::try_from(t).unwrap()), + }; + let text = |s: &str| Block::Text { text: s.into() }; + let call = |id: &str, tool: Tool| Block::ToolUse { + id: id.into(), + tool, + }; + let result = |id: &str, out: ToolOutput, is_error: bool| Block::ToolResult { + tool_use_id: id.into(), + content: out, + is_error, + }; + let body = vec![ + user( + vec![ + text("Fix the flaky test in clock.rs. 日本語のコメントも直して 🚀"), + Block::Image { + source: ImageSource { + source_type: "base64".into(), + media_type: "image/png".into(), + data: "iVBORw0KGgo=".into(), + }, + }, + ], + 1, + ), + assistant( + vec![ + Block::Thinking { + text: "The test reads the wall clock.".into(), + signature: Some("sig-aaa".into()), + encrypted: None, + }, + text("Let me run it first."), + call( + "c1", + Tool::Bash { + command: "cargo test clock".into(), + workdir: None, + timeout_ms: None, + description: Some("Run the clock tests".into()), + run_in_background: false, + }, + ), + ], + 2, + StopReason::ToolUse, + ), + user( + vec![result( + "c1", + ToolOutput::Text("running 3 tests\n3 passed".into()), + false, + )], + 3, + ), + assistant( + vec![ + text("Pinning the clock."), + call( + "c2", + Tool::Edit { + file_path: "/work/repo/src/clock.rs".into(), + old_string: "Utc::now()".into(), + new_string: "self.now".into(), + replace_all: false, + }, + ), + ], + 4, + StopReason::ToolUse, + ), + user( + vec![result("c2", ToolOutput::Text("patched".into()), false)], + 5, + ), + assistant( + vec![call( + "c3", + Tool::Raw { + tool_name: "mcp__docs__lookup".into(), + input: json!({"q": "clock", "filters": {"b": 1, "a": 2}}), + }, + )], + 6, + StopReason::ToolUse, + ), + user( + vec![result( + "c3", + ToolOutput::Json(json!([{"title": "Clock"}, {"title": "Time"}])), + false, + )], + 7, + ), + assistant( + vec![call( + "c4", + Tool::Bash { + command: "false".into(), + workdir: None, + timeout_ms: None, + description: None, + run_in_background: false, + }, + )], + 8, + StopReason::ToolUse, + ), + user(vec![result("c4", ToolOutput::Text("boom".into()), true)], 9), + user( + vec![call( + "c5", + Tool::Command { + command: "/release".into(), + args: Some("patch".into()), + }, + )], + 10, + ), + user( + vec![result( + "c5", + ToolOutput::Text("released v1.2.3".into()), + false, + )], + 11, + ), + assistant( + vec![ + Block::Thinking { + text: "Everything passes now.".into(), + signature: None, + encrypted: None, + }, + text("All green."), + ], + 12, + StopReason::EndTurn, + ), + user(vec![text("thanks")], 13), + assistant(vec![text("Anytime.")], 14, StopReason::Aborted), + ]; + Transcript::new(meta, body) +} + +/// The targets `continue --with` writes, minus the pull-only and mint-only +/// ones: the default targets of a fidelity report. +const WRITABLE: [HarnessId; 11] = [ + HarnessId::ClaudeCode, + HarnessId::Codex, + HarnessId::OpenCode, + HarnessId::Pi, + HarnessId::Campfire, + HarnessId::Cursor, + HarnessId::CursorDesktop, + HarnessId::Grok, + HarnessId::Fx, + HarnessId::Antigravity, + HarnessId::Cowork, +]; + +/// One line per harness and severity, then one per loss, sorted. +const GOLDEN: &[&str] = &[ + "amp lost tool_call x1", + "amp severity content", + "antigravity changed title x1", + "antigravity lost message_model x6", + "antigravity lost model x1", + "antigravity lost tool_args Bash.description x1", + "antigravity lost tool_call x1", + "antigravity lost tool_result x1", + "antigravity severity content", + "campfire lost git_branch x1", + "campfire lost model x1", + "campfire lost thinking_signature x1", + "campfire lost title x1", + "campfire lost tool_call x1", + "campfire severity content", + "claude_code changed stop_reason x1", + "claude_code severity accounting", + "codex changed tool_name /release -> _release x1", + "codex lost is_error x1", + "codex lost model x1", + "codex lost stop_reason x6", + "codex lost thinking_signature x1", + "codex lost title x1", + "codex lost tool_args Bash.description x1", + "codex lost usage x2", + "codex severity content", + "cowork changed stop_reason x1", + "cowork severity accounting", + "cursor lost git_branch x1", + "cursor lost stop_reason x6", + "cursor lost tool_call x1", + "cursor lost usage x6", + "cursor severity content", + "cursor_desktop lost git_branch x1", + "cursor_desktop lost image x1", + "cursor_desktop lost stop_reason x6", + "cursor_desktop lost tool_args Bash.description x1", + "cursor_desktop lost tool_call x1", + "cursor_desktop lost tool_result x1", + "cursor_desktop lost usage cache_read_input_tokens x6", + "cursor_desktop severity content", + "fx lost git_branch x1", + "fx lost stop_reason x5", + "fx lost tool_args Bash.description x1", + "fx lost tool_call x1", + "fx lost usage x6", + "fx severity content", + "grok lost stop_reason x4", + "grok lost thinking_signature x1", + "grok lost tool_call x1", + "grok lost usage x6", + "grok severity content", + "grok_bot changed title x1", + "grok_bot lost cwd x1", + "grok_bot lost git_branch x1", + "grok_bot lost image x1", + "grok_bot lost message_model x6", + "grok_bot lost model x1", + "grok_bot lost stop_reason x6", + "grok_bot lost usage x6", + "grok_bot severity content", + "hermes lost thinking_signature x1", + "hermes lost tool_call x1", + "hermes lost usage x6", + "hermes severity content", + "opencode changed stop_reason x1", + "opencode lost git_branch x1", + "opencode lost thinking_signature x1", + "opencode lost tool_call x1", + "opencode lost tool_result x1", + "opencode severity content", + "pi lost git_branch x1", + "pi lost model x1", + "pi lost thinking_signature x1", + "pi lost title x1", + "pi lost tool_call x1", + "pi severity content", + "simple severity none", +]; + +/// The `rich` session assessed for `target`. +fn report(target: HarnessId) -> fidelity::Report { + fidelity::assess(target, &rich()).unwrap_or_else(|e| panic!("{target}: {e}")) +} + +fn golden_lines(target: HarnessId, report: &fidelity::Report) -> Vec { + let mut lines = vec![format!("{target} severity {}", report.severity.as_str())]; + for item in &report.items { + if item.kind == LossKind::Synthesized || item.category == Category::Timestamp { + continue; + } + let kind = match item.kind { + LossKind::Lost => "lost", + _ => "changed", + }; + let category = serde_json::to_value(item.category).unwrap(); + let path = item + .path + .as_deref() + .map(|p| format!(" {p}")) + .unwrap_or_default(); + lines.push(format!( + "{target} {kind} {}{path} x{}", + category.as_str().unwrap(), + item.count + )); + } + lines +} + +#[test] +fn every_target_matches_the_golden_measurement() { + let mut actual: Vec = fidelity::TARGETS + .into_iter() + .flat_map(|target| golden_lines(target, &report(target))) + .collect(); + actual.sort(); + let expected: Vec = GOLDEN.iter().map(ToString::to_string).collect(); + if actual != expected { + let table = actual + .iter() + .map(|l| format!(" \"{l}\",")) + .collect::>() + .join("\n"); + panic!( + "the measured fidelity differs from GOLDEN. If a codec change caused this on \ + purpose, replace the table with:\n{table}" + ); + } +} + +#[test] +fn claude_code_and_simple_lose_no_content() { + for target in [HarnessId::ClaudeCode, HarnessId::Simple, HarnessId::Cowork] { + let r = report(target); + assert!( + r.severity < Severity::Content, + "{target} lost content: {:?}", + r.items + ); + } + let simple = report(HarnessId::Simple); + assert!(simple.items.is_empty(), "{:?}", simple.items); + assert_eq!(simple.messages_before, simple.messages_after); +} + +#[test] +fn measurement_is_deterministic() { + let src = rich(); + for target in fidelity::TARGETS { + let first = fidelity::assess(target, &src).unwrap(); + for _ in 0..3 { + assert_eq!(fidelity::assess(target, &src).unwrap(), first, "{target}"); + } + } +} + +#[test] +fn check_all_reports_every_target_and_never_aborts() { + let results = fidelity::check_all(&rich()); + let targets: Vec = results.iter().map(|(t, _)| *t).collect(); + assert_eq!(targets, fidelity::TARGETS); + for (target, result) in results { + assert_eq!(result.unwrap().target, target); + } +} + +#[test] +fn read_only_sources_cannot_be_assessed_as_targets() { + // Not in `TARGETS`, and a direct call fails cleanly instead of panicking. + for target in [ + HarnessId::ClaudeChat, + HarnessId::CoworkRemote, + HarnessId::ChatGpt, + ] { + assert!(!fidelity::TARGETS.contains(&target)); + let _ = fidelity::assess(target, &rich()); + } +} + +fn status(report: &fidelity::Report, class: Severity) -> &'static str { + let any = |kind: LossKind| { + report.items.iter().any(|i| { + i.kind == kind && i.category.severity() == class && i.category != Category::Timestamp + }) + }; + if any(LossKind::Lost) { + "lost" + } else if any(LossKind::Changed) { + "changed" + } else { + "ok" + } +} + +/// One cell of the docs table: `lost: a, b; changed: c`, or a dash. +fn cell(report: &fidelity::Report, class: Severity) -> String { + let names = |kind: LossKind| -> Vec { + let mut names: Vec = Vec::new(); + for item in report.items.iter().filter(|i| { + i.kind == kind && i.category.severity() == class && i.category != Category::Timestamp + }) { + let name = serde_json::to_value(item.category).unwrap(); + let name = name.as_str().unwrap().to_string(); + if !names.contains(&name) { + names.push(name); + } + } + names + }; + let parts: Vec = [(LossKind::Lost, "lost"), (LossKind::Changed, "changed")] + .into_iter() + .filter_map(|(kind, label)| { + let names = names(kind); + (!names.is_empty()).then(|| format!("{label}: {}", names.join(", "))) + }) + .collect(); + if parts.is_empty() { + "ok".to_string() + } else { + parts.join("; ") + } +} + +/// The per-harness support table in `docs/formats/README.md` is this +/// measurement, rendered. When it drifts, the failure prints the rows to paste. +#[test] +fn docs_table_matches_the_measurement() { + let docs = include_str!("../../docs/formats/README.md"); + let rows: Vec = WRITABLE + .into_iter() + .map(|target| { + let r = report(target); + format!( + "| {target} | {} | {} | {} |", + cell(&r, Severity::Content), + cell(&r, Severity::Accounting), + cell(&r, Severity::Metadata) + ) + }) + .collect(); + let missing: Vec<&String> = rows + .iter() + .filter(|row| !docs.contains(row.as_str())) + .collect(); + assert!( + missing.is_empty(), + "docs/formats/README.md is out of date; its table rows should be:\n{}", + rows.join("\n") + ); + // The status words the CLI matrix would print agree with the cells. + for target in WRITABLE { + let r = report(target); + for class in [Severity::Content, Severity::Accounting, Severity::Metadata] { + let word = status(&r, class); + let c = cell(&r, class); + assert_eq!(word == "ok", c == "ok", "{target} {class:?}: {word} vs {c}"); + } + } +} + +// ── generated transcripts ────────────────────────────────────────────── + +#[derive(Debug, Clone)] +enum Step { + UserText(String), + UserImage, + Reply { + thinking: Option<(String, Option)>, + text: String, + usage: Option<(u64, u64)>, + stop: Option, + model: Option, + }, + Call { + tool: Tool, + result: ToolOutput, + is_error: bool, + }, + Command { + name: String, + args: Option, + output: String, + }, +} + +fn arb_text() -> impl Strategy { + prop_oneof![ + 3 => "[a-zA-Z0-9][a-zA-Z0-9 ,.!?<>&'/_-]{0,38}", + 1 => "[a-zA-Z0-9 日本語é🚀\n\t]{1,30}", + 1 => Just(" padded\n".to_string()), + 1 => Just(String::new()), + ] +} + +fn arb_stop() -> impl Strategy { + prop_oneof![ + Just(StopReason::EndTurn), + Just(StopReason::ToolUse), + Just(StopReason::MaxTokens), + Just(StopReason::StopSequence), + Just(StopReason::Aborted), + Just(StopReason::Error), + "[a-z_]{1,8}".prop_map(StopReason::Other), + ] +} + +fn arb_output() -> impl Strategy { + prop_oneof![ + arb_text().prop_map(ToolOutput::Text), + arb_text().prop_map(|t| ToolOutput::Json(json!([{"type": "text", "text": t}]))), + arb_text().prop_map(|t| ToolOutput::Json(json!({"k": t, "n": [1, 2, 3]}))), + ] +} + +fn arb_tool() -> impl Strategy { + prop_oneof![ + (arb_text(), proptest::option::of(arb_text())).prop_map(|(command, description)| { + Tool::Bash { + command, + workdir: None, + timeout_ms: None, + description, + run_in_background: false, + } + }), + ("/repo/[a-z]{1,8}\\.rs", arb_text(), arb_text()).prop_map( + |(file_path, old_string, new_string)| Tool::Edit { + file_path, + old_string, + new_string, + replace_all: false, + } + ), + "/repo/[a-z]{1,8}\\.rs".prop_map(|file_path| Tool::Read { + file_path, + offset: None, + limit: None, + }), + ("mcp__[a-z]{1,5}__[a-z]{1,6}", arb_text()).prop_map(|(tool_name, q)| Tool::Raw { + tool_name, + input: json!({"q": q, "opts": {"b": 1, "a": [true, null]}}), + }), + ] +} + +fn arb_step() -> impl Strategy { + prop_oneof![ + arb_text().prop_map(Step::UserText), + Just(Step::UserImage), + ( + proptest::option::of((arb_text(), proptest::option::of("[a-z0-9]{4,10}"))), + arb_text(), + proptest::option::of((0u64..5000, 0u64..500)), + proptest::option::of(arb_stop()), + proptest::option::of("[a-z0-9-]{3,12}"), + ) + .prop_map(|(thinking, text, usage, stop, model)| Step::Reply { + thinking, + text, + usage, + stop, + model, + }), + (arb_tool(), arb_output(), any::()).prop_map(|(tool, result, is_error)| { + Step::Call { + tool, + result, + is_error, + } + }), + ("/[a-z]{2,8}", proptest::option::of(arb_text()), arb_text()) + .prop_map(|(name, args, output)| Step::Command { name, args, output }), + ] +} + +fn message(role: Role, content: Vec, n: usize) -> Message { + Message { + role, + content, + timestamp: ts(i64::try_from(n).unwrap() + 1), + model: None, + stop_reason: None, + usage: None, + } +} + +fn build(steps: &[Step]) -> Transcript { + let mut t = rich(); + t.body.clear(); + for (i, step) in steps.iter().enumerate() { + let id = format!("c{i}"); + let n = t.body.len(); + match step { + Step::UserText(text) => t.body.push(message( + Role::User, + vec![Block::Text { text: text.clone() }], + n, + )), + Step::UserImage => t.body.push(message( + Role::User, + vec![Block::Image { + source: ImageSource { + source_type: "base64".into(), + media_type: "image/png".into(), + data: "iVBORw0KGgo=".into(), + }, + }], + n, + )), + Step::Reply { + thinking, + text, + usage, + stop, + model, + } => { + let mut content = Vec::new(); + if let Some((text, signature)) = thinking { + content.push(Block::Thinking { + text: text.clone(), + signature: signature.clone(), + encrypted: None, + }); + } + content.push(Block::Text { text: text.clone() }); + let mut reply = message(Role::Assistant, content, n); + reply.model.clone_from(model); + reply.stop_reason.clone_from(stop); + reply.usage = usage.map(|(input_tokens, output_tokens)| Usage { + input_tokens, + output_tokens, + cache_read_input_tokens: None, + cache_creation_input_tokens: None, + }); + t.body.push(reply); + } + Step::Call { + tool, + result, + is_error, + } => { + t.body.push(message( + Role::Assistant, + vec![Block::ToolUse { + id: id.clone(), + tool: tool.clone(), + }], + n, + )); + t.body.push(message( + Role::User, + vec![Block::ToolResult { + tool_use_id: id, + content: result.clone(), + is_error: *is_error, + }], + n + 1, + )); + } + Step::Command { name, args, output } => { + t.body.push(message( + Role::User, + vec![Block::ToolUse { + id: id.clone(), + tool: Tool::Command { + command: name.clone(), + args: args.clone(), + }, + }], + n, + )); + t.body.push(message( + Role::User, + vec![Block::ToolResult { + tool_use_id: id, + content: ToolOutput::Text(output.clone()), + is_error: false, + }], + n + 1, + )); + } + } + } + t +} + +proptest! { + /// Whatever the transcript, assessing it never panics for any target, + /// repeats exactly, and measures the lossless Simple codec as lossless: + /// a false positive from the comparator shows up here. + #[test] + fn assessing_generated_transcripts_is_total_and_stable( + steps in proptest::collection::vec(arb_step(), 0..8) + ) { + let src = build(&steps); + for target in fidelity::TARGETS { + let first = fidelity::assess(target, &src); + let second = fidelity::assess(target, &src); + match (first, second) { + (Ok(a), Ok(b)) => { + prop_assert_eq!(&a, &b); + prop_assert_eq!(a.messages_before, src.body.len()); + if target == HarnessId::Simple { + prop_assert!(a.items.is_empty(), "simple is lossless: {:?}", a.items); + } + } + (Err(a), Err(b)) => prop_assert_eq!(a.to_string(), b.to_string()), + _ => prop_assert!(false, "{target}: flaky result"), + } + } + } +} diff --git a/tests/integration/main.rs b/tests/integration/main.rs index 4eb9cce..47f0786 100644 --- a/tests/integration/main.rs +++ b/tests/integration/main.rs @@ -1,8 +1,8 @@ //! Integration tests: every module exercises public API against real backing //! stores (temp dirs, real `SQLite`) — no mocks. Per-harness modules cover //! store round-trip fidelity, codec fixpoints through Common, and discovery; -//! the rest cover cross-harness conversion, adversarial path handling, -//! search, and deletion. Property-based codec invariants live in +//! the rest cover cross-harness conversion, conversion fidelity, adversarial +//! path handling, search, and deletion. Property-based codec invariants live in //! `properties`. Tests that pin a specific shipped bug belong in //! `tests/regression/`, not here — see `tests/README.md`. @@ -16,6 +16,7 @@ mod cowork; mod cross_harness; mod cursor; mod cursor_desktop; +mod fidelity; mod fx; mod grok; mod grok_bot;