diff --git a/CHANGELOG.md b/CHANGELOG.md index b4fae10..3badfec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Each release is published to [crates.io](https://crates.io/crates/txcript), ### Added +- Add `txcript check [#range] [--with ]...`, a conversion fidelity report: what converting a session into each harness would lose, change, or make up, before any copy is written. A table of every writable harness by default, a full report per `--with`, `--json` (schema 1), `--detail`, and `--fail-on ` for CI. `continue` and `crop` print a one-line note on stderr when the chosen target loses something (`--quiet` silences it), `continue --strict` refuses to convert when content would be lost, and `continue --dry-run` is `check` for the same arguments. The library gains `txcript::fidelity` (`assess`, `check_all`) and `txcript::dispatch` (the text codec for a `HarnessId`, shared with the WASM bindings). The measurement is codec-level; see the [fidelity docs](docs/usage.md#fidelity) and the [per-harness table](docs/formats/README.md#conversion-fidelity). - Add read-side session lineage (`Meta::lineage`): fork/continue/spawn relationships read from Codex's `session_meta` (subagent spawns, guardian review, `/fork`), pi/Campfire's `parentSession` (`/fork`, `/clone`), OpenCode's `session.parent_id`, and Claude Code diff --git a/README.md b/README.md index 9727819..7f588ea 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,8 @@ The common model represents messages, reasoning, tool calls and results, images, Conversion carries conversation history. The destination supplies its own system instructions and tools, and project files must be available separately. Native load/save and conversion have different preservation guarantees; see each [format's caveats](docs/formats/README.md). +To see what a conversion would lose before you do it, run `txcript check --with `; `continue` also prints a one-line note when the target drops something ([fidelity](docs/usage.md#fidelity)). + ## Documentation - [CLI reference](docs/usage.md#cli): commands, search, cropping, MCP, and shell integration. diff --git a/cli/src/check.rs b/cli/src/check.rs new file mode 100644 index 0000000..280dff2 --- /dev/null +++ b/cli/src/check.rs @@ -0,0 +1,1034 @@ +//! `txcript check` — what converting a session into a harness would lose. +//! +//! The source is resolved exactly like `continue` (a session id or title, a +//! Simple document file or `-`, an optional `#range`), measured with +//! [`txcript::fidelity`], and reported without writing anything or launching +//! an agent. The same measurement backs the one-line note and the `--strict` +//! gate on `continue` and `crop`, and `continue --dry-run`. + +use std::fmt::Write as _; +use std::process::ExitCode; + +use serde::Serialize; +use txcript::fidelity::{self, Category, Loss, LossKind, Options, Report, Severity}; +use txcript::{Common, HarnessId, Transcript}; + +use crate::{fragment, style}; + +/// The JSON document's `schema` number. Bump it only for a change that +/// breaks existing readers; adding fields does not. +const SCHEMA: u32 = 1; + +/// When `--fail-on` should make `check` exit non-zero. +#[derive(Clone, Copy, Debug, PartialEq, Eq, clap::ValueEnum)] +pub enum FailOn { + /// Never (the default) + None, + /// When content is lost or changed: text, thinking, tool calls, results, + /// images, artifacts + Content, + /// When accounting or content is lost or changed: usage, stop reasons, + /// per-message models, thinking signatures + Accounting, + /// On any loss or change, session metadata included + Metadata, +} + +impl FailOn { + /// The lowest severity that fails, if any. + pub(crate) fn threshold(self) -> Option { + match self { + FailOn::None => Option::None, + FailOn::Content => Some(Severity::Content), + FailOn::Accounting => Some(Severity::Accounting), + FailOn::Metadata => Some(Severity::Metadata), + } + } +} + +/// Why a harness cannot be written to, if it cannot. These are the harnesses +/// `local::write_with` refuses outright; a test keeps the two in step. +pub(crate) fn unwritable(target: HarnessId) -> Option<&'static str> { + match target { + HarnessId::ClaudeChat | HarnessId::ChatGpt => { + Some("it is a live read-only source and is never continued into") + } + HarnessId::CoworkRemote => Some("cloud Cowork is pull-only"), + HarnessId::Hermes => Some("its state.db is read-only and Hermes has no session import"), + HarnessId::Amp => Some("amp threads live on the server and cannot be imported"), + HarnessId::Simple => { + Some("simple is an interchange input; sessions are not written into it") + } + _ => None, + } +} + +fn not_writable(target: HarnessId, reason: &str) -> String { + format!("{target} is not writable: {reason}") +} + +/// The targets a bare `check` compares: every harness `continue --with` +/// writes. `grok_bot` mints a live agent and only appears when named. +fn matrix_targets() -> Vec { + fidelity::TARGETS + .into_iter() + .filter(|t| unwritable(*t).is_none() && *t != HarnessId::GrokBot) + .collect() +} + +/// A session resolved for checking. +pub(crate) struct Resolved { + pub harness: HarnessId, + /// The transcript as it would be written: already cut to any `#range`. + pub common: Transcript, + /// Whether a `#range` cut it down. + pub sliced: bool, +} + +fn slice( + harness: HarnessId, + common: Transcript, + request: Option<&fragment::SpanReq>, +) -> Result { + match request { + Some(req) => Ok(Resolved { + harness, + common: fragment::sliced(&common, req)?, + sliced: true, + }), + None => Ok(Resolved { + harness, + common, + sliced: false, + }), + } +} + +/// Resolve `id` the way `continue` does: a Simple document first, then a live +/// source, then a local session by id or title, then an Amp server thread. +pub(crate) fn resolve(id: &str, from: Option) -> Result { + if let Some((input, request)) = crate::document_source(id) { + if from.is_some() { + return Err(crate::FROM_WITH_DOCUMENT.to_string()); + } + let common = crate::load_document(&input)?; + return slice(HarnessId::Simple, common, request.as_ref()); + } + if let Some((harness, loaded)) = crate::load_direct_remote(id, from) { + let (common, request) = loaded?; + return slice(harness, common, request.as_ref()); + } + let sessions = crate::discover_with_spinner(from)?; + let (src, request) = match fragment::parse_ref(id) { + (_, Some(_)) if crate::find_exact(&sessions, from, id).is_some() => (id, None), + parsed => parsed, + }; + match crate::find_session(&sessions, from, src)? { + Some(found) => { + let common = found.read().map_err(|e| e.to_string())?; + slice(found.harness, common, request.as_ref()) + } + None if matches!(from, None | Some(HarnessId::Amp)) && crate::is_amp_thread_id(src) => { + let common = crate::load_amp_server_thread(src)?; + slice(HarnessId::Amp, common, request.as_ref()) + } + None => Err(crate::no_session_error(from, src)), + } +} + +/// One target's outcome. +struct Outcome { + target: HarnessId, + result: Result, +} + +fn assess_all(common: &Transcript, targets: &[HarnessId], detail: bool) -> Vec { + let options = Options { detail }; + targets + .iter() + .map(|&target| { + let result = match unwritable(target) { + Some(reason) => Err(not_writable(target, reason)), + None => fidelity::assess_with(target, common, options) + .map_err(|e| format!("{target}: {e}")), + }; + Outcome { target, result } + }) + .collect() +} + +/// How `check` presents its result. +#[derive(Clone, Copy)] +pub(crate) struct Present { + pub json: bool, + pub detail: bool, + pub fail_on: Option, +} + +/// `txcript check`. With no `with`, one row per writable target; with some, +/// a full report for each. +pub(crate) fn cmd_check( + id: &str, + with: &[HarnessId], + from: Option, + present: Present, +) -> Result { + let source = resolve(id, from)?; + let targets = if with.is_empty() { + matrix_targets() + } else { + with.to_vec() + }; + let outcomes = assess_all(&source.common, &targets, present.detail); + let text = if present.json { + render_json(&source, &outcomes)? + } else if with.is_empty() { + render_matrix(&source, &outcomes, id) + } else { + outcomes + .iter() + .map(|o| render_target(&source, o, present.detail)) + .collect::>() + .join("\n") + }; + // A failed write means the reader is gone (`txcript check … | head`): + // finish quietly instead of panicking the way `print!` would. + let _ = std::io::Write::write_all(&mut std::io::stdout(), text.as_bytes()); + Ok(exit_code(&outcomes, present.fail_on)) +} + +/// `continue --dry-run`: what `continue` would write, and what it would cost. +pub(crate) fn cmd_dry_run( + id: &str, + with: Option, + from: Option, + strict: bool, +) -> Result { + let source = resolve(id, from)?; + let target = match (with, source.harness) { + (Some(target), _) => target, + (None, HarnessId::Simple) => return Err(crate::DOCUMENT_NEEDS_WITH.to_string()), + (None, harness) => harness, + }; + crate::ensure_resumable_source(source.harness, target)?; + // What `continue` does without a range: resume the original where it is. + let in_place = target == source.harness && !source.sliced && target != HarnessId::Simple; + let (text, code) = if in_place { + let text = format!( + "continue {} in {target}: resumes the session in place; nothing is converted\n", + style::scrub(&short(&source.common)), + ); + (text, ExitCode::SUCCESS) + } else { + let outcome = assess_all(&source.common, &[target], false).remove(0); + let text = render_target(&source, &outcome, false); + let code = exit_code(&[outcome], strict.then_some(Severity::Content)); + (text, code) + }; + let _ = std::io::Write::write_all(&mut std::io::stdout(), text.as_bytes()); + Ok(code) +} + +fn exit_code(outcomes: &[Outcome], fail_on: Option) -> ExitCode { + let failed = outcomes.iter().any(|o| match &o.result { + Err(_) => true, + Ok(report) => fail_on.is_some_and(|min| report.severity >= min), + }); + if failed { + ExitCode::FAILURE + } else { + ExitCode::SUCCESS + } +} + +// ── the `continue` / `crop` gate ─────────────────────────────────────── + +/// What `continue` and `crop` do about loss: say so on stderr, and with +/// `strict` refuse to write. +#[derive(Clone, Copy)] +pub(crate) struct Gate<'a> { + pub quiet: bool, + pub strict: bool, + /// The session argument as the user typed it, for the suggested command. + pub reference: &'a str, + pub from: Option, +} + +impl<'a> Gate<'a> { + /// The default: note any loss, never refuse. + pub(crate) fn note_only(reference: &'a str) -> Gate<'a> { + Gate { + quiet: false, + strict: false, + reference, + from: None, + } + } + + /// Measure `common` as it is about to be written into `target`. Returns + /// the note to print once the write succeeds, or an error that stops the + /// write when `strict` and content would be lost or changed. + pub(crate) fn before_write( + &self, + target: HarnessId, + common: &Transcript, + ) -> Result, String> { + if (self.quiet && !self.strict) || unwritable(target).is_some() { + return Ok(None); + } + // A codec failure surfaces from the write itself, with its own error. + let Ok(report) = fidelity::assess(target, common) else { + return Ok(None); + }; + let command = self.command(target); + if self.strict && report.severity >= Severity::Content { + return Err(format!( + "refusing to convert to {target} with --strict: content would be lost or \ + changed ({}); nothing was written; run '{command}' for details", + summarize(&report, Severity::Content).unwrap_or_default() + )); + } + // "Lossy" only when something is actually gone; a representation + // that merely changed is not a loss. + let lossy = report.items.iter().any(|i| i.kind == LossKind::Lost); + let what = if lossy { + "lossy conversion" + } else { + "conversion changes" + }; + Ok(summarize(&report, Severity::Metadata) + .filter(|_| !self.quiet) + .map(|s| format!("note: {what}: {s}; run '{command}' for details"))) + } + + fn command(&self, target: HarnessId) -> String { + let from = self.from.map_or(String::new(), |h| format!(" --from {h}")); + format!( + "{} check {}{from} --with {target}", + crate::program(), + shell_quote(self.reference) + ) + } +} + +/// `s` as one shell word. +fn shell_quote(s: &str) -> String { + let plain = !s.is_empty() + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || "_./#:@%+=,-".contains(c)); + if plain { + s.to_string() + } else { + format!("'{}'", s.replace('\'', "'\\''")) + } +} + +/// What was lost or changed in categories at or above `floor`, in a few +/// words: `200 usage records, title`. `None` when nothing was. +fn summarize(report: &Report, floor: Severity) -> Option { + const SHOWN: usize = 6; + let mut parts: Vec<(Category, usize)> = Vec::new(); + for item in report + .items + .iter() + .filter(|i| i.kind != LossKind::Synthesized && i.category.severity() >= floor) + { + match parts.iter_mut().find(|(c, _)| *c == item.category) { + Some((_, n)) => *n += item.count, + None => parts.push((item.category, item.count)), + } + } + if parts.is_empty() { + return None; + } + let hidden = parts.len().saturating_sub(SHOWN); + let mut text = parts + .iter() + .take(SHOWN) + .map(|&(category, n)| phrase(category, n, None)) + .collect::>() + .join(", "); + if hidden > 0 { + let _ = write!(text, ", and {hidden} more"); + } + Some(text) +} + +// ── wording ──────────────────────────────────────────────────────────── + +/// `1 image` / `3 images`; a singular session-level field has no count. +fn phrase(category: Category, n: usize, path: Option<&str>) -> String { + let (one, many) = match category { + Category::Text => ("text block", "text blocks"), + Category::Thinking => ("thinking block", "thinking blocks"), + Category::ToolCall => ("tool call", "tool calls"), + Category::ToolName => ("tool name renamed", "tool names renamed"), + Category::ToolArgs => ("tool argument", "tool arguments"), + Category::ToolResult => ("tool result", "tool results"), + Category::Image => ("image", "images"), + Category::Artifact => ("artifact", "artifacts"), + Category::IsError => ("error flag", "error flags"), + Category::Order => ("block moved out of order", "blocks moved out of order"), + Category::ThinkingSignature => ("thinking signature", "thinking signatures"), + Category::Usage => ("usage record", "usage records"), + Category::StopReason => ("stop reason", "stop reasons"), + Category::MessageModel => ("per-message model", "per-message models"), + Category::Title => return "title".to_string(), + Category::Model => return "model".to_string(), + Category::GitBranch => return "git branch".to_string(), + Category::Cwd => return "working directory".to_string(), + Category::Lineage => return "lineage".to_string(), + Category::Timestamp => ("message timestamp", "message timestamps"), + }; + let (one, many) = match (category, path) { + (Category::Usage, Some(_)) => ("usage field", "usage fields"), + _ => (one, many), + }; + let noun = if n == 1 { one } else { many }; + let mut text = format!("{n} {noun}"); + if let Some(path) = path { + let _ = write!(text, " ({path})"); + } + text +} + +fn class_name(severity: Severity) -> &'static str { + match severity { + Severity::Content => "content", + Severity::Accounting => "accounting", + Severity::Metadata => "metadata", + Severity::None => "none", + } +} + +const CLASSES: [Severity; 3] = [Severity::Content, Severity::Accounting, Severity::Metadata]; + +fn preserved(class: Severity) -> &'static str { + match class { + Severity::Content => "text, thinking, tool calls, results, images preserved", + Severity::Accounting => "usage, stop reasons, per-message models preserved", + _ => "title, model, branch, working directory preserved", + } +} + +fn short(common: &Transcript) -> String { + let id: String = common.meta.id.chars().take(8).collect(); + if id.is_empty() { "-".to_string() } else { id } +} + +fn header(source: &Resolved) -> String { + format!( + "check {} ({}, {})", + style::scrub(&short(&source.common)), + source.harness, + messages(source.common.body.len()) + ) +} + +fn messages(n: usize) -> String { + format!("{n} message{}", if n == 1 { "" } else { "s" }) +} + +// ── human output ─────────────────────────────────────────────────────── + +/// What a class did overall: the worst thing among its lost and changed items. +fn status(report: &Report, class: Severity) -> &'static str { + let in_class = |kind| { + report + .items + .iter() + .any(|i| i.kind == kind && i.category.severity() == class) + }; + if in_class(LossKind::Lost) { + "lost" + } else if in_class(LossKind::Changed) { + "changed" + } else { + "ok" + } +} + +fn item_line(item: &Loss, detail: bool) -> String { + let tag = match item.kind { + LossKind::Lost => "LOST", + LossKind::Changed => "CHANGED", + LossKind::Synthesized => "ADDED", + }; + // A rename shows as `/release -> _release`; every other path qualifies + // the noun, as in `tool arguments (Bash.description)`. + let (path, example) = match item.category { + Category::ToolName => (None, item.path.as_deref()), + Category::Timestamp => (None, item.example.as_deref()), + _ => (item.path.as_deref(), item.example.as_deref()), + }; + let mut line = format!("{tag:<8} {}", phrase(item.category, item.count, path)); + if let Some(example) = example { + let _ = write!(line, " {}", style::scrub(example)); + } + if detail && let Some(at) = item.first_at { + let _ = write!(line, " (first at message {})", at + 1); + } + line +} + +fn render_target(source: &Resolved, outcome: &Outcome, detail: bool) -> String { + let mut out = String::new(); + let _ = writeln!(out, "{} -> {}", header(source), outcome.target); + let report = match &outcome.result { + Ok(report) => report, + Err(error) => { + let _ = writeln!(out, " error {}", style::scrub(error)); + return out; + } + }; + for class in CLASSES { + let items: Vec<&Loss> = report + .items + .iter() + .filter(|i| i.category.severity() == class) + .collect(); + let name = class_name(class); + if items.iter().all(|i| i.kind == LossKind::Synthesized) { + let _ = writeln!(out, " {name:<10} ok: {}", preserved(class)); + } + for (n, item) in items.iter().enumerate() { + let label = if n == 0 && status(report, class) != "ok" { + name + } else { + "" + }; + let _ = writeln!(out, " {label:<10} {}", item_line(item, detail)); + } + } + if report.messages_before != report.messages_after { + let how = if report.messages_after > report.messages_before { + "the target splits turns" + } else { + "the target merges turns" + }; + let _ = writeln!( + out, + " {:<10} {} become {} ({how})", + "structure", + messages(report.messages_before), + report.messages_after + ); + } + for warning in &report.warnings { + let _ = writeln!(out, " {:<10} {warning}", "warning"); + } + let _ = writeln!( + out, + " codec-level: measured on the target's session text; a store's own additions are not counted" + ); + out +} + +fn render_matrix(source: &Resolved, outcomes: &[Outcome], id: &str) -> String { + let mut out = String::new(); + let _ = writeln!( + out, + "{}, codec-level, {} targets", + header(source), + outcomes.len() + ); + let _ = writeln!( + out, + "{:<15} {:<8} {:<11} {:<9} notes", + "target", "content", "accounting", "metadata" + ); + for outcome in outcomes { + match &outcome.result { + Err(error) => { + let _ = writeln!( + out, + "{:<15} error: {}", + outcome.target.as_str(), + style::scrub(error) + ); + } + Ok(report) => { + let mut notes = Vec::new(); + if report.messages_before != report.messages_after { + notes.push(format!( + "messages {}->{}", + report.messages_before, report.messages_after + )); + } + let added = report.total(LossKind::Synthesized); + if added > 0 { + notes.push(format!("{added} added")); + } + let row = format!( + "{:<15} {:<8} {:<11} {:<9} {}", + outcome.target.as_str(), + status(report, Severity::Content), + status(report, Severity::Accounting), + status(report, Severity::Metadata), + notes.join(", ") + ); + let _ = writeln!(out, "{}", row.trim_end()); + } + } + } + let _ = writeln!( + out, + "run '{} check {} --with ' for details", + crate::program(), + shell_quote(id) + ); + out +} + +// ── JSON ─────────────────────────────────────────────────────────────── + +#[derive(Serialize)] +struct Document<'a> { + schema: u32, + source: SourceJson<'a>, + targets: Vec>, +} + +#[derive(Serialize)] +struct SourceJson<'a> { + harness: HarnessId, + id: &'a str, + messages: usize, +} + +#[derive(Serialize)] +struct TargetJson<'a> { + harness: HarnessId, + error: Option<&'a str>, + messages_after: Option, + severity: Option, + items: Vec>, + warnings: &'a [String], + counts: Option, +} + +#[derive(Serialize)] +struct ItemJson<'a> { + kind: LossKind, + category: Category, + path: Option<&'a str>, + count: usize, + /// 1-based, as `view` numbers messages and `#range` selects them. + first_message: Option, + example: Option<&'a str>, +} + +#[derive(Serialize)] +struct Counts { + lost: usize, + changed: usize, + synthesized: usize, +} + +fn render_json(source: &Resolved, outcomes: &[Outcome]) -> Result { + let targets = outcomes + .iter() + .map(|outcome| match &outcome.result { + Err(error) => TargetJson { + harness: outcome.target, + error: Some(error), + messages_after: None, + severity: None, + items: Vec::new(), + warnings: &[], + counts: None, + }, + Ok(report) => TargetJson { + harness: outcome.target, + error: None, + messages_after: Some(report.messages_after), + severity: Some(report.severity), + items: report + .items + .iter() + .map(|item| ItemJson { + kind: item.kind, + category: item.category, + path: item.path.as_deref(), + count: item.count, + first_message: item.first_at.map(|at| at + 1), + example: item.example.as_deref(), + }) + .collect(), + warnings: &report.warnings, + counts: Some(Counts { + lost: report.total(LossKind::Lost), + changed: report.total(LossKind::Changed), + synthesized: report.total(LossKind::Synthesized), + }), + }, + }) + .collect(); + let document = Document { + schema: SCHEMA, + source: SourceJson { + harness: source.harness, + id: &source.common.meta.id, + messages: source.common.body.len(), + }, + targets, + }; + let mut text = serde_json::to_string_pretty(&document).map_err(|e| e.to_string())?; + text.push('\n'); + Ok(text) +} + +#[cfg(test)] +mod tests { + use chrono::DateTime; + use txcript::common::{Block, Message, Meta, Role, StopReason, Tool}; + + use super::*; + + fn session(title: Option<&str>, command: bool) -> Transcript { + let at = DateTime::from_timestamp(1_790_000_000, 0).unwrap(); + let message = |role, content| Message { + role, + content, + timestamp: at, + model: None, + stop_reason: None, + usage: None, + }; + let mut body = vec![ + message( + Role::User, + vec![Block::Text { + text: "hello".into(), + }], + ), + Message { + stop_reason: Some(StopReason::EndTurn), + ..message(Role::Assistant, vec![Block::Text { text: "hi".into() }]) + }, + ]; + if command { + body.push(message( + Role::User, + vec![Block::ToolUse { + id: "c1".into(), + tool: Tool::Command { + command: "/release".into(), + args: None, + }, + }], + )); + } + Transcript::new( + Meta { + id: "abcdef012345".into(), + timestamp: at, + cwd: None, + git_branch: None, + title: title.map(Into::into), + cli_version: None, + model: None, + lineage: None, + }, + body, + ) + } + + fn gate(quiet: bool, strict: bool) -> Gate<'static> { + Gate { + quiet, + strict, + reference: "my session", + from: Some(HarnessId::ClaudeCode), + } + } + + #[test] + fn the_note_summarizes_what_was_lost_and_names_the_command() { + let note = gate(false, false) + .before_write(HarnessId::Pi, &session(Some("T"), false)) + .unwrap() + .unwrap(); + assert_eq!( + note, + "note: lossy conversion: title; run 'txcript check 'my session' \ + --from claude_code --with pi' for details" + ); + } + + #[test] + fn a_change_without_a_loss_is_not_called_lossy() { + let mut t = session(None, false); + t.body[1].stop_reason = Some(StopReason::Aborted); + let note = gate(false, false) + .before_write(HarnessId::ClaudeCode, &t) + .unwrap() + .unwrap(); + assert!( + note.starts_with("note: conversion changes: 1 stop reason; run "), + "{note}" + ); + } + + #[test] + fn nothing_lost_means_no_note() { + let t = session(None, false); + assert_eq!( + gate(false, false).before_write(HarnessId::Pi, &t).unwrap(), + None + ); + assert_eq!( + gate(false, true) + .before_write(HarnessId::ClaudeCode, &t) + .unwrap(), + None + ); + } + + #[test] + fn quiet_silences_the_note_but_not_strict() { + let titled = session(Some("T"), false); + assert_eq!( + gate(true, false) + .before_write(HarnessId::Pi, &titled) + .unwrap(), + None + ); + let cmd = session(None, true); + let err = gate(true, true) + .before_write(HarnessId::Pi, &cmd) + .unwrap_err(); + assert!( + err.starts_with("refusing to convert to pi with --strict"), + "{err}" + ); + assert!(err.contains("nothing was written"), "{err}"); + } + + #[test] + fn strict_only_refuses_content_loss() { + // Metadata loss alone is a note, not a refusal. + let titled = session(Some("T"), false); + let note = gate(false, true) + .before_write(HarnessId::Pi, &titled) + .unwrap(); + assert!(note.is_some()); + // A lost call is content. + let cmd = session(None, true); + assert!(gate(false, true).before_write(HarnessId::Pi, &cmd).is_err()); + assert!( + gate(false, false) + .before_write(HarnessId::Pi, &cmd) + .unwrap() + .is_some() + ); + } + + #[test] + fn an_unwritable_target_is_left_to_the_write() { + let cmd = session(Some("T"), true); + for target in [ + HarnessId::Hermes, + HarnessId::Amp, + HarnessId::Simple, + HarnessId::ChatGpt, + ] { + assert_eq!(gate(false, true).before_write(target, &cmd).unwrap(), None); + } + } + + /// `unwritable` mirrors what `local::write_with` refuses outright; if the + /// library grows or loses a refusal this fails and the list is updated. + #[test] + fn unwritable_agrees_with_the_store_layer() { + let dir = tempfile::tempdir().unwrap(); + let t = session(Some("T"), false); + for harness in HarnessId::ALL { + let written = txcript::local::write_with( + harness, + &t, + txcript::local::WriteOpts { + root: Some(dir.path()), + metadata: None, + }, + ); + if unwritable(harness).is_some() { + assert!(written.is_err(), "{harness} is listed unwritable but wrote"); + } + } + for harness in [ + HarnessId::ClaudeCode, + HarnessId::Codex, + HarnessId::Pi, + HarnessId::Campfire, + HarnessId::Cursor, + HarnessId::Grok, + HarnessId::Fx, + HarnessId::Antigravity, + ] { + assert!(unwritable(harness).is_none(), "{harness}"); + } + } + + #[test] + fn the_matrix_covers_writable_targets_and_leaves_out_grok_bot() { + let targets = matrix_targets(); + assert_eq!(targets.len(), 11); + assert!(!targets.contains(&HarnessId::GrokBot)); + assert!(!targets.contains(&HarnessId::Simple)); + assert!(targets.iter().all(|t| unwritable(*t).is_none())); + } + + #[test] + fn shell_words_are_quoted_only_when_needed() { + assert_eq!(shell_quote("abc123"), "abc123"); + assert_eq!(shell_quote("abc#5-12"), "abc#5-12"); + assert_eq!(shell_quote("/tmp/run.json"), "/tmp/run.json"); + assert_eq!(shell_quote("Fix the bug"), "'Fix the bug'"); + assert_eq!(shell_quote("it's"), "'it'\\''s'"); + assert_eq!(shell_quote(""), "''"); + assert_eq!(shell_quote("a;b"), "'a;b'"); + } + + #[test] + fn fail_on_maps_to_a_threshold() { + assert_eq!(FailOn::None.threshold(), None); + assert_eq!(FailOn::Content.threshold(), Some(Severity::Content)); + assert_eq!(FailOn::Accounting.threshold(), Some(Severity::Accounting)); + assert_eq!(FailOn::Metadata.threshold(), Some(Severity::Metadata)); + } + + #[test] + fn exit_codes_follow_severity_and_errors() { + let report = |severity| Outcome { + target: HarnessId::Pi, + result: Ok(Report { + target: HarnessId::Pi, + severity, + messages_before: 1, + messages_after: 1, + items: Vec::new(), + warnings: Vec::new(), + }), + }; + let failed = |outcomes: &[Outcome], at| exit_code(outcomes, at) == ExitCode::FAILURE; + assert!(!failed(&[report(Severity::Content)], None)); + assert!(failed( + &[report(Severity::Content)], + Some(Severity::Content) + )); + assert!(!failed( + &[report(Severity::Accounting)], + Some(Severity::Content) + )); + assert!(failed( + &[report(Severity::Accounting)], + Some(Severity::Metadata) + )); + assert!(!failed(&[report(Severity::None)], Some(Severity::Metadata))); + let broken = Outcome { + target: HarnessId::Hermes, + result: Err("nope".into()), + }; + assert!(failed(&[report(Severity::None), broken], None)); + } + + #[test] + fn phrases_count_and_qualify() { + assert_eq!(phrase(Category::Usage, 1, None), "1 usage record"); + assert_eq!(phrase(Category::Usage, 40, None), "40 usage records"); + assert_eq!( + phrase(Category::Usage, 3, Some("cache_read_input_tokens")), + "3 usage fields (cache_read_input_tokens)" + ); + assert_eq!(phrase(Category::Title, 1, None), "title"); + assert_eq!( + phrase(Category::ToolArgs, 2, Some("Bash.description")), + "2 tool arguments (Bash.description)" + ); + } + + #[test] + fn the_summary_is_content_first_and_capped() { + let item = |category, count| Loss { + kind: LossKind::Lost, + category, + path: None, + count, + first_at: None, + example: None, + }; + let report = Report { + target: HarnessId::Pi, + severity: Severity::Content, + messages_before: 1, + messages_after: 1, + items: vec![ + item(Category::Text, 2), + item(Category::ToolCall, 1), + item(Category::Image, 1), + item(Category::Usage, 5), + item(Category::StopReason, 1), + item(Category::MessageModel, 1), + item(Category::Title, 1), + item(Category::Model, 1), + ], + warnings: Vec::new(), + }; + assert_eq!( + summarize(&report, Severity::Metadata).unwrap(), + "2 text blocks, 1 tool call, 1 image, 5 usage records, 1 stop reason, \ + 1 per-message model, and 2 more" + ); + assert_eq!( + summarize(&report, Severity::Content).unwrap(), + "2 text blocks, 1 tool call, 1 image" + ); + let mut synthesized = report; + synthesized.items = vec![Loss { + kind: LossKind::Synthesized, + ..item(Category::Text, 3) + }]; + assert_eq!(summarize(&synthesized, Severity::Metadata), None); + } + + #[test] + fn the_matrix_has_no_trailing_spaces_and_marks_errors() { + let source = Resolved { + harness: HarnessId::Simple, + common: session(Some("T"), false), + sliced: false, + }; + let outcomes = assess_all(&source.common, &[HarnessId::Pi, HarnessId::Hermes], false); + let text = render_matrix(&source, &outcomes, "abcdef012345"); + assert!(text.starts_with("check abcdef01 (simple, 2 messages), codec-level, 2 targets\n")); + assert!(text.lines().all(|l| l == l.trim_end()), "{text:?}"); + assert!( + text.contains("pi ok ok lost"), + "{text}" + ); + assert!( + text.contains("hermes error: hermes is not writable"), + "{text}" + ); + } + + #[test] + fn json_nulls_the_fields_of_a_target_that_failed() { + let source = Resolved { + harness: HarnessId::Simple, + common: session(None, false), + sliced: false, + }; + let outcomes = assess_all(&source.common, &[HarnessId::Amp], false); + let value: serde_json::Value = + serde_json::from_str(&render_json(&source, &outcomes).unwrap()).unwrap(); + let target = &value["targets"][0]; + assert_eq!(target["harness"], "amp"); + assert!(target["error"].as_str().unwrap().contains("not writable")); + for key in ["messages_after", "severity", "counts"] { + assert!(target[key].is_null(), "{key}"); + } + assert_eq!(value["schema"], 1); + } +} diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 30feb8b..7f4a66c 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -15,13 +15,23 @@ //! [--metadata ] # harness mint options (repeatable; //! # key=value or JSON object; grok_bot //! # uses name/description) +//! [--quiet] # no stderr note about a lossy conversion +//! [--strict] # refuse to convert when content would be lost +//! [--dry-run] # like `check`: report, write nothing //! txcript continue [#range] # continue a Simple document (file, or stdin //! --with [...] # for `-`) into ; see //! # docs/formats/simple.md //! # --with grok_bot never launches a CLI +//! txcript check [#range] # what converting would lose; writes nothing +//! [--with ]... # one detailed report per harness (default: +//! # a matrix of every writable harness) +//! [--from ] # scope the id lookup to one harness +//! [--json] [--detail] # machine-readable; longer examples +//! [--fail-on ] # exit 1 at or above this severity //! txcript crop [#range] # interactively cut messages and save a copy //! [--with ] # optionally convert the cropped copy //! [--from ] # scope the source lookup +//! [--quiet] [--strict] # as for `continue` //! txcript view [#range] # view a session; compact text when piped //! [--from ] # scope the id lookup to one harness //! [--no-pager] # print the terminal view directly @@ -58,6 +68,7 @@ use txcript::harness::{amp, chatgpt, claude_chat, cowork_remote, simple}; use txcript::{Codec, Common, HarnessId, Store, TextCodec, Transcript, local}; pub mod cache; +mod check; mod draft; mod editpane; mod export; @@ -68,6 +79,8 @@ pub mod mcp; mod pager; mod view; +pub use check::FailOn; + pub const HARNESSES: &str = "harnesses: claude_code, claude_chat, chatgpt, codex, opencode, pi, campfire, cursor, cursor_desktop, grok, fx, hermes, \ amp, antigravity, simple, cowork, cowork_remote"; @@ -184,6 +197,65 @@ pub enum SessionCommand { /// recognizes `name` and `description` for the minted agent profile. #[arg(long, value_name = "SPEC")] metadata: Vec, + /// Don't print the note about what a lossy conversion drops + #[arg(long)] + quiet: bool, + /// Refuse to convert when content (text, thinking, tool calls, + /// results, images) would be lost or changed: error out before + /// writing or launching anything. Same as `check --fail-on content` + #[arg(long)] + strict: bool, + /// Report what would be written and what it would lose, as + /// `check` does, then stop without writing or launching anything + #[arg(long)] + dry_run: bool, + }, + /// Report what converting a session into a harness would lose + /// + /// Without --with, compares every harness `continue --with` can write, one + /// row each. With --with (repeatable), prints a full report per harness. + /// Nothing is written and no agent is launched. The source is resolved as + /// `continue` does: a session id or title, a Simple document file, or `-` + /// for stdin, with an optional `#range` so the report covers exactly what + /// would be written. + /// + /// The report is codec-level: it converts the session to the target's + /// native format and reads it back, so it shows what the format stores, + /// not what a store adds around it (a fresh id and timestamp, artifact + /// files, a re-homed working directory). Content is text, thinking, tool + /// calls, results, images. Accounting is usage, stop reasons, per-message + /// models, thinking signatures. Metadata is title, model, branch, working + /// directory, timestamps. + /// + /// By default the report shows counts and short examples only (80 + /// characters, never a tool result or image). --detail adds longer + /// examples, a preview of changed tool results, and message numbers. + /// Exit status is 0, or 1 when a target cannot be assessed or --fail-on's + /// severity is reached. + Check { + /// Session id (any unambiguous prefix) or its exact title; or a Simple + /// document (a file path, `-` for stdin). Takes an optional `#range` + /// of 1-based inclusive message numbers (`abc#5-12`, `#7`, `#5-`) + // Other: without a hint, generated completions fall back to filenames. + #[arg(value_hint = clap::ValueHint::Other)] + id: String, + /// Report on this harness (repeatable); without it, a table of every + /// writable harness + #[arg(long, value_name = "HARNESS", value_parser = HarnessParser)] + with: Vec, + /// Only look for the session in this harness + #[arg(long, value_name = "HARNESS", value_parser = HarnessParser)] + from: Option, + /// Print a versioned JSON document (`schema: 1`) instead of text + #[arg(long)] + json: bool, + /// Longer examples, a preview of changed tool results, and the first + /// message each difference appears in + #[arg(long)] + detail: bool, + /// Exit 1 when a target's severity reaches this level + #[arg(long, value_enum, value_name = "SEVERITY", default_value = "none")] + fail_on: FailOn, }, /// Interactively crop a session into a new, resumable session /// @@ -204,6 +276,13 @@ pub enum SessionCommand { /// Only look for the source session in this harness #[arg(long, value_name = "HARNESS", value_parser = HarnessParser)] from: Option, + /// Don't print the note about what a lossy conversion drops + #[arg(long)] + quiet: bool, + /// Refuse to save when content would be lost or changed in the + /// target harness; checked when you save, before anything is written + #[arg(long)] + strict: bool, }, /// View a session in the terminal or print compact text to a pipe /// @@ -401,8 +480,53 @@ pub fn run_session(command: SessionCommand, options: &Options) -> Result cmd_continue(&id, with, from, out.as_ref(), no_resume, &metadata), - SessionCommand::Crop { source, with, from } => cmd_crop(&source, with, from), + quiet, + strict, + dry_run, + } => { + if dry_run { + return check::cmd_dry_run(&id, with, from, strict); + } + let gate = check::Gate { + quiet, + strict, + reference: &id, + from, + }; + cmd_continue(&id, with, from, out.as_ref(), no_resume, &metadata, gate) + } + SessionCommand::Check { + id, + with, + from, + json, + detail, + fail_on, + } => check::cmd_check( + &id, + &with, + from, + check::Present { + json, + detail, + fail_on: fail_on.threshold(), + }, + ), + SessionCommand::Crop { + source, + with, + from, + quiet, + strict, + } => { + let gate = check::Gate { + quiet, + strict, + reference: &source, + from, + }; + cmd_crop(&source, with, from, gate) + } SessionCommand::View { source, from, @@ -1056,12 +1180,20 @@ mod crop_command_tests { ]) .unwrap(); - let Command::Session(SessionCommand::Crop { source, from, with }) = cli.command else { + let Command::Session(SessionCommand::Crop { + source, + from, + with, + quiet, + strict, + }) = cli.command + else { panic!("expected crop command"); }; assert_eq!(source, "session-123#2-4"); assert_eq!(from, Some(HarnessId::ClaudeCode)); assert_eq!(with, Some(HarnessId::Codex)); + assert!(!quiet && !strict); } #[test] @@ -1336,6 +1468,7 @@ fn cmd_crop( source: &str, with: Option, from: Option, + gate: check::Gate<'_>, ) -> Result { if !std::io::stdin().is_terminal() || !std::io::stdout().is_terminal() { return Err("crop is interactive and requires a terminal".to_string()); @@ -1353,7 +1486,7 @@ fn cmd_crop( let target = with.unwrap_or(source_harness); ensure_crop_target(target)?; let (common, request) = loaded?; - return crop_loaded(&common, source_harness, target, request.as_ref()); + return crop_loaded(&common, source_harness, target, request.as_ref(), gate); } let sessions = discover_with_spinner(from)?; let (id, request) = match crop_ref(source) { @@ -1365,13 +1498,13 @@ fn cmd_crop( let target = with.unwrap_or(found.harness); ensure_crop_target(target)?; let common = found.read().map_err(|error| error.to_string())?; - crop_loaded(&common, found.harness, target, request.as_ref()) + crop_loaded(&common, found.harness, target, request.as_ref(), gate) } None if matches!(from, None | Some(HarnessId::Amp)) && is_amp_thread_id(id) => { let target = with.unwrap_or(HarnessId::Amp); ensure_crop_target(target)?; let common = load_amp_server_thread(id)?; - crop_loaded(&common, HarnessId::Amp, target, request.as_ref()) + crop_loaded(&common, HarnessId::Amp, target, request.as_ref(), gate) } None => Err(match from { Some(harness) => format!( @@ -1388,6 +1521,7 @@ fn crop_loaded( source: HarnessId, target: HarnessId, request: Option<&fragment::SpanReq>, + gate: check::Gate<'_>, ) -> Result { ensure_crop_target(target)?; let total = common.body.len(); @@ -1410,7 +1544,7 @@ fn crop_loaded( .map_err(|error| error.to_string())?; fresh_identity(&mut cropped, target, None); stamp_live_cwd(&mut cropped, None); - let cropped_id = write_and_report(source, target, &cropped, None, None)?; + let cropped_id = write_and_report(source, target, &cropped, None, None, gate)?; let edited = match outcome.edited { 0 => String::new(), 1 => " (1 message edited)".to_string(), @@ -1431,6 +1565,7 @@ fn cmd_continue( out: Option<&PathBuf>, no_resume: bool, metadata_specs: &[String], + gate: check::Gate<'_>, ) -> Result { let metadata = parse_metadata_specs(metadata_specs)?; let metadata = @@ -1442,17 +1577,9 @@ fn cmd_continue( // input directly, so no session can shadow it. if let Some((input, request)) = document_source(id) { if from.is_some() { - return Err( - "--from scopes the search for a local session; a Simple document \ - is its own input and takes --with only" - .to_string(), - ); + return Err(FROM_WITH_DOCUMENT.to_string()); } - let target = with.ok_or_else(|| { - "a Simple document has no harness of its own to resume; \ - pass --with (e.g. --with claude_code)" - .to_string() - })?; + let target = with.ok_or_else(|| DOCUMENT_NEEDS_WITH.to_string())?; let resume = wants_resume(target, out.is_some(), no_resume); return continue_document( &input, @@ -1461,6 +1588,7 @@ fn cmd_continue( out.map(PathBuf::as_path), resume, metadata, + gate, ); } @@ -1479,6 +1607,7 @@ fn cmd_continue( out.map(PathBuf::as_path), wants_resume(target, out.is_some(), no_resume), metadata, + gate, ); } let sessions = discover_with_spinner(from)?; @@ -1503,6 +1632,7 @@ fn cmd_continue( out.map(PathBuf::as_path), wants_resume(target, out.is_some(), no_resume), metadata, + gate, ) } // Modern Amp CLIs are server-authoritative and write no local thread @@ -1517,21 +1647,35 @@ fn cmd_continue( out.map(PathBuf::as_path), wants_resume(target, out.is_some(), no_resume), metadata, + gate, ) } - None => Err(match from { - Some(h) => format!("no {h} session matches `{src}` (try `{} list`)", program()), - // A path-looking argument that named no file falls through to the - // session lookup; say so, or the "no session" alone misleads. - None if src.contains(['/', '\\']) => format!( - "no local session matches `{src}`, and no such file exists \ - (a Simple document must be an existing file, or `-` for stdin)" - ), - None => format!( - "no local session matches `{src}` (try `{} list`)", - program() - ), - }), + None => Err(no_session_error(from, src)), + } +} + +/// `--from` scopes the lookup of a local session, which a document is not. +const FROM_WITH_DOCUMENT: &str = "--from scopes the search for a local session; a Simple document \ + is its own input and takes --with only"; + +/// A document has no harness to resume in. +const DOCUMENT_NEEDS_WITH: &str = "a Simple document has no harness of its own to resume; \ + pass --with (e.g. --with claude_code)"; + +/// The error for a session reference that matched nothing. +fn no_session_error(from: Option, src: &str) -> String { + match from { + Some(h) => format!("no {h} session matches `{src}` (try `{} list`)", program()), + // A path-looking argument that named no file falls through to the + // session lookup; say so, or the "no session" alone misleads. + None if src.contains(['/', '\\']) => format!( + "no local session matches `{src}`, and no such file exists \ + (a Simple document must be an existing file, or `-` for stdin)" + ), + None => format!( + "no local session matches `{src}` (try `{} list`)", + program() + ), } } @@ -1628,30 +1772,10 @@ fn continue_document( out: Option<&std::path::Path>, resume: bool, metadata: Option<&serde_json::Value>, + gate: check::Gate<'_>, ) -> Result { - let target = with.ok_or_else(|| { - "a Simple document has no harness of its own to resume; \ - pass --with (e.g. --with claude_code)" - .to_string() - })?; - let text = match input { - DocInput::Stdin => { - let mut buffer = String::new(); - std::io::Read::read_to_string(&mut std::io::stdin(), &mut buffer) - .map_err(|e| format!("reading stdin: {e}"))?; - buffer - } - DocInput::File(path) => { - std::fs::read_to_string(path).map_err(|e| format!("reading {}: {e}", path.display()))? - } - }; - // Only a real file's stem is a meaningful identity; a process - // substitution's `/dev/fd/N` would leak "N" as the session id. - let origin = match input { - DocInput::Stdin => None, - DocInput::File(path) => Some(path.as_path()).filter(|p| p.is_file()), - }; - let common = document_to_common(&text, origin)?; + let target = with.ok_or_else(|| DOCUMENT_NEEDS_WITH.to_string())?; + let common = load_document(input)?; let mut copy = match span_req { Some(req) => fragment::sliced(&common, req)?, @@ -1667,7 +1791,7 @@ fn continue_document( // stores shard by cwd, and "the directory the user ran txcript in" is the // only sensible home for a transcript that never had one. stamp_live_cwd(&mut copy, out); - let resume_id = write_and_report(HarnessId::Simple, target, ©, out, metadata)?; + let resume_id = write_and_report(HarnessId::Simple, target, ©, out, metadata, gate)?; // Stdin was consumed by the document; hand the launched harness the // terminal instead, or an interactive resume would read EOF. let stdin_tty = matches!(input, DocInput::Stdin); @@ -1680,6 +1804,28 @@ fn continue_document( ) } +/// Read a Simple document from stdin or a file into the canonical model. +fn load_document(input: &DocInput) -> Result, String> { + let text = match input { + DocInput::Stdin => { + let mut buffer = String::new(); + std::io::Read::read_to_string(&mut std::io::stdin(), &mut buffer) + .map_err(|e| format!("reading stdin: {e}"))?; + buffer + } + DocInput::File(path) => { + std::fs::read_to_string(path).map_err(|e| format!("reading {}: {e}", path.display()))? + } + }; + // Only a real file's stem is a meaningful identity; a process + // substitution's `/dev/fd/N` would leak "N" as the session id. + let origin = match input { + DocInput::Stdin => None, + DocInput::File(path) => Some(path.as_path()).filter(|p| p.is_file()), + }; + document_to_common(&text, origin) +} + /// Parse a Simple document into the canonical model, backfilling an empty id /// from the file stem (the same fallback the file-backed stores use). fn document_to_common( @@ -1806,6 +1952,7 @@ fn continue_amp_server_thread( out: Option<&std::path::Path>, resume: bool, metadata: Option<&serde_json::Value>, + gate: check::Gate<'_>, ) -> Result { let common = load_amp_server_thread(id)?; @@ -1817,7 +1964,7 @@ fn continue_amp_server_thread( let mut copy = common.clone(); fresh_identity(&mut copy, target, out); stamp_live_cwd(&mut copy, out); - write_and_report(HarnessId::Amp, target, ©, out, metadata)? + write_and_report(HarnessId::Amp, target, ©, out, metadata, gate)? } // A sliced continue always rewrites — the server thread can't resume // a subset of itself in place. @@ -1825,7 +1972,7 @@ fn continue_amp_server_thread( let mut copy = fragment::sliced(&common, req)?; fresh_identity(&mut copy, target, out); stamp_live_cwd(&mut copy, out); - write_and_report(HarnessId::Amp, target, ©, out, metadata)? + write_and_report(HarnessId::Amp, target, ©, out, metadata, gate)? } }; launch(target, &resume_id, common.meta.cwd.as_deref(), resume) @@ -1842,6 +1989,7 @@ fn continue_session( out: Option<&std::path::Path>, resume: bool, metadata: Option<&serde_json::Value>, + gate: check::Gate<'_>, ) -> Result { let target = with.unwrap_or(found.harness); ensure_resumable_source(found.harness, target)?; @@ -1854,20 +2002,21 @@ fn continue_session( let mut common = found.read().map_err(|e| e.to_string())?; fresh_identity(&mut common, target, out); stamp_live_cwd(&mut common, out); - write_and_report(found.harness, target, &common, out, metadata)? + write_and_report(found.harness, target, &common, out, metadata, gate)? } (Some(req), _) => { let common = found.read().map_err(|e| e.to_string())?; let mut copy = fragment::sliced(&common, req)?; fresh_identity(&mut copy, target, out); stamp_live_cwd(&mut copy, out); - write_and_report(found.harness, target, ©, out, metadata)? + write_and_report(found.harness, target, ©, out, metadata, gate)? } }; launch(target, &resume_id, found.meta.cwd.as_deref(), resume) } +#[allow(clippy::too_many_arguments)] fn continue_loaded_remote( common: Transcript, source: HarnessId, @@ -1876,6 +2025,7 @@ fn continue_loaded_remote( out: Option<&std::path::Path>, resume: bool, metadata: Option<&serde_json::Value>, + gate: check::Gate<'_>, ) -> Result { let cwd = common.meta.cwd.clone(); let mut copy = match span_req { @@ -1884,7 +2034,7 @@ fn continue_loaded_remote( }; fresh_identity(&mut copy, target, out); stamp_live_cwd(&mut copy, out); - let resume_id = write_and_report(source, target, ©, out, metadata)?; + let resume_id = write_and_report(source, target, ©, out, metadata, gate)?; launch(target, &resume_id, cwd.as_deref(), resume) } @@ -1981,7 +2131,10 @@ fn write_and_report( common: &Transcript, out: Option<&std::path::Path>, metadata: Option<&serde_json::Value>, + gate: check::Gate<'_>, ) -> Result { + // Measured on exactly what is about to be written, before anything is. + let note = gate.before_write(target, common)?; let written = local::write_with( target, common, @@ -2000,6 +2153,9 @@ fn write_and_report( // shed the quotes it puts around paths. style::dim(written.location.trim_matches('"'), on) ); + if let Some(note) = note { + eprintln!("{}", style::scrub(¬e)); + } Ok(written.id) } @@ -2509,6 +2665,7 @@ mod query { None, with != Some(HarnessId::GrokBot), None, + super::check::Gate::note_only(&session.meta.id), ) } } diff --git a/cli/tests/check.rs b/cli/tests/check.rs new file mode 100644 index 0000000..157381f --- /dev/null +++ b/cli/tests/check.rs @@ -0,0 +1,1343 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] + +//! `txcript check` and the conversion-loss gate on `continue`, through the +//! real binary. Every store environment variable points into a temp dir, so no +//! real history is read or written, and sources are named with `--from` or +//! passed as Simple documents. The prediction-vs-reality tests are the ones +//! that matter most: they convert for real, read the copy back, and hold the +//! report to what actually changed. + +use std::collections::BTreeMap; +use std::fs; +use std::io::Write as _; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; + +use chrono::{DateTime, Utc}; +use serde_json::{Value, json}; +use txcript::HarnessId; +use txcript::common::{ + Block, ImageSource, Message, Meta, Role, StopReason, Tool, ToolOutput, Usage, +}; + +struct Env { + home: tempfile::TempDir, +} + +/// Every variable the stores read, so nothing outside the temp dir is touched. +const STORE_VARS: [&str; 15] = [ + "CODEX_HOME", + "XDG_DATA_HOME", + "XDG_CONFIG_HOME", + "XDG_STATE_HOME", + "PI_CODING_AGENT_DIR", + "OPENCODE_DB", + "HERMES_HOME", + "FX_HOME", + "GROK_HOME", + "COWORK_SESSIONS_DIR", + "AMP_THREADS_DIR", + "CURSOR_DESKTOP_USER_DIR", + "TXCRIPT_GROK_BOT_ROOT", + "GROK_BOT_TRANSCRIPTS", + "TXCRIPT_GROK_BOT_AGENTS", +]; + +impl Env { + fn new() -> Self { + Self { + home: tempfile::tempdir().unwrap(), + } + } + + fn path(&self) -> &Path { + self.home.path() + } + + /// A project directory that exists, so the copy keeps its recorded cwd. + fn project(&self) -> String { + let dir = self.path().join("project"); + fs::create_dir_all(&dir).unwrap(); + dir.to_string_lossy().into_owned() + } + + fn command(&self) -> Command { + let empty = self.path().join("empty"); + fs::create_dir_all(&empty).unwrap(); + let mut cmd = Command::new(env!("CARGO_BIN_EXE_txcript")); + cmd.env_clear() + .env("HOME", self.path()) + .env("USERPROFILE", self.path()) + .env("CLAUDE_CONFIG_DIR", self.path().join(".claude")) + .env("NO_COLOR", "1") + .current_dir(self.path()); + for var in STORE_VARS { + cmd.env(var, &empty); + } + cmd + } + + fn run(&self, args: &[&str]) -> Output { + self.command().args(args).output().unwrap() + } + + fn run_stdin(&self, args: &[&str], stdin: &str) -> Output { + let mut child = self + .command() + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(stdin.as_bytes()) + .unwrap(); + child.wait_with_output().unwrap() + } + + /// Write a Simple document and return its path. + fn document(&self, name: &str, text: &str) -> String { + let path = self.path().join(name); + fs::write(&path, text).unwrap(); + path.to_string_lossy().into_owned() + } + + /// Every file under the temp dir, with its size, excluding the inputs. + fn snapshot(&self) -> BTreeMap { + fn walk(dir: &Path, out: &mut BTreeMap) { + let Ok(entries) = fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + walk(&path, out); + } else { + out.insert(path.clone(), entry.metadata().map_or(0, |m| m.len())); + } + } + } + let mut out = BTreeMap::new(); + walk(self.path(), &mut out); + out + } +} + +fn stdout(out: &Output) -> String { + String::from_utf8_lossy(&out.stdout).into_owned() +} + +fn stderr(out: &Output) -> String { + String::from_utf8_lossy(&out.stderr).into_owned() +} + +fn json(out: &Output) -> Value { + serde_json::from_slice(&out.stdout) + .unwrap_or_else(|e| panic!("not JSON ({e}): {}", stdout(out))) +} + +fn ts(secs: i64) -> DateTime { + DateTime::from_timestamp(1_790_000_000 + secs, 0).unwrap() +} + +fn message(role: Role, content: Vec, secs: i64) -> Message { + Message { + role, + content, + timestamp: ts(secs), + model: None, + stop_reason: None, + usage: None, + } +} + +fn text(s: &str) -> Block { + Block::Text { text: s.into() } +} + +fn assistant(content: Vec, secs: i64, stop: StopReason) -> Message { + Message { + model: Some("claude-opus-4-8".into()), + stop_reason: Some(stop), + usage: Some(Usage { + input_tokens: 1000 + u64::try_from(secs).unwrap(), + output_tokens: 100, + cache_read_input_tokens: Some(900), + cache_creation_input_tokens: None, + }), + ..message(Role::Assistant, content, secs) + } +} + +fn result(id: &str, out: ToolOutput, is_error: bool) -> Block { + Block::ToolResult { + tool_use_id: id.into(), + content: out, + is_error, + } +} + +/// A session with every kind of thing a conversion can lose: signed thinking, +/// a Bash description, an MCP call with a structured result, an error result, +/// a slash command, an image, usage, stop reasons, a title, a branch. +// One session with every kind of thing a conversion can lose; length grows +// with the kinds covered. +#[allow(clippy::too_many_lines)] +fn rich(cwd: &str) -> txcript::Transcript { + let meta = Meta { + id: "11111111-2222-4333-8444-555555555555".into(), + timestamp: ts(0), + cwd: Some(cwd.into()), + git_branch: Some("feat/clock".into()), + title: Some("Fix the flaky clock test".into()), + cli_version: None, + model: Some("claude-opus-4-8".into()), + lineage: None, + }; + let body = vec![ + message( + Role::User, + vec![ + text("Fix the flaky test in clock.rs 🚀"), + Block::Image { + source: ImageSource { + source_type: "base64".into(), + media_type: "image/png".into(), + data: "iVBORw0KGgo=".into(), + }, + }, + ], + 1, + ), + assistant( + vec![ + Block::Thinking { + text: "It reads the wall clock.".into(), + signature: Some("sig-aaa".into()), + encrypted: None, + }, + text("Running it first."), + Block::ToolUse { + id: "c1".into(), + tool: Tool::Bash { + command: "cargo test clock".into(), + workdir: None, + timeout_ms: None, + description: Some("Run the clock tests".into()), + run_in_background: false, + }, + }, + ], + 2, + StopReason::ToolUse, + ), + message( + Role::User, + vec![result("c1", ToolOutput::Text("3 passed".into()), false)], + 3, + ), + assistant( + vec![Block::ToolUse { + id: "c2".into(), + tool: Tool::Raw { + tool_name: "mcp__docs__lookup".into(), + input: json!({"q": "clock", "filters": {"b": 1, "a": 2}}), + }, + }], + 4, + StopReason::ToolUse, + ), + message( + Role::User, + vec![result( + "c2", + ToolOutput::Json(json!([{"title": "Clock"}, {"title": "Time"}])), + false, + )], + 5, + ), + assistant( + vec![Block::ToolUse { + id: "c3".into(), + tool: Tool::Bash { + command: "false".into(), + workdir: None, + timeout_ms: None, + description: None, + run_in_background: false, + }, + }], + 6, + StopReason::ToolUse, + ), + message( + Role::User, + vec![result("c3", ToolOutput::Text("boom".into()), true)], + 7, + ), + message( + Role::User, + vec![Block::ToolUse { + id: "c4".into(), + tool: Tool::Command { + command: "/release".into(), + args: Some("patch".into()), + }, + }], + 8, + ), + message( + Role::User, + vec![result( + "c4", + ToolOutput::Text("released v1.2.3".into()), + false, + )], + 9, + ), + assistant(vec![text("All green.")], 10, StopReason::EndTurn), + ]; + txcript::Transcript::new(meta, body) +} + +/// Text only: nothing for a conversion to lose in the formats that keep text. +fn plain(cwd: &str) -> txcript::Transcript { + let meta = Meta { + id: "99999999-2222-4333-8444-555555555555".into(), + timestamp: ts(0), + cwd: Some(cwd.into()), + git_branch: None, + title: None, + cli_version: None, + model: None, + lineage: None, + }; + let body = vec![ + message(Role::User, vec![text("hello there")], 1), + message(Role::Assistant, vec![text("hi")], 2), + ]; + txcript::Transcript::new(meta, body) +} + +fn simple_text(t: &txcript::Transcript) -> String { + txcript::dispatch::render_from_common(HarnessId::Simple, t).unwrap() +} + +fn rich_doc(env: &Env) -> String { + env.document("rich.json", &simple_text(&rich(&env.project()))) +} + +fn plain_doc(env: &Env) -> String { + env.document("plain.json", &simple_text(&plain(&env.project()))) +} + +fn target<'a>(doc: &'a Value, harness: &str) -> &'a Value { + doc["targets"] + .as_array() + .unwrap() + .iter() + .find(|t| t["harness"] == harness) + .unwrap_or_else(|| panic!("no {harness} target in {doc}")) +} + +fn has_item(target: &Value, kind: &str, category: &str, path: Option<&str>) -> bool { + target["items"].as_array().unwrap().iter().any(|i| { + i["kind"] == kind && i["category"] == category && path.is_none_or(|p| i["path"] == p) + }) +} + +// ── check ────────────────────────────────────────────────────────────── + +#[test] +fn check_with_a_target_prints_a_report() { + let env = Env::new(); + let doc = rich_doc(&env); + let out = env.run(&["check", &doc, "--with", "codex"]); + assert!(out.status.success(), "{}", stderr(&out)); + let text = stdout(&out); + assert!( + text.starts_with("check 11111111 (simple, 10 messages) -> codex\n"), + "{text}" + ); + assert!(text.contains("CHANGED 1 tool name renamed /release -> _release")); + assert!( + text.contains("LOST 1 tool argument (Bash.description)"), + "{text}" + ); + assert!(text.contains("LOST 1 error flag"), "{text}"); + assert!(text.contains("LOST 1 thinking signature"), "{text}"); + assert!( + text.contains("LOST title Fix the flaky clock test"), + "{text}" + ); + assert!(text.contains("LOST model claude-opus-4-8"), "{text}"); + assert!(text.contains("structure"), "{text}"); + assert!(text.contains("codec-level"), "{text}"); + // Every class is labelled once, in order. + let at = |needle: &str| { + text.find(needle) + .unwrap_or_else(|| panic!("{needle}: {text}")) + }; + assert!(at(" content ") < at(" accounting ") && at(" accounting ") < at(" metadata ")); + // Privacy: no tool output body, and no image data. + assert!( + !text.contains("3 passed") && !text.contains("iVBOR"), + "{text}" + ); +} + +#[test] +fn check_json_is_versioned_and_complete() { + let env = Env::new(); + let doc = rich_doc(&env); + let out = env.run(&["check", &doc, "--with", "codex", "--json"]); + assert!(out.status.success(), "{}", stderr(&out)); + let value = json(&out); + assert_eq!(value["schema"], 1); + assert_eq!(value["source"]["harness"], "simple"); + assert_eq!( + value["source"]["id"], + "11111111-2222-4333-8444-555555555555" + ); + assert_eq!(value["source"]["messages"], 10); + let targets = value["targets"].as_array().unwrap(); + assert_eq!(targets.len(), 1); + let codex = &targets[0]; + assert_eq!(codex["harness"], "codex"); + assert_eq!(codex["error"], Value::Null); + assert_eq!(codex["severity"], "content"); + assert!(codex["messages_after"].as_u64().unwrap() >= 10); + assert!(has_item( + codex, + "changed", + "tool_name", + Some("/release -> _release") + )); + assert!(has_item( + codex, + "lost", + "tool_args", + Some("Bash.description") + )); + assert!(has_item(codex, "lost", "title", None)); + let counts = &codex["counts"]; + assert!(counts["lost"].as_u64().unwrap() >= 5, "{counts}"); + assert_eq!(counts["changed"], 1); + for item in codex["items"].as_array().unwrap() { + for key in [ + "kind", + "category", + "path", + "count", + "first_message", + "example", + ] { + assert!(item.get(key).is_some(), "{key} missing from {item}"); + } + assert!(item["count"].as_u64().unwrap() >= 1); + // Examples stay short. + assert!( + item["example"] + .as_str() + .is_none_or(|e| e.chars().count() <= 80) + ); + } + // 1-based, like `view` and `#range`. + let rename = codex["items"] + .as_array() + .unwrap() + .iter() + .find(|i| i["category"] == "tool_name") + .unwrap(); + assert_eq!(rename["first_message"], 8); +} + +#[test] +fn bare_check_compares_every_writable_target() { + let env = Env::new(); + let doc = rich_doc(&env); + let out = env.run(&["check", &doc]); + assert!(out.status.success(), "{}", stderr(&out)); + let text = stdout(&out); + assert!(text.contains("codec-level, 11 targets"), "{text}"); + assert!(text.contains("target content accounting metadata notes")); + let rows: Vec<&str> = text + .lines() + .filter_map(|l| l.split_whitespace().next()) + .collect(); + for harness in [ + "claude_code", + "codex", + "opencode", + "pi", + "campfire", + "cursor", + "cursor_desktop", + "grok", + "fx", + "antigravity", + "cowork", + ] { + assert!(rows.contains(&harness), "{harness} missing from {text}"); + } + for harness in [ + "grok_bot", + "hermes", + "amp", + "claude_chat", + "chatgpt", + "simple", + ] { + assert!(!rows.contains(&harness), "{harness} listed in {text}"); + } + let codex = text.lines().find(|l| l.starts_with("codex ")).unwrap(); + let cells: Vec<&str> = codex.split_whitespace().collect(); + assert_eq!(&cells[1..4], ["lost", "lost", "lost"], "{codex}"); + assert!(codex.contains("messages 10->"), "{codex}"); + assert!(text.contains("for details"), "{text}"); + assert!( + text.lines().all(|l| l == l.trim_end()), + "trailing whitespace in {text:?}" + ); +} + +#[test] +fn bare_check_json_lists_the_same_targets() { + let env = Env::new(); + let doc = rich_doc(&env); + let out = env.run(&["check", &doc, "--json"]); + let value = json(&out); + let names: Vec<&str> = value["targets"] + .as_array() + .unwrap() + .iter() + .map(|t| t["harness"].as_str().unwrap()) + .collect(); + assert_eq!(names.len(), 11, "{names:?}"); + assert!(!names.contains(&"grok_bot")); + assert_eq!(target(&value, "claude_code")["severity"], "none"); + assert_eq!(target(&value, "pi")["severity"], "content"); +} + +#[test] +fn grok_bot_is_assessed_when_named() { + let env = Env::new(); + let doc = rich_doc(&env); + let out = env.run(&["check", &doc, "--with", "grok_bot", "--json"]); + assert!(out.status.success(), "{}", stderr(&out)); + assert_eq!(target(&json(&out), "grok_bot")["error"], Value::Null); +} + +#[test] +fn unwritable_targets_report_per_target_without_aborting_the_rest() { + let env = Env::new(); + let doc = rich_doc(&env); + let out = env.run(&[ + "check", &doc, "--with", "hermes", "--with", "codex", "--with", "amp", "--json", + ]); + // A target that cannot be assessed fails the command, but not the report. + assert_eq!(out.status.code(), Some(1)); + let value = json(&out); + let hermes = target(&value, "hermes"); + assert!( + hermes["error"].as_str().unwrap().contains("not writable"), + "{hermes}" + ); + assert_eq!(hermes["items"].as_array().unwrap().len(), 0); + assert_eq!(hermes["severity"], Value::Null); + assert!(target(&value, "amp")["error"].is_string()); + let codex = target(&value, "codex"); + assert_eq!(codex["error"], Value::Null); + assert_eq!(codex["severity"], "content"); + + let human = env.run(&["check", &doc, "--with", "hermes", "--with", "codex"]); + assert_eq!(human.status.code(), Some(1)); + let text = stdout(&human); + assert!(text.contains("hermes is not writable"), "{text}"); + assert!(text.contains("-> codex"), "{text}"); +} + +#[test] +fn fail_on_sets_the_exit_code_from_the_severity() { + let env = Env::new(); + let rich_doc = rich_doc(&env); + let plain_doc = plain_doc(&env); + // (document, target, severity) and what each threshold does with it. + // codex on the rich session is content; claude_code is accounting only (a + // stop reason's representation); pi on the plain session with a title is + // metadata; claude_code on the plain one is lossless. + let titled = env.document( + "titled.json", + &simple_text(&{ + let mut t = plain(&env.project()); + t.meta.title = Some("A title".into()); + t + }), + ); + let aborted = env.document( + "aborted.json", + &simple_text(&{ + let mut t = plain(&env.project()); + t.body[1].stop_reason = Some(StopReason::Aborted); + t + }), + ); + let cases: [(&str, &str, [i32; 4]); 5] = [ + (&rich_doc, "codex", [0, 1, 1, 1]), + (&aborted, "claude_code", [0, 0, 1, 1]), + (&titled, "pi", [0, 0, 0, 1]), + (&plain_doc, "claude_code", [0, 0, 0, 0]), + (&rich_doc, "claude_code", [0, 0, 0, 0]), + ]; + for (doc, harness, expected) in cases { + for (threshold, want) in ["none", "content", "accounting", "metadata"] + .into_iter() + .zip(expected) + { + let out = env.run(&["check", doc, "--with", harness, "--fail-on", threshold]); + assert_eq!( + out.status.code(), + Some(want), + "{harness} --fail-on {threshold}: {}{}", + stdout(&out), + stderr(&out) + ); + // The report prints either way. + assert!(stdout(&out).contains(&format!("-> {harness}"))); + } + } + // Matrix mode: the worst of all targets decides. + let out = env.run(&["check", &rich_doc, "--fail-on", "content"]); + assert_eq!(out.status.code(), Some(1)); + let out = env.run(&["check", &rich_doc]); + assert_eq!(out.status.code(), Some(0)); + let out = env.run(&["check", &rich_doc, "--fail-on", "bogus"]); + assert_eq!(out.status.code(), Some(2)); +} + +#[test] +fn a_lossless_conversion_says_so() { + let env = Env::new(); + let doc = plain_doc(&env); + let out = env.run(&["check", &doc, "--with", "claude_code"]); + let text = stdout(&out); + assert!(text.contains("content ok: text, thinking"), "{text}"); + assert!(text.contains("accounting ok:"), "{text}"); + assert!(text.contains("metadata ok:"), "{text}"); + assert!( + !text.contains("LOST") && !text.contains("CHANGED"), + "{text}" + ); +} + +#[test] +fn detail_adds_message_numbers_and_longer_examples() { + let env = Env::new(); + let doc = rich_doc(&env); + let plain_out = stdout(&env.run(&["check", &doc, "--with", "codex"])); + assert!(!plain_out.contains("first at message")); + let detailed = stdout(&env.run(&["check", &doc, "--with", "codex", "--detail"])); + assert!(detailed.contains("(first at message 8)"), "{detailed}"); +} + +#[test] +fn help_explains_the_privacy_and_the_codec_level_caveat() { + let env = Env::new(); + let out = env.run(&["check", "--help"]); + // Help wraps to the terminal width; compare the words. + let text = stdout(&out) + .split_whitespace() + .collect::>() + .join(" "); + assert!(text.contains("codec-level"), "{text}"); + assert!(text.contains("never a tool result or image"), "{text}"); + assert!(text.contains("--detail")); + assert!(text.contains("--fail-on")); + let long = stdout(&env.run(&["continue", "--help"])); + for flag in ["--quiet", "--strict", "--dry-run"] { + assert!(long.contains(flag), "{flag} missing from continue --help"); + } + let crop = stdout(&env.run(&["crop", "--help"])); + assert!(crop.contains("--quiet") && crop.contains("--strict")); +} + +// ── sources ──────────────────────────────────────────────────────────── + +#[test] +fn check_reads_a_simple_document_from_stdin() { + let env = Env::new(); + let text = simple_text(&rich(&env.project())); + let out = env.run_stdin(&["check", "-", "--with", "codex", "--json"], &text); + assert!(out.status.success(), "{}", stderr(&out)); + let value = json(&out); + assert_eq!(value["source"]["messages"], 10); + assert!(has_item( + target(&value, "codex"), + "changed", + "tool_name", + None + )); +} + +#[test] +fn a_range_reports_exactly_what_would_be_written() { + let env = Env::new(); + let doc = rich_doc(&env); + // Messages 1-3: no slash command, so no tool rename. + let out = env.run(&["check", &format!("{doc}#1-3"), "--with", "codex", "--json"]); + assert!(out.status.success(), "{}", stderr(&out)); + let value = json(&out); + assert_eq!(value["source"]["messages"], 3); + assert!(!has_item( + target(&value, "codex"), + "changed", + "tool_name", + None + )); + // The slash command alone is message 8. + let out = env.run(&["check", &format!("{doc}#8-9"), "--with", "codex", "--json"]); + let value = json(&out); + assert_eq!(value["source"]["messages"], 2); + assert!(has_item( + target(&value, "codex"), + "changed", + "tool_name", + None + )); + // A range that cuts a call from its result fails as `continue` does. + let out = env.run(&["check", &format!("{doc}#2"), "--with", "codex"]); + assert_eq!(out.status.code(), Some(1)); + assert!( + stderr(&out).contains("cuts a tool call away from its result"), + "{}", + stderr(&out) + ); + let out = env.run(&["check", &format!("{doc}#40"), "--with", "codex"]); + assert!(stderr(&out).contains("out of bounds"), "{}", stderr(&out)); +} + +#[test] +fn a_missing_source_is_an_error_like_continue() { + let env = Env::new(); + let out = env.run(&["check", "nothing-here", "--from", "codex"]); + assert_eq!(out.status.code(), Some(1)); + assert_eq!( + stderr(&out), + "error: no codex session matches `nothing-here` (try `txcript list`)\n" + ); + let doc = rich_doc(&env); + let out = env.run(&["check", &doc, "--from", "codex"]); + assert!(stderr(&out).contains("--from scopes the search for a local session")); +} + +#[test] +fn an_empty_session_warns() { + let env = Env::new(); + let mut t = plain(&env.project()); + t.body.clear(); + let doc = env.document("empty.json", &simple_text(&t)); + let out = env.run(&["check", &doc, "--with", "claude_code"]); + assert!(out.status.success(), "{}", stderr(&out)); + assert!(stdout(&out).contains("0 messages"), "{}", stdout(&out)); + let out = env.run(&["check", &doc, "--json"]); + assert!(out.status.success()); + assert_eq!(json(&out)["source"]["messages"], 0); +} + +#[test] +fn a_stored_session_is_checked_by_id() { + let env = Env::new(); + let doc = rich_doc(&env); + let written = env.run(&[ + "continue", + &doc, + "--with", + "claude_code", + "--no-resume", + "--quiet", + ]); + assert!(written.status.success(), "{}", stderr(&written)); + let out = env.run(&[ + "check", + "11111111", + "--from", + "claude_code", + "--with", + "pi", + "--json", + ]); + // The store re-ids the copy, so look it up by prefix of what was written. + let listed = stdout(&env.run(&["list", "--from", "claude_code"])); + let id = listed + .split_whitespace() + .find(|w| w.len() == 36 && w.matches('-').count() == 4) + .unwrap() + .to_string(); + let out2 = env.run(&[ + "check", + &id[..8], + "--from", + "claude_code", + "--with", + "pi", + "--json", + ]); + assert!(out2.status.success(), "{}{}", stderr(&out), stderr(&out2)); + let value = json(&out2); + assert_eq!(value["source"]["harness"], "claude_code"); + assert!(has_item(target(&value, "pi"), "lost", "title", None)); +} + +// ── nothing is written ───────────────────────────────────────────────── + +#[test] +fn check_and_dry_run_write_nothing() { + let env = Env::new(); + let doc = rich_doc(&env); + let before = env.snapshot(); + for args in [ + vec!["check", &doc], + vec!["check", &doc, "--with", "codex", "--detail"], + vec!["check", &doc, "--json"], + vec!["continue", &doc, "--with", "codex", "--dry-run"], + vec![ + "continue", + &doc, + "--with", + "cursor", + "--dry-run", + "--strict", + ], + vec![ + "continue", + &doc, + "--with", + "pi", + "--dry-run", + "--out", + "out-dir", + ], + ] { + let out = env.run(&args); + assert!( + out.status.success() || args.contains(&"--strict"), + "{args:?}: {}", + stderr(&out) + ); + assert_eq!(env.snapshot(), before, "{args:?} wrote something"); + } + assert!(!env.path().join("out-dir").exists()); +} + +#[test] +fn dry_run_is_check_for_the_same_target() { + let env = Env::new(); + let doc = rich_doc(&env); + let check = env.run(&["check", &doc, "--with", "codex"]); + let dry = env.run(&["continue", &doc, "--with", "codex", "--dry-run"]); + assert_eq!(stdout(&check), stdout(&dry)); + assert!(dry.status.success()); + assert_eq!(stderr(&dry), ""); + // `--strict` is `--fail-on content`. + let strict = env.run(&["continue", &doc, "--with", "codex", "--dry-run", "--strict"]); + assert_eq!(strict.status.code(), Some(1)); + assert_eq!(stdout(&strict), stdout(&check)); + let lossless = env.run(&[ + "continue", + &plain_doc(&env), + "--with", + "claude_code", + "--dry-run", + "--strict", + ]); + assert!(lossless.status.success()); + // A document has no harness of its own, so the target is required. + let out = env.run(&["continue", &doc, "--dry-run"]); + assert_eq!(out.status.code(), Some(1)); + assert!( + stderr(&out).contains("pass --with "), + "{}", + stderr(&out) + ); +} + +#[test] +fn dry_run_of_an_in_place_resume_says_nothing_is_converted() { + let env = Env::new(); + let doc = plain_doc(&env); + let written = env.run(&[ + "continue", + &doc, + "--with", + "claude_code", + "--no-resume", + "--quiet", + ]); + assert!(written.status.success(), "{}", stderr(&written)); + let listed = stdout(&env.run(&["list", "--from", "claude_code"])); + let id = listed + .split_whitespace() + .find(|w| w.len() == 36 && w.matches('-').count() == 4) + .unwrap(); + let before = env.snapshot(); + let out = env.run(&["continue", id, "--from", "claude_code", "--dry-run"]); + assert!(out.status.success(), "{}", stderr(&out)); + assert!(stdout(&out).contains("resumes the session in place; nothing is converted")); + // With a range the copy is rewritten, so there is something to report. + let out = env.run(&[ + "continue", + &format!("{id}#1-2"), + "--from", + "claude_code", + "--dry-run", + ]); + assert!(stdout(&out).contains("-> claude_code"), "{}", stdout(&out)); + assert_eq!(env.snapshot(), before); +} + +// ── the gate on continue ─────────────────────────────────────────────── + +/// `continue --with codex --no-resume` on `doc`: output with the per-run ids +/// and paths smoothed out. +fn normalized(out: &Output) -> String { + stdout(out) + .lines() + .map(|l| { + if let Some(rest) = l.strip_prefix("simple → codex ") { + assert!( + Path::new(rest).extension().is_some_and(|e| e == "jsonl") + && rest.contains("rollout-"), + "unexpected location in {l}" + ); + "simple → codex ".to_string() + } else if l.starts_with(" resume with: codex resume ") { + " resume with: codex resume ".to_string() + } else { + l.to_string() + } + }) + .collect::>() + .join("\n") +} + +#[test] +fn continue_notes_the_loss_once_on_stderr_and_leaves_stdout_alone() { + let env = Env::new(); + let doc = rich_doc(&env); + let noisy = env.run(&["continue", &doc, "--with", "codex", "--no-resume"]); + assert!(noisy.status.success(), "{}", stderr(&noisy)); + let err = stderr(&noisy); + assert_eq!(err.lines().count(), 1, "{err}"); + assert!(err.starts_with("note: lossy conversion: "), "{err}"); + // Content first; what does not fit is counted. + assert!( + err.contains("1 tool name renamed, 1 tool argument"), + "{err}" + ); + assert!(err.contains(", and 2 more; run "), "{err}"); + assert!( + err.trim_end().ends_with(&format!( + "run 'txcript check {doc} --with codex' for details" + )), + "{err}" + ); + + // `--quiet` removes the note and nothing else: stdout is byte-for-byte the + // pre-existing output format either way. + let quiet = env.run(&[ + "continue", + &doc, + "--with", + "codex", + "--no-resume", + "--quiet", + ]); + assert!(quiet.status.success()); + assert_eq!(stderr(&quiet), ""); + assert_eq!(normalized(&noisy), normalized(&quiet)); + assert_eq!( + normalized(&noisy), + "simple → codex \n resume with: codex resume " + ); + // The note is for stderr only, and appears whether or not stderr is a + // terminal (it never is here). + assert!(!stdout(&noisy).contains("lossy")); +} + +#[test] +fn the_note_names_only_the_chosen_target() { + let env = Env::new(); + let doc = rich_doc(&env); + let out = env.run(&["continue", &doc, "--with", "fx", "--no-resume"]); + let err = stderr(&out); + assert!(err.contains("--with fx"), "{err}"); + assert!(!err.contains("codex"), "{err}"); +} + +#[test] +fn a_lossless_conversion_prints_no_note() { + let env = Env::new(); + let doc = plain_doc(&env); + let out = env.run(&["continue", &doc, "--with", "claude_code", "--no-resume"]); + assert!(out.status.success(), "{}", stderr(&out)); + assert_eq!(stderr(&out), ""); +} + +#[test] +fn strict_refuses_before_writing_anything() { + let env = Env::new(); + let doc = rich_doc(&env); + let before = env.snapshot(); + let out = env.run(&[ + "continue", + &doc, + "--with", + "codex", + "--no-resume", + "--strict", + ]); + assert_eq!(out.status.code(), Some(1)); + assert_eq!(stdout(&out), "", "nothing may be reported as written"); + let err = stderr(&out); + assert!( + err.starts_with("error: refusing to convert to codex with --strict"), + "{err}" + ); + assert!(err.contains("nothing was written"), "{err}"); + assert!(err.contains("tool name"), "{err}"); + assert!( + err.contains(&format!("run 'txcript check {doc} --with codex'")), + "{err}" + ); + assert_eq!(env.snapshot(), before, "strict wrote something"); + // `--out` is a write too. + let out = env.run(&[ + "continue", &doc, "--with", "codex", "--out", "exported", "--strict", + ]); + assert_eq!(out.status.code(), Some(1)); + assert_eq!(env.snapshot(), before); + assert!(!env.path().join("exported").exists()); +} + +#[test] +fn strict_lets_through_what_only_loses_accounting_or_metadata() { + let env = Env::new(); + let doc = plain_doc(&env); + let mut t = plain(&env.project()); + t.meta.title = Some("A title".into()); + let titled = env.document("titled.json", &simple_text(&t)); + let out = env.run(&[ + "continue", + &titled, + "--with", + "pi", + "--no-resume", + "--strict", + ]); + assert!(out.status.success(), "{}", stderr(&out)); + assert!( + stderr(&out).contains("lossy conversion: title"), + "{}", + stderr(&out) + ); + let out = env.run(&[ + "continue", + &doc, + "--with", + "claude_code", + "--no-resume", + "--strict", + ]); + assert!(out.status.success(), "{}", stderr(&out)); +} + +#[test] +fn strict_wins_over_quiet() { + let env = Env::new(); + let doc = rich_doc(&env); + let out = env.run(&[ + "continue", + &doc, + "--with", + "codex", + "--no-resume", + "--strict", + "--quiet", + ]); + assert_eq!(out.status.code(), Some(1)); + assert!( + stderr(&out).contains("refusing to convert"), + "{}", + stderr(&out) + ); +} + +#[test] +fn an_unwritable_target_keeps_its_existing_error() { + let env = Env::new(); + let doc = rich_doc(&env); + let out = env.run(&["continue", &doc, "--with", "hermes", "--no-resume"]); + assert_eq!(out.status.code(), Some(1)); + let err = stderr(&out); + assert!(err.contains("Hermes state.db is read-only"), "{err}"); + assert!(!err.contains("lossy"), "{err}"); +} + +#[test] +fn a_ranged_continue_is_measured_on_the_range() { + let env = Env::new(); + let doc = rich_doc(&env); + // The slash command is outside messages 1-3, so strict passes there. + let out = env.run(&[ + "continue", + &format!("{doc}#1-3"), + "--with", + "pi", + "--no-resume", + "--quiet", + "--strict", + ]); + assert!(out.status.success(), "{}", stderr(&out)); +} + +#[test] +fn a_broken_pipe_is_not_a_panic() { + let env = Env::new(); + let doc = rich_doc(&env); + let mut child = env + .command() + .args(["check", &doc, "--json"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + drop(child.stdout.take()); + let out = child.wait_with_output().unwrap(); + assert!(!stderr(&out).contains("panicked"), "{}", stderr(&out)); +} + +// ── prediction against reality ───────────────────────────────────────── + +/// What a session has, counted the way a reader of the exported document +/// would count it. Independent of the comparator under test. +fn facets(doc: &Value) -> BTreeMap { + let mut f: BTreeMap = BTreeMap::new(); + let mut add = |key: &str| *f.entry(key.to_string()).or_insert(0) += 1; + for message in doc["messages"].as_array().unwrap() { + for key in ["usage", "stop_reason", "model"] { + if message.get(key).is_some_and(|v| !v.is_null()) { + add(if key == "model" { "message_model" } else { key }); + } + } + let Some(blocks) = message["content"].as_array() else { + continue; + }; + for block in blocks { + match block["type"].as_str().unwrap() { + "thinking" if block.get("signature").is_some_and(|s| !s.is_null()) => { + add("thinking_signature"); + } + "tool_use" => { + if block["input"].get("description").is_some() { + add("tool_description"); + } + add(&format!("tool:{}", block["name"].as_str().unwrap())); + } + "tool_result" if block["is_error"] == true => add("is_error"), + "image" => add("image"), + _ => {} + } + } + } + for key in ["title", "model", "git_branch"] { + if doc.get(key).is_some_and(|v| !v.is_null()) { + add(&format!("meta:{key}")); + } + } + f +} + +/// Session ids the store lists for a harness. +fn ids(env: &Env, harness: &str) -> Vec { + stdout(&env.run(&["list", "--from", harness])) + .split_whitespace() + .filter(|w| w.len() == 36 && w.matches('-').count() == 4) + .map(String::from) + .collect() +} + +/// Accounting, thinking, and metadata counts that fell between two sessions. +fn really_lost(was: &BTreeMap, now: &BTreeMap) -> BTreeMap { + let mut real = BTreeMap::new(); + for (key, count) in was { + let kept = now.get(key).copied().unwrap_or(0); + if !key.starts_with("tool:") && *count > kept { + real.insert(key.clone(), count - kept); + } + } + real +} + +/// The same counts, as the report claims them lost. +fn claimed_lost(predicted: &Value) -> BTreeMap { + let mut claimed = BTreeMap::new(); + for item in predicted["items"].as_array().unwrap() { + if item["kind"] != "lost" { + continue; + } + let count = i64::try_from(item["count"].as_u64().unwrap()).unwrap(); + let category = item["category"].as_str().unwrap(); + let key = match (category, item["path"].as_str()) { + ("usage", None) => "usage", + ("stop_reason", _) => "stop_reason", + ("message_model", _) => "message_model", + ("thinking_signature", _) => "thinking_signature", + ("is_error", _) => "is_error", + ("image", _) => "image", + ("tool_args", Some("Bash.description")) => "tool_description", + ("title", _) => "meta:title", + ("model", _) => "meta:model", + ("git_branch", _) => "meta:git_branch", + _ => continue, + }; + *claimed.entry(key.to_string()).or_insert(0) += count; + } + claimed +} + +/// Convert for real into `harness`, read the copy back as a Simple document, +/// and hold what `check` predicted against what changed. +fn assert_prediction_matches_reality(harness: &str) { + let env = Env::new(); + let doc = rich_doc(&env); + let source: Value = serde_json::from_str(&fs::read_to_string(&doc).unwrap()).unwrap(); + + let report = json(&env.run(&["check", &doc, "--with", harness, "--json"])); + let predicted = target(&report, harness); + assert_eq!(predicted["error"], Value::Null, "{harness}: {predicted}"); + + let before = ids(&env, harness); + let written = env.run(&[ + "continue", + &doc, + "--with", + harness, + "--no-resume", + "--quiet", + ]); + assert!(written.status.success(), "{harness}: {}", stderr(&written)); + let new: Vec = ids(&env, harness) + .into_iter() + .filter(|id| !before.contains(id)) + .collect(); + assert_eq!(new.len(), 1, "{harness}: {new:?}"); + let exported = env.run(&["export", &new[0], "--from", harness]); + assert!( + exported.status.success(), + "{harness}: {}", + stderr(&exported) + ); + let copy: Value = serde_json::from_slice(&exported.stdout).unwrap(); + + // What really disappeared, against what the report says was lost. + let (was, now) = (facets(&source), facets(©)); + let real = really_lost(&was, &now); + let claimed = claimed_lost(predicted); + assert_eq!( + claimed, real, + "{harness}: the report claims {claimed:?} lost but {real:?} really was\n{predicted}" + ); + + // Every claimed rename really happened. + let renames: Vec<&str> = predicted["items"] + .as_array() + .unwrap() + .iter() + .filter(|i| i["category"] == "tool_name") + .map(|i| i["path"].as_str().unwrap()) + .collect(); + for path in &renames { + let (old, new) = path.split_once(" -> ").unwrap(); + assert_eq!( + now.get(&format!("tool:{new}")).copied(), + Some(1), + "{harness}: {path}" + ); + assert_eq!(now.get(&format!("tool:{old}")), None, "{harness}: {path}"); + } + // With none claimed, every tool call that is gone is claimed as lost. + if renames.is_empty() { + let gone: i64 = was + .iter() + .filter(|(k, _)| k.starts_with("tool:")) + .map(|(k, v)| (v - now.get(k).copied().unwrap_or(0)).max(0)) + .sum(); + let lost_calls: i64 = predicted["items"] + .as_array() + .unwrap() + .iter() + .filter(|i| i["kind"] == "lost" && i["category"] == "tool_call") + .map(|i| i64::try_from(i["count"].as_u64().unwrap()).unwrap()) + .sum(); + assert_eq!( + gone, lost_calls, + "{harness}: tool calls gone against claimed" + ); + } + + // The message count is exact. + assert_eq!( + copy["messages"].as_array().unwrap().len(), + usize::try_from(predicted["messages_after"].as_u64().unwrap()).unwrap(), + "{harness}: messages after" + ); +} + +#[test] +fn prediction_matches_reality_for_claude_code() { + assert_prediction_matches_reality("claude_code"); +} + +#[test] +fn prediction_matches_reality_for_codex() { + assert_prediction_matches_reality("codex"); +} + +#[test] +fn prediction_matches_reality_for_pi() { + assert_prediction_matches_reality("pi"); +} + +#[test] +fn prediction_matches_reality_for_campfire() { + assert_prediction_matches_reality("campfire"); +} + +#[test] +fn prediction_matches_reality_for_cursor() { + assert_prediction_matches_reality("cursor"); +} + +#[test] +fn prediction_matches_reality_for_cursor_desktop() { + assert_prediction_matches_reality("cursor_desktop"); +} + +#[test] +fn prediction_matches_reality_for_grok() { + assert_prediction_matches_reality("grok"); +} + +#[test] +fn prediction_matches_reality_for_fx() { + assert_prediction_matches_reality("fx"); +} + +#[test] +fn prediction_matches_reality_for_antigravity() { + assert_prediction_matches_reality("antigravity"); +} diff --git a/docs/formats/README.md b/docs/formats/README.md index 633fdde..d7f369d 100644 --- a/docs/formats/README.md +++ b/docs/formats/README.md @@ -57,3 +57,23 @@ than none. | [pi.md](pi.md) | pi | `src/harness/pi.rs` | | [campfire.md](campfire.md) | Campfire (embeds pi) | `src/harness/campfire.rs` | | [simple.md](simple.md) | Simple (txcript's own interchange format) | `src/harness/simple.rs` | + +## Conversion fidelity + +What converting *into* each harness loses, measured by `txcript check` (see [Fidelity](../usage.md#fidelity)) on one synthetic session that has every kind of block: signed and unsigned thinking, a Bash call with a description, an edit, an MCP call with a structured result, an error result, a slash command, an image, token usage, stop reasons, per-message models, a title, a git branch. Each cell names the categories that were lost or changed (`ok`: nothing). Items the target only made up, and message timestamps, are not listed. + +| Target | Content | Accounting | Metadata | +| --- | --- | --- | --- | +| claude_code | ok | changed: stop_reason | ok | +| codex | lost: tool_args, is_error; changed: tool_name | lost: thinking_signature, usage, stop_reason | lost: title, model | +| opencode | lost: tool_call, tool_result | lost: thinking_signature; changed: stop_reason | lost: git_branch | +| pi | lost: tool_call | lost: thinking_signature | lost: title, model, git_branch | +| campfire | lost: tool_call | lost: thinking_signature | lost: title, model, git_branch | +| cursor | lost: tool_call | lost: usage, stop_reason | lost: git_branch | +| cursor_desktop | lost: tool_call, tool_args, tool_result, image | lost: usage, stop_reason | lost: git_branch | +| grok | lost: tool_call | lost: thinking_signature, usage, stop_reason | ok | +| fx | lost: tool_call, tool_args | lost: usage, stop_reason | lost: git_branch | +| antigravity | lost: tool_call, tool_args, tool_result | lost: message_model | lost: model; changed: title | +| cowork | ok | changed: stop_reason | ok | + +The `tool_call` and `tool_result` losses are the slash command, which several formats have no way to record; `tool_name` is it written as `_release`. Hermes, Amp, and Simple can be rendered but not written by `continue`, and `grok_bot` mints a live agent, so `check` leaves them out of its default table. The measurement is codec-level: it does not include what a store adds, such as artifact files. It is a regression oracle as well as documentation: `tests/integration/fidelity.rs` fails when the table drifts from the codecs, and prints the rows to paste. diff --git a/docs/usage.md b/docs/usage.md index c1c71cc..347091e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -17,11 +17,21 @@ txcript continue [#range] # continue , then launch its harnes [--from ] # scope the id lookup to one harness [--out ] # write under ; implies --no-resume [--no-resume] # write the session but don't launch + [--quiet] # no stderr note about what a conversion loses + [--strict] # refuse to convert if content would be lost + [--dry-run] # report what it would cost; write nothing txcript continue [#range] # continue a Simple document instead: --with [...] # a file, or stdin (`-`), from any agent +txcript check [#range] # what would converting lose? writes nothing + [--with ]... # a full report per harness (default: a table + # of every harness `continue` can write) + [--from ] # scope the id lookup to one harness + [--json] [--detail] # machine-readable; longer examples + [--fail-on ] # exit 1 at or above this severity txcript crop [#range] # interactively cut messages and save a copy [--with ] # optionally convert the cropped copy [--from ] # scope the source lookup + [--quiet] [--strict] # as for `continue` txcript view [#range] # view a session; compact text when piped [--from ] # scope the id lookup to one harness [--no-pager] # print the terminal view directly @@ -57,6 +67,109 @@ txcript continue ./session.json --with claude_code # on the other one The recorded working directory is kept when it exists on the importing machine and otherwise replaced by the directory `continue` runs in. `export` accepts the same `#range` suffix and `--from` scope as `view`. +### Fidelity + +Converting a session between harnesses can lose things: a target may have no place for the title, token usage, or a tool call's description, or may write a slash command under another name. `txcript check` shows what a conversion would cost, before you do it: + +```text +$ txcript check session.json --with codex +check 8d116ece (simple, 23 messages) -> codex + content ADDED 1 text block [artifact: report.md] /txcript-artifacts/18-2/report.md + CHANGED 1 tool name renamed /release -> _release + LOST 1 tool argument (Bash.description) Run tests + LOST 1 tool argument (Bash.timeout_ms) 60000 + LOST 1 tool argument (Edit.replace_all) true + CHANGED 1 tool argument (Write.content) # notes\n -> # notes + LOST 1 artifact report.md + LOST 1 error flag + accounting LOST 1 thinking signature + LOST 4 usage records + LOST 1 usage field (cache_creation_input_tokens) + LOST 8 stop reasons tool_use + ADDED 8 per-message models claude-opus-5 + metadata LOST title Session everything + LOST model claude-opus-5 + structure 23 messages become 30 (the target splits turns) + codec-level: measured on the target's session text; a store's own additions are not counted +``` + +Without `--with`, one row per target `continue` can write: + +```text +$ txcript check session.json +check 8d116ece (simple, 23 messages), codec-level, 11 targets +target content accounting metadata notes +claude_code ok changed ok messages 23->24, 1 added +codex lost lost lost messages 23->30, 9 added +opencode lost lost lost messages 23->26, 10 added +pi lost lost lost messages 23->22, 1 added +campfire lost lost lost messages 23->22, 1 added +cursor lost lost lost 2 added +cursor_desktop lost lost lost messages 23->25, 8 added +grok lost lost lost messages 23->24 +fx lost lost lost messages 23->17, 2 added +antigravity lost lost lost messages 23->21 +cowork ok changed ok messages 23->24, 1 added +run 'txcript check session.json --with ' for details +``` + +`check` takes the same sources as `continue`: a session id or title, a Simple document file, or `-` for stdin, with an optional `#range`, so the report covers exactly the messages that would be written. It never writes anything and never launches an agent. + +**What it measures.** The session is converted to the target's native session text and read back, and the two are compared. The comparison matches content, not messages, because targets split and merge turns: a session that becomes more messages in Codex is not a loss. Text is trimmed, JSON compares by value, so key order and spacing never count, and regenerated ids are ignored. Each difference is one of: + +- `LOST`: in the source, absent from the result. +- `CHANGED`: in both, but different: a renamed tool or tool argument, a rewritten argument, an edited text, a stop reason written another way, blocks in another order. +- `ADDED`: absent from the source, present in the result: a stop reason or model the target made up. Listed, but never counted toward severity. + +Severity is the worst class of loss or change in the report: + +| Severity | Covers | +| --- | --- | +| `content` | text, thinking, tool calls and arguments, results, images, artifacts, error flags, block order | +| `accounting` | usage, stop reasons, per-message models, thinking signatures | +| `metadata` | title, model, git branch, working directory, message timestamps, lineage | + +The measurement is **codec-level**: it covers what the target's format stores, not what a store adds around it. A real `continue` also gives the copy a fresh id and timestamp, re-homes a working directory that no longer exists, writes artifact files (compared here by name only), imports through the `opencode` binary, and writes Cursor's database; none of that is counted. Message timestamps are compared by the second. A copy made by a lossy conversion no longer has what was lost, so run `check` on the original with the final target rather than on each copy: a chain of conversions loses at least what its worst hop does. + +By default a report shows counts and short examples only: at most 80 characters, control characters replaced, and never the body of a tool result or the data of an image. `--detail` adds longer examples, a preview of changed tool results, and the first message each difference appears in. + +`continue --with ` and `crop --with ` print one line to stderr when the chosen target loses or changes anything, and nothing else changes: standard output is what it always was. + +```text +note: lossy conversion: 1 text block, 1 tool call, 3 tool arguments, 1 artifact, 8 usage records, 7 stop reasons, and 2 more; run 'txcript check session.json --with fx' for details +``` + +The note is printed whether or not stderr is a terminal. That matches the other status lines `continue` prints there (`resuming: …`), keeps the loss visible in CI logs, and costs a script nothing, since stdout is untouched. `--quiet` removes it. `--strict` is for scripts that must not lose content: it refuses before writing or launching anything, with a message and exit status 1, when text, thinking, tool calls or arguments, results, images, or artifacts would be lost or changed (`crop` checks when you save). Losing only accounting or metadata still prints the note; a conversion that only rewrites something, such as a stop reason written another way, says `conversion changes` instead of `lossy conversion`. Resuming a session in its own harness without a `#range` converts nothing, so none of this applies there. `continue --dry-run` runs `check` for the same arguments, with `--strict` meaning `--fail-on content`, and prints that it would resume in place when no conversion is involved. + +`check` exits 0, or 1 when `--fail-on` is set and any target's severity is at or above it, or when a target cannot be assessed (a harness `continue` cannot write, such as Hermes, Amp, or a live source, reports that under its own name without stopping the others). `--fail-on content` is the usual CI gate. + +With `--json`, `check` prints one document. Version 1 of the schema: + +```json +{ + "schema": 1, + "source": { "harness": "claude_code", "id": "0dc114bf-…", "messages": 81 }, + "targets": [ + { + "harness": "codex", + "error": null, + "messages_after": 125, + "severity": "content", + "items": [ + { "kind": "changed", "category": "tool_name", "path": "/release -> _release", + "count": 2, "first_message": 14, "example": null } + ], + "warnings": [], + "counts": { "lost": 87, "changed": 2, "synthesized": 44 } + } + ] +} +``` + +`kind` is `lost`, `changed`, or `synthesized`. `category` is one of `text`, `thinking`, `tool_call`, `tool_name`, `tool_args`, `tool_result`, `image`, `artifact`, `is_error`, `order`, `thinking_signature`, `usage`, `stop_reason`, `message_model`, `title`, `model`, `git_branch`, `cwd`, `timestamp`, `lineage`. `path` refines a category (`Bash.description`, `/release -> _release`, a usage field) and `first_message` is 1-based like `#range`; either, and `example`, can be `null`. A target that could not be assessed has `error` set and `null` for `messages_after`, `severity`, and `counts`. New fields may be added within a schema version; `schema` changes only when a field's meaning does. + +[Per-harness results](formats/README.md#conversion-fidelity) on a reference session are listed with the formats. Not covered yet, and tracked as follow-ups: counting malformed lines in a source, redacting secrets before a conversion, verifying a real write by reloading it, and estimating token cost. + ### Search ```sh @@ -156,6 +269,8 @@ let codex = convert::<_, codex::Codex>(&claude)?; codex::CodexStore::default_root().expect("home dir").save(&codex)?; // resumable on disk ``` +`dispatch::{parse_to_common, render_from_common}` pick the text codec by `HarnessId`. On top of them, `fidelity::assess(target, &common)` renders a transcript as a harness's text, reads it back, and reports what was lost, changed, or made up; no I/O, so it runs in WASM builds too (see [Fidelity](#fidelity)). + The canonical model is `Transcript`: `Meta` + `Vec`, where a `Message` holds typed `Block`s (`Text`, `Thinking`, `ToolUse`, `ToolResult`, `Image`) and a typed `Tool` enum. Crop a canonical transcript in memory without changing the source: diff --git a/src/dispatch.rs b/src/dispatch.rs new file mode 100644 index 0000000..512558c --- /dev/null +++ b/src/dispatch.rs @@ -0,0 +1,77 @@ +//! Text-level codec dispatch: parse a harness's native session text into the +//! canonical model, or render the canonical model back to native text, keyed +//! by a runtime [`HarnessId`]. +//! +//! No filesystem, no `SQLite`, no subprocess: this is the codec layer alone +//! (see [`TextCodec`]), so it is available on every target, WASM included. The +//! WASM bindings and [`fidelity`](crate::fidelity) both route through it. + +use crate::harness::{ + amp, antigravity, campfire, chatgpt, claude_chat, claude_code, codex, cowork, cowork_remote, + cursor, cursor_desktop, fx, grok, grok_bot, hermes, opencode, pi, simple, +}; +use crate::transcript::{Codec, Common, HarnessId, TextCodec, Transcript}; + +/// Parse `harness`'s native session text into the canonical model. +/// +/// # Errors +/// When the text is not that harness's session format. +pub fn parse_to_common(harness: HarnessId, text: &str) -> crate::Result> { + fn go(text: &str) -> crate::Result> { + H::to_common(&H::from_text(text)?) + } + match harness { + HarnessId::ClaudeCode => go::(text), + HarnessId::ClaudeChat => go::(text), + HarnessId::CoworkRemote => go::(text), + HarnessId::ChatGpt => go::(text), + HarnessId::Codex => go::(text), + HarnessId::OpenCode => go::(text), + HarnessId::Pi => go::(text), + HarnessId::Campfire => go::(text), + HarnessId::Cursor => go::(text), + HarnessId::CursorDesktop => go::(text), + HarnessId::Grok => go::(text), + HarnessId::GrokBot => go::(text), + HarnessId::Fx => go::(text), + HarnessId::Hermes => go::(text), + HarnessId::Amp => go::(text), + HarnessId::Antigravity => go::(text), + HarnessId::Simple => go::(text), + HarnessId::Cowork => go::(text), + } +} + +/// Render the canonical model as `harness`'s native session text. +/// +/// # Errors +/// When the harness cannot represent the transcript (the live read-only +/// sources, for one). +pub fn render_from_common( + harness: HarnessId, + common: &Transcript, +) -> crate::Result { + fn go(common: &Transcript) -> crate::Result { + H::to_text(&H::from_common(common)?) + } + match harness { + HarnessId::ClaudeCode => go::(common), + HarnessId::ClaudeChat => go::(common), + HarnessId::CoworkRemote => go::(common), + HarnessId::ChatGpt => go::(common), + HarnessId::Codex => go::(common), + HarnessId::OpenCode => go::(common), + HarnessId::Pi => go::(common), + HarnessId::Campfire => go::(common), + HarnessId::Cursor => go::(common), + HarnessId::CursorDesktop => go::(common), + HarnessId::Grok => go::(common), + HarnessId::GrokBot => go::(common), + HarnessId::Fx => go::(common), + HarnessId::Hermes => go::(common), + HarnessId::Amp => go::(common), + HarnessId::Antigravity => go::(common), + HarnessId::Simple => go::(common), + HarnessId::Cowork => go::(common), + } +} diff --git a/src/fidelity.rs b/src/fidelity.rs new file mode 100644 index 0000000..fa41d5f --- /dev/null +++ b/src/fidelity.rs @@ -0,0 +1,2602 @@ +//! Conversion fidelity: what a conversion into a harness would lose, before +//! doing it. +//! +//! [`assess`] renders a canonical transcript as the target harness's native +//! text, parses that text back, and compares the two. Everything the target's +//! codec fails to persist shows up as a [`Loss`]. The result is a +//! *codec-level* measurement: it covers exactly what the codec writes into the +//! harness's session text, and nothing a [`Store`](crate::Store) adds or +//! changes around it (artifacts materialized as files, a fresh id and +//! timestamp, a re-homed working directory, the `OpenCode` import binary, the +//! Cursor `SQLite` database). There is no I/O, so it runs everywhere the +//! codecs do, WASM included. +//! +//! # How the comparison works +//! +//! Targets split and merge messages, so messages are never compared one to +//! one. Each transcript is flattened to a stream of typed items (text, +//! thinking, tool calls, tool results, images, artifacts) and the streams are +//! matched by content. What matches is preserved; leftovers that resemble +//! each other are [`LossKind::Changed`] (a renamed tool, an edited text); the +//! rest are [`LossKind::Lost`] when only the source has them and +//! [`LossKind::Synthesized`] when only the round trip does. Per-message +//! accounting (usage, stop reason, model, timestamp) and session metadata are +//! compared the same way, field by field. +//! +//! Normalization keeps formatting out of the verdict: text is trimmed, JSON +//! (tool inputs, structured or JSON-looking tool results) compares by value so +//! key order and whitespace never count, a list of text parts equals its +//! joined text, and regenerated ids are ignored. +//! +//! # Severity +//! +//! [`Severity`] ranks the worst thing a conversion does. Only +//! [`LossKind::Lost`] and [`LossKind::Changed`] items raise it; synthesized +//! items are listed but never fail a check. +//! +//! - [`Severity::Content`]: text, thinking, tool calls and arguments, results, +//! images, artifacts, error flags, block order. +//! - [`Severity::Accounting`]: usage, stop reasons, per-message models, +//! thinking signatures. +//! - [`Severity::Metadata`]: title, model, git branch, working directory, +//! timestamps, lineage. + +use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::hash::{DefaultHasher, Hash, Hasher}; + +use serde::Serialize; +use serde_json::Value; + +use crate::common::{Artifact, ArtifactSource, Block, Role, StopReason, ToolOutput, Usage}; +use crate::dispatch::{parse_to_common, render_from_common}; +use crate::{Common, HarnessId, Result, Transcript}; + +/// The harnesses a canonical transcript can be rendered into: every codec +/// except the live read-only sources (Claude Chat, cloud Cowork, `ChatGPT`). +/// +/// Whether a harness can also be *written* to disk (`continue --with`) is a +/// store question this module does not answer: Simple, Hermes, and Amp have +/// codecs but no store to write into. +pub const TARGETS: [HarnessId; 15] = [ + HarnessId::ClaudeCode, + HarnessId::Codex, + HarnessId::OpenCode, + HarnessId::Pi, + HarnessId::Campfire, + HarnessId::Cursor, + HarnessId::CursorDesktop, + HarnessId::Grok, + HarnessId::GrokBot, + HarnessId::Fx, + HarnessId::Hermes, + HarnessId::Amp, + HarnessId::Antigravity, + HarnessId::Simple, + HarnessId::Cowork, +]; + +/// Longest example kept in a [`Loss`], in characters. +const EXAMPLE_CAP: usize = 80; +/// Longest example kept under [`Options::detail`]. +const DETAIL_CAP: usize = 240; +/// How many look-alike candidates one leftover is compared against. +const PAIR_CANDIDATES: usize = 64; + +/// How bad a conversion is, by the worst class of thing it loses or changes. +/// Ordered: `None < Metadata < Accounting < Content`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Severity { + /// Nothing is lost or changed. + None, + /// Only session metadata differs. + Metadata, + /// Token accounting, stop reasons, or per-message models differ. + Accounting, + /// The conversation itself differs. + Content, +} + +impl Severity { + /// The stable lowercase name, as used in JSON and `--fail-on`. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Severity::None => "none", + Severity::Metadata => "metadata", + Severity::Accounting => "accounting", + Severity::Content => "content", + } + } +} + +/// What happened to an item on the way through the target. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum LossKind { + /// Present in the source, absent after the round trip. + Lost, + /// Present in both, but different. + Changed, + /// Absent in the source, present after the round trip. + Synthesized, +} + +/// What part of a transcript an item is about. Declared content first, then +/// accounting, then metadata; items sort in this order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Category { + /// Message text. + Text, + /// Thinking text. + Thinking, + /// A tool call that does not survive at all. + ToolCall, + /// A tool call that survives under another name. + ToolName, + /// A tool call that survives with different arguments. + ToolArgs, + /// A tool result. + ToolResult, + /// An inline image. + Image, + /// A named file. + Artifact, + /// A tool result's error flag. + IsError, + /// Blocks that survive but in a different order. + Order, + /// The opaque token that lets a harness replay thinking. + ThinkingSignature, + /// Per-message token accounting. + Usage, + /// Why a turn ended. + StopReason, + /// The model that produced a message. + MessageModel, + /// The session title. + Title, + /// The session's primary model. + Model, + /// The git branch. + GitBranch, + /// The working directory. + Cwd, + /// Session or message timestamps. + Timestamp, + /// How the session relates to another. + Lineage, +} + +impl Category { + /// The severity class this category belongs to. + #[must_use] + pub const fn severity(self) -> Severity { + match self { + Category::Text + | Category::Thinking + | Category::ToolCall + | Category::ToolName + | Category::ToolArgs + | Category::ToolResult + | Category::Image + | Category::Artifact + | Category::IsError + | Category::Order => Severity::Content, + Category::ThinkingSignature + | Category::Usage + | Category::StopReason + | Category::MessageModel => Severity::Accounting, + Category::Title + | Category::Model + | Category::GitBranch + | Category::Cwd + | Category::Timestamp + | Category::Lineage => Severity::Metadata, + } + } +} + +/// One kind of difference, with how many times it happened. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Loss { + /// Lost, changed, or synthesized. + pub kind: LossKind, + /// What it is about. + pub category: Category, + /// A finer label within the category: `Bash.description` for a dropped + /// tool argument, `/release -> _release` for a renamed tool, the usage + /// field that was dropped. + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, + /// How many items. + pub count: usize, + /// Index in the source of the first affected message. `None` for + /// synthesized items, which are not in the source. + #[serde(skip_serializing_if = "Option::is_none")] + pub first_at: Option, + /// A short sample: at most 80 characters, control characters replaced by + /// spaces, and never a tool result body or image data (unless + /// [`Options::detail`]). + #[serde(skip_serializing_if = "Option::is_none")] + pub example: Option, +} + +/// What converting a transcript into one harness costs. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Report { + /// The harness converted into. + pub target: HarnessId, + /// The worst class of loss; see the module docs. + pub severity: Severity, + /// Messages in the source. + pub messages_before: usize, + /// Messages after the round trip. Targets split and merge turns, so this + /// differing is not itself a loss. + pub messages_after: usize, + /// The differences, sorted by category, kind, then path. + pub items: Vec, + /// Things worth knowing that are not differences, such as an empty + /// session. + pub warnings: Vec, +} + +impl Report { + /// Total number of items of `kind` across every category. + #[must_use] + pub fn total(&self, kind: LossKind) -> usize { + self.items + .iter() + .filter(|item| item.kind == kind) + .map(|item| item.count) + .sum() + } +} + +/// Knobs for [`assess_with`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Options { + /// Keep longer examples, and a preview of tool results. Off by default so + /// a report stays safe to paste: counts and short samples only. + pub detail: bool, +} + +/// Assess converting `source` into `target`. +/// +/// # Errors +/// When the target's codec cannot render the transcript or cannot read its +/// own output back. +pub fn assess(target: HarnessId, source: &Transcript) -> Result { + assess_with(target, source, Options::default()) +} + +/// [`assess`] with options. +/// +/// # Errors +/// As [`assess`]. +pub fn assess_with( + target: HarnessId, + source: &Transcript, + options: Options, +) -> Result { + // Stores write artifacts out as files before the codec sees them; do the + // same on paper so the codec gets the shape it gets in real use. + let prepared = with_artifact_paths(source); + let text = render_from_common(target, &prepared)?; + let round_trip = parse_to_common(target, &text)?; + Ok(compare(target, source, &round_trip, options)) +} + +/// Whether an artifact's content travels in the transcript rather than in a +/// file. +fn held_inline(artifact: &Artifact) -> bool { + !matches!(artifact.source, ArtifactSource::Path { .. }) +} + +/// `source` with every inline artifact replaced by a file path, as the store +/// layer does before writing. Without such artifacts, `source` itself. +fn with_artifact_paths(source: &Transcript) -> Cow<'_, Transcript> { + let any = source + .body + .iter() + .flat_map(|m| &m.content) + .any(|b| matches!(b, Block::Artifact { artifact } if held_inline(artifact))); + if !any { + return Cow::Borrowed(source); + } + let mut prepared = source.clone(); + for (at, message) in prepared.body.iter_mut().enumerate() { + for (n, block) in message.content.iter_mut().enumerate() { + if let Block::Artifact { artifact } = block + && held_inline(artifact) + { + let media_type = match &artifact.source { + ArtifactSource::Text { media_type, .. } + | ArtifactSource::Base64 { media_type, .. } + | ArtifactSource::Path { media_type, .. } => media_type.clone(), + }; + let name = artifact.name.replace(['/', '\\'], "_"); + artifact.source = ArtifactSource::Path { + path: format!("/txcript-artifacts/{at}-{n}/{name}"), + media_type, + }; + } + } + } + Cow::Owned(prepared) +} + +/// Assess `source` against every harness in [`TARGETS`]. A target whose codec +/// fails is reported as an error under its own entry and never stops the +/// others. +#[must_use] +pub fn check_all(source: &Transcript) -> Vec<(HarnessId, Result)> { + TARGETS + .into_iter() + .map(|target| (target, assess(target, source))) + .collect() +} + +// ── comparison ───────────────────────────────────────────────────────── + +fn compare( + target: HarnessId, + source: &Transcript, + round_trip: &Transcript, + options: Options, +) -> Report { + let mut acc = Acc::new(options); + let src = flatten(source); + let rt = flatten(round_trip); + compare_content(&src, &rt, &mut acc); + compare_accounting(source, round_trip, &mut acc); + compare_meta(source, round_trip, &mut acc); + + let items: Vec = acc.items.into_values().collect(); + let severity = items + .iter() + .filter(|item| item.kind != LossKind::Synthesized) + .map(|item| item.category.severity()) + .max() + .unwrap_or(Severity::None); + let mut warnings = Vec::new(); + if source.body.is_empty() { + warnings.push("the session has 0 messages; there is nothing to lose".to_string()); + } + Report { + target, + severity, + messages_before: source.body.len(), + messages_after: round_trip.body.len(), + items, + warnings, + } +} + +/// Collects items, merging repeats of the same kind of difference. +struct Acc { + items: BTreeMap<(Category, LossKind, Option), Loss>, + cap: usize, + detail: bool, +} + +impl Acc { + fn new(options: Options) -> Acc { + Acc { + items: BTreeMap::new(), + cap: if options.detail { + DETAIL_CAP + } else { + EXAMPLE_CAP + }, + detail: options.detail, + } + } + + fn add( + &mut self, + kind: LossKind, + category: Category, + path: Option, + at: Option, + example: impl FnOnce() -> Option, + ) { + self.add_n(kind, category, path, at, 1, example); + } + + fn add_n( + &mut self, + kind: LossKind, + category: Category, + path: Option, + at: Option, + count: usize, + example: impl FnOnce() -> Option, + ) { + if count == 0 { + return; + } + let cap = self.cap; + let entry = self + .items + .entry((category, kind, path.clone())) + .or_insert_with(|| Loss { + kind, + category, + path, + count: 0, + first_at: at, + example: example().map(|e| clip(&e, cap)), + }); + entry.count += count; + entry.first_at = match (entry.first_at, at) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }; + } +} + +/// `s` on one line, at most `cap` characters, with an ellipsis when cut. +fn clip(s: &str, cap: usize) -> String { + let mut out = String::new(); + let mut chars = s + .trim() + .chars() + .map(|c| if c.is_control() { ' ' } else { c }); + for _ in 0..cap.saturating_sub(1) { + match chars.next() { + Some(c) => out.push(c), + None => return out, + } + } + match (chars.next(), chars.next()) { + (None, _) => {} + (Some(c), None) => out.push(c), + (Some(_), Some(_)) => out.push('…'), + } + out +} + +/// `before -> after`, each side cut to the stretch around where the two start +/// to differ, with line breaks and tabs shown rather than collapsed, so a +/// change in trailing whitespace is visible. +fn arrow(before: &str, after: &str, cap: usize) -> String { + let half = cap.saturating_sub(4) / 2; + let fits = |s: &str| s.chars().take(half + 1).count() <= half; + let skip = if fits(before) && fits(after) { + 0 + } else { + let shared = before + .chars() + .zip(after.chars()) + .take_while(|(x, y)| x == y) + .count(); + shared.saturating_sub(8) + }; + let side = |s: &str| { + let mut out: Vec = Vec::new(); + if skip > 0 { + out.push('…'); + } + for c in s.chars().skip(skip) { + match c { + '\n' => out.extend(['\\', 'n']), + '\t' => out.extend(['\\', 't']), + '\r' => out.extend(['\\', 'r']), + c if c.is_control() => out.push(' '), + c => out.push(c), + } + if out.len() > half { + out.truncate(half.saturating_sub(1)); + out.push('…'); + break; + } + } + out.into_iter().collect::() + }; + format!("{} -> {}", side(before), side(after)) +} + +// ── content stream ───────────────────────────────────────────────────── + +/// A tool result reduced to what it says. +enum Proj<'a> { + Text(Cow<'a, str>), + Json(Cow<'a, Value>), +} + +impl<'a> Proj<'a> { + fn into_text(self) -> Cow<'a, str> { + match self { + Proj::Text(s) => s, + Proj::Json(v) => Cow::Owned(v.to_string()), + } + } +} + +enum Body<'a> { + Text { + role: Role, + text: &'a str, + }, + Thinking { + text: &'a str, + signature: Option<&'a str>, + encrypted: Option<&'a str>, + }, + Call { + name: String, + input: Value, + }, + Result { + output: &'a ToolOutput, + is_error: bool, + }, + Image { + media_type: &'a str, + }, + Artifact { + name: &'a str, + }, +} + +/// One block of a flattened transcript. +struct Item<'a> { + /// Identity of the block's content, ids and formatting excluded. + key: u64, + /// The message the block came from. + at: usize, + body: Body<'a>, +} + +fn hasher(tag: u8) -> DefaultHasher { + let mut h = DefaultHasher::new(); + tag.hash(&mut h); + h +} + +/// Hash a JSON value by content: object keys in sorted order, so neither key +/// order nor whitespace in the source text affects it. +fn hash_value(v: &Value, h: &mut DefaultHasher) { + match v { + Value::Null => 0u8.hash(h), + Value::Bool(b) => (1u8, b).hash(h), + Value::Number(n) => { + 2u8.hash(h); + if let Some(u) = n.as_u64() { + (0u8, u).hash(h); + } else if let Some(i) = n.as_i64() { + (1u8, i).hash(h); + } else { + (2u8, n.as_f64().map(f64::to_bits)).hash(h); + } + } + Value::String(s) => (3u8, s).hash(h), + Value::Array(items) => { + (4u8, items.len()).hash(h); + for item in items { + hash_value(item, h); + } + } + Value::Object(map) => { + (5u8, map.len()).hash(h); + let mut keys: Vec<&String> = map.keys().collect(); + keys.sort(); + for key in keys { + key.hash(h); + hash_value(&map[key], h); + } + } + } +} + +/// The text of a result that is a list of plain text parts. +fn text_parts(parts: &[Value]) -> Option { + let mut texts = Vec::with_capacity(parts.len()); + for part in parts { + let map = part.as_object()?; + if map.get("type")?.as_str()? != "text" || map.keys().any(|k| k != "type" && k != "text") { + return None; + } + texts.push(map.get("text")?.as_str()?); + } + (!texts.is_empty()).then(|| texts.join("\n")) +} + +fn project(output: &ToolOutput) -> Proj<'_> { + /// Text that is JSON compares as that JSON, so re-spacing it is not a + /// change. `depth` bounds how many layers of JSON-in-a-string are peeled. + fn of_text(t: Cow<'_, str>, depth: u8) -> Proj<'_> { + if depth > 0 + && t.starts_with(['{', '[']) + && let Ok(v) = serde_json::from_str::(&t) + { + return of_value(Cow::Owned(v), depth - 1); + } + Proj::Text(t) + } + fn of_value(v: Cow<'_, Value>, depth: u8) -> Proj<'_> { + match &*v { + Value::String(s) => of_text(Cow::Owned(s.trim().to_string()), depth), + Value::Array(parts) => match text_parts(parts) { + Some(joined) => of_text(Cow::Owned(joined.trim().to_string()), depth), + None => Proj::Json(v), + }, + _ => Proj::Json(v), + } + } + match output { + ToolOutput::Text(s) => of_text(Cow::Borrowed(s.trim()), 3), + ToolOutput::Json(v) => of_value(Cow::Borrowed(v), 3), + } +} + +fn flatten(t: &Transcript) -> Vec> { + let mut out = Vec::new(); + for (at, message) in t.body.iter().enumerate() { + for block in &message.content { + let item = match block { + Block::Text { text } => { + let text = text.trim(); + if text.is_empty() { + continue; + } + let mut h = hasher(0); + (message.role == Role::User, text).hash(&mut h); + Item { + key: h.finish(), + at, + body: Body::Text { + role: message.role, + text, + }, + } + } + Block::Thinking { + text, + signature, + encrypted, + } => { + let text = text.trim(); + if text.is_empty() { + continue; + } + let mut h = hasher(1); + text.hash(&mut h); + Item { + key: h.finish(), + at, + body: Body::Thinking { + text, + signature: signature.as_deref(), + encrypted: encrypted.as_deref(), + }, + } + } + Block::ToolUse { tool, .. } => { + let (name, input) = tool.to_canonical(); + let mut h = hasher(2); + name.hash(&mut h); + hash_value(&input, &mut h); + Item { + key: h.finish(), + at, + body: Body::Call { name, input }, + } + } + Block::ToolResult { + content, is_error, .. + } => { + // Only the hash is kept: a result can be megabytes, and the + // projection is rebuilt for the few that need comparing. + let mut h = hasher(3); + match &project(content) { + Proj::Text(s) => (0u8, s.as_ref()).hash(&mut h), + Proj::Json(v) => { + 1u8.hash(&mut h); + hash_value(v, &mut h); + } + } + Item { + key: h.finish(), + at, + body: Body::Result { + output: content, + is_error: *is_error, + }, + } + } + Block::Image { source } => { + let mut h = hasher(4); + (&source.media_type, &source.data).hash(&mut h); + Item { + key: h.finish(), + at, + body: Body::Image { + media_type: &source.media_type, + }, + } + } + Block::Artifact { artifact } => Item { + key: artifact_key(artifact), + at, + body: Body::Artifact { + name: &artifact.name, + }, + }, + }; + out.push(item); + } + } + out +} + +/// An artifact's identity is its name: the store writes its content to a file +/// of its own, which the codec never carries, so content is not compared. +fn artifact_key(artifact: &Artifact) -> u64 { + let mut h = hasher(5); + artifact.name.hash(&mut h); + h.finish() +} + +/// Match equal keys one to one, in order. Returns the matched index pairs, +/// then the unmatched indices of each side. +fn match_keys(src: &[u64], rt: &[u64]) -> (Vec<(usize, usize)>, Vec, Vec) { + let mut index: HashMap> = HashMap::new(); + for (j, key) in rt.iter().enumerate() { + index.entry(*key).or_default().push_back(j); + } + let mut used = vec![false; rt.len()]; + let mut pairs = Vec::new(); + let mut src_left = Vec::new(); + for (i, key) in src.iter().enumerate() { + match index.get_mut(key).and_then(VecDeque::pop_front) { + Some(j) => { + used[j] = true; + pairs.push((i, j)); + } + None => src_left.push(i), + } + } + let rt_left = (0..rt.len()).filter(|j| !used[*j]).collect(); + (pairs, src_left, rt_left) +} + +/// The source positions of matched items that left the longest in-order run +/// behind: the blocks that moved. `pairs` is sorted by source position. +fn moved(pairs: &[(usize, usize)]) -> Vec { + let mut tails: Vec = Vec::new(); + let mut prev = vec![None; pairs.len()]; + for (k, &(_, j)) in pairs.iter().enumerate() { + let pos = tails.partition_point(|&t| pairs[t].1 < j); + prev[k] = pos.checked_sub(1).map(|p| tails[p]); + if pos == tails.len() { + tails.push(k); + } else { + tails[pos] = k; + } + } + let mut keep = vec![false; pairs.len()]; + let mut cursor = tails.last().copied(); + while let Some(k) = cursor { + keep[k] = true; + cursor = prev[k]; + } + pairs + .iter() + .zip(keep) + .filter(|(_, kept)| !kept) + .map(|(pair, _)| pair.0) + .collect() +} + +/// Whitespace-insensitive equality, without allocating. +fn same_words(a: &str, b: &str) -> bool { + a.split_whitespace().eq(b.split_whitespace()) +} + +/// The first few characters of `s` with whitespace collapsed: two texts can +/// only resemble each other when these agree. +fn lead(s: &str) -> u64 { + let mut h = DefaultHasher::new(); + let mut taken = 0; + for word in s.split_whitespace() { + for c in word.chars() { + if taken >= 6 { + return h.finish(); + } + c.hash(&mut h); + taken += 1; + } + ' '.hash(&mut h); + taken += 1; + } + h.finish() +} + +/// Whether two leftover texts are probably the same text, edited: equal but +/// for whitespace, or opening with the same eight characters. +fn related(a: &str, b: &str) -> bool { + same_words(a, b) || a.chars().zip(b.chars()).take_while(|(p, q)| p == q).count() >= 8 +} + +/// A leftover tool call's identity for pairing: its name and each argument. +fn call_keys(name: &str, input: &Value) -> Vec<(u8, u64)> { + let mut by_name = hasher(10); + name.hash(&mut by_name); + let mut by_input = hasher(11); + hash_value(input, &mut by_input); + vec![(0, by_name.finish()), (1, by_input.finish())] +} + +/// Whether two leftover calls are the same call, altered. +fn related_calls(a: (&str, &Value), b: (&str, &Value)) -> bool { + if a.1 == b.1 { + return true; + } + if a.0 != b.0 { + return false; + } + match (a.1.as_object(), b.1.as_object()) { + (Some(x), Some(y)) => x.iter().any(|(k, v)| y.get(k) == Some(v)), + _ => false, + } +} + +/// Pair leftover source items with leftover round-trip items that resemble +/// them. Candidates come from `keys` buckets; `related` has the last word. +/// Returns the pairs and what stayed unpaired on each side. +fn pair_up( + src_left: &[usize], + rt_left: &[usize], + src_keys: &dyn Fn(usize) -> Vec<(u8, u64)>, + rt_keys: &dyn Fn(usize) -> Vec<(u8, u64)>, + related: &dyn Fn(usize, usize) -> bool, +) -> (Vec<(usize, usize)>, Vec, Vec) { + let mut buckets: HashMap<(u8, u64), Vec> = HashMap::new(); + for &j in rt_left { + for key in rt_keys(j) { + buckets.entry(key).or_default().push(j); + } + } + let mut used: HashSet = HashSet::new(); + let mut cursors: HashMap<(u8, u64), usize> = HashMap::new(); + let mut pairs = Vec::new(); + let mut src_unpaired = Vec::new(); + for &i in src_left { + let mut found = None; + for key in src_keys(i) { + let Some(bucket) = buckets.get(&key) else { + continue; + }; + let cursor = cursors.entry(key).or_insert(0); + while *cursor < bucket.len() && used.contains(&bucket[*cursor]) { + *cursor += 1; + } + found = bucket[*cursor..] + .iter() + .filter(|j| !used.contains(j)) + .take(PAIR_CANDIDATES) + .find(|&&j| related(i, j)) + .copied(); + if found.is_some() { + break; + } + } + match found { + Some(j) => { + used.insert(j); + pairs.push((i, j)); + } + None => src_unpaired.push(i), + } + } + let rt_unpaired = rt_left + .iter() + .copied() + .filter(|j| !used.contains(j)) + .collect(); + (pairs, src_unpaired, rt_unpaired) +} + +fn category_of(body: &Body<'_>) -> Category { + match body { + Body::Text { .. } => Category::Text, + Body::Thinking { .. } => Category::Thinking, + Body::Call { .. } => Category::ToolCall, + Body::Result { .. } => Category::ToolResult, + Body::Image { .. } => Category::Image, + Body::Artifact { .. } => Category::Artifact, + } +} + +/// A short description of an item for an example. +fn describe(body: &Body<'_>, detail: bool) -> Option { + match body { + Body::Text { text, .. } | Body::Thinking { text, .. } => Some((*text).to_string()), + Body::Call { name, input } => Some(call_label(name, input)), + Body::Result { output, .. } => detail.then(|| project(output).into_text().into_owned()), + Body::Image { media_type } => Some((*media_type).to_string()), + Body::Artifact { name } => Some((*name).to_string()), + } +} + +/// `Bash: cargo test`: the tool and its most identifying argument. +fn call_label(name: &str, input: &Value) -> String { + const PRIMARY: [&str; 6] = ["command", "file_path", "path", "url", "pattern", "query"]; + let arg = input.as_object().and_then(|map| { + PRIMARY + .iter() + .find_map(|key| map.get(*key).and_then(Value::as_str)) + }); + match arg { + Some(arg) => format!("{name}: {arg}"), + None => name.to_string(), + } +} + +/// One round of leftover pairing for a kind of block: the pairs, and what +/// stayed unpaired on each side. +type Pairing = (Vec<(usize, usize)>, Vec, Vec); + +fn leftover_texts(src: &[Item<'_>], rt: &[Item<'_>], s: &[usize], r: &[usize]) -> Pairing { + fn text<'a>(items: &[Item<'a>], n: usize) -> &'a str { + match &items[n].body { + Body::Text { text, .. } | Body::Thinking { text, .. } => text, + _ => "", + } + } + let keys = |items: &[Item<'_>], n: usize| { + let tag = u8::from(matches!(items[n].body, Body::Thinking { .. })); + vec![(tag, lead(text(items, n)))] + }; + pair_up(s, r, &|i| keys(src, i), &|j| keys(rt, j), &|i, j| { + // A text never pairs with a thinking block. + keys(src, i) == keys(rt, j) && related(text(src, i), text(rt, j)) + }) +} + +fn leftover_calls(src: &[Item<'_>], rt: &[Item<'_>], s: &[usize], r: &[usize]) -> Pairing { + fn call<'i>(items: &'i [Item<'_>], n: usize) -> Option<(&'i str, &'i Value)> { + match &items[n].body { + Body::Call { name, input } => Some((name.as_str(), input)), + _ => None, + } + } + let keys = |items: &[Item<'_>], n: usize| { + call(items, n).map_or_else(Vec::new, |(name, input)| call_keys(name, input)) + }; + pair_up( + s, + r, + &|i| keys(src, i), + &|j| keys(rt, j), + &|i, j| match (call(src, i), call(rt, j)) { + (Some(a), Some(b)) => related_calls(a, b), + _ => false, + }, + ) +} + +fn leftover_results(src: &[Item<'_>], rt: &[Item<'_>], s: &[usize], r: &[usize]) -> Pairing { + /// What each leftover result says, worked out once. + fn texts<'a>(items: &[Item<'a>], idx: &[usize]) -> HashMap> { + idx.iter() + .filter_map(|&n| match &items[n].body { + Body::Result { output, .. } => Some((n, project(output).into_text())), + _ => None, + }) + .collect() + } + let (said_src, said_rt) = (texts(src, s), texts(rt, r)); + let key = |said: &HashMap>, n: usize| { + vec![(2, lead(said.get(&n).map_or("", AsRef::as_ref)))] + }; + pair_up( + s, + r, + &|i| key(&said_src, i), + &|j| key(&said_rt, j), + &|i, j| match (said_src.get(&i), said_rt.get(&j)) { + (Some(a), Some(b)) => related(a, b), + _ => false, + }, + ) +} + +/// Pair leftovers by a coarse label alone (an image's media type, an +/// artifact's name): same label, same item. +fn leftover_labeled( + src: &[Item<'_>], + rt: &[Item<'_>], + s: &[usize], + r: &[usize], + label: for<'a> fn(&'a Body<'_>) -> Option<&'a str>, +) -> Pairing { + let keys = |items: &[Item<'_>], n: usize| { + label(&items[n].body).map_or_else(Vec::new, |l| { + let mut h = hasher(14); + l.hash(&mut h); + vec![(5, h.finish())] + }) + }; + pair_up(s, r, &|i| keys(src, i), &|j| keys(rt, j), &|_, _| true) +} + +/// Blocks that matched by content may still differ in what rides along: +/// a thinking block's replay token, a result's error flag. +fn compare_matched(src: &[Item<'_>], rt: &[Item<'_>], matched: &[(usize, usize)], acc: &mut Acc) { + for &(i, j) in matched { + match (&src[i].body, &rt[j].body) { + (Body::Thinking { .. }, Body::Thinking { .. }) => { + compare_token(&src[i], &rt[j], acc); + } + (Body::Result { is_error: was, .. }, Body::Result { is_error: now, .. }) + if was != now => + { + if *was { + acc.add( + LossKind::Lost, + Category::IsError, + None, + Some(src[i].at), + || None, + ); + } else { + acc.add(LossKind::Synthesized, Category::IsError, None, None, || { + None + }); + } + } + _ => {} + } + } +} + +fn compare_content(src: &[Item<'_>], rt: &[Item<'_>], acc: &mut Acc) { + let src_keys: Vec = src.iter().map(|i| i.key).collect(); + let rt_keys: Vec = rt.iter().map(|i| i.key).collect(); + let (matched, src_left, rt_left) = match_keys(&src_keys, &rt_keys); + let (detail, cap) = (acc.detail, acc.cap); + + compare_matched(src, rt, &matched, acc); + + let out_of_order = moved(&matched); + if let Some(&first) = out_of_order.first() { + acc.add_n( + LossKind::Changed, + Category::Order, + None, + Some(src[first].at), + out_of_order.len(), + || None, + ); + } + + // Leftovers that resemble each other are one item changed; the rest are + // lost (source only) or synthesized (round trip only). + let split = |items: &[Item<'_>], idx: &[usize], category: Category| -> Vec { + idx.iter() + .copied() + .filter(|&n| category_of(&items[n].body) == category) + .collect() + }; + let by = |category: Category| { + ( + split(src, &src_left, category), + split(rt, &rt_left, category), + ) + }; + let mut unpaired_src: Vec = Vec::new(); + let mut unpaired_rt: Vec = Vec::new(); + let mut settle = |(pairs, su, ru): Pairing, on_pair: &mut dyn FnMut(usize, usize)| { + for (i, j) in pairs { + on_pair(i, j); + } + unpaired_src.extend(su); + unpaired_rt.extend(ru); + }; + + // Text and thinking pair separately from each other. + for category in [Category::Text, Category::Thinking] { + let (s, r) = by(category); + settle(leftover_texts(src, rt, &s, &r), &mut |i, j| { + changed_text(&src[i], &rt[j], acc); + }); + } + let (s, r) = by(Category::ToolCall); + settle(leftover_calls(src, rt, &s, &r), &mut |i, j| { + changed_call(&src[i], &rt[j], acc); + }); + let (s, r) = by(Category::ToolResult); + settle(leftover_results(src, rt, &s, &r), &mut |i, j| { + acc.add( + LossKind::Changed, + Category::ToolResult, + None, + Some(src[i].at), + || { + detail.then(|| { + arrow( + &describe_full(&src[i].body), + &describe_full(&rt[j].body), + cap, + ) + }) + }, + ); + }); + let (s, r) = by(Category::Image); + settle( + leftover_labeled(src, rt, &s, &r, |b| match b { + Body::Image { media_type } => Some(media_type), + _ => None, + }), + &mut |i, _| { + acc.add( + LossKind::Changed, + Category::Image, + None, + Some(src[i].at), + || describe(&src[i].body, false), + ); + }, + ); + let (s, r) = by(Category::Artifact); + settle( + leftover_labeled(src, rt, &s, &r, |b| match b { + Body::Artifact { name } => Some(name), + _ => None, + }), + &mut |i, _| { + acc.add( + LossKind::Changed, + Category::Artifact, + None, + Some(src[i].at), + || describe(&src[i].body, false), + ); + }, + ); + + emit_unpaired(src, rt, unpaired_src, unpaired_rt, acc); +} + +/// What no counterpart was found for: lost from the source, synthesized in +/// the round trip. +fn emit_unpaired( + src: &[Item<'_>], + rt: &[Item<'_>], + mut unpaired_src: Vec, + mut unpaired_rt: Vec, + acc: &mut Acc, +) { + let detail = acc.detail; + unpaired_src.sort_unstable(); + unpaired_rt.sort_unstable(); + for i in unpaired_src { + let item = &src[i]; + acc.add( + LossKind::Lost, + category_of(&item.body), + None, + Some(item.at), + || describe(&item.body, detail), + ); + } + for j in unpaired_rt { + let item = &rt[j]; + acc.add( + LossKind::Synthesized, + category_of(&item.body), + None, + None, + || describe(&item.body, detail), + ); + } +} + +fn describe_full(body: &Body<'_>) -> String { + describe(body, true).unwrap_or_default() +} + +/// A matched thinking block whose replay token did not survive intact. +fn compare_token(src: &Item<'_>, rt: &Item<'_>, acc: &mut Acc) { + let ( + Body::Thinking { + signature: s_sig, + encrypted: s_enc, + .. + }, + Body::Thinking { + signature: r_sig, + encrypted: r_enc, + .. + }, + ) = (&src.body, &rt.body) + else { + return; + }; + if (s_sig, s_enc) == (r_sig, r_enc) { + return; + } + let had = s_sig.is_some() || s_enc.is_some(); + let has = r_sig.is_some() || r_enc.is_some(); + let (kind, at) = match (had, has) { + (true, false) => (LossKind::Lost, Some(src.at)), + (false, true) => (LossKind::Synthesized, None), + _ => (LossKind::Changed, Some(src.at)), + }; + acc.add(kind, Category::ThinkingSignature, None, at, || None); +} + +/// Two leftover texts that resemble each other. +fn changed_text(src: &Item<'_>, rt: &Item<'_>, acc: &mut Acc) { + let cap = acc.cap; + match (&src.body, &rt.body) { + ( + Body::Text { + role: was, + text: before, + }, + Body::Text { + role: now, + text: after, + }, + ) => { + if was != now && same_words(before, after) { + let name = |r: &Role| { + if *r == Role::User { + "user" + } else { + "assistant" + } + }; + acc.add( + LossKind::Changed, + Category::Text, + Some("role".to_string()), + Some(src.at), + || Some(format!("{} -> {}", name(was), name(now))), + ); + } else { + acc.add( + LossKind::Changed, + Category::Text, + None, + Some(src.at), + || Some(arrow(before, after, cap)), + ); + } + } + (Body::Thinking { text: before, .. }, Body::Thinking { text: after, .. }) => { + acc.add( + LossKind::Changed, + Category::Thinking, + None, + Some(src.at), + || Some(arrow(before, after, cap)), + ); + compare_token(src, rt, acc); + } + _ => {} + } +} + +fn show(v: &Value) -> String { + v.as_str().map_or_else(|| v.to_string(), str::to_string) +} + +/// Two leftover calls that are the same call, altered: a new name, or +/// arguments that were dropped or rewritten. +fn changed_call(src: &Item<'_>, rt: &Item<'_>, acc: &mut Acc) { + let ( + Body::Call { + name: s_name, + input: s_in, + }, + Body::Call { + name: r_name, + input: r_in, + }, + ) = (&src.body, &rt.body) + else { + return; + }; + let at = Some(src.at); + let cap = acc.cap; + if s_name != r_name { + acc.add( + LossKind::Changed, + Category::ToolName, + Some(format!("{s_name} -> {r_name}")), + at, + || None, + ); + } + match (s_in.as_object(), r_in.as_object()) { + (Some(before), Some(after)) => { + fn sorted( + map: &serde_json::Map, + keep: impl Fn(&String) -> bool, + ) -> Vec<&String> { + let mut keys: Vec<&String> = map.keys().filter(|k| keep(k)).collect(); + keys.sort(); + keys + } + for key in sorted(before, |k| after.contains_key(k)) { + if after[key] != before[key] { + acc.add( + LossKind::Changed, + Category::ToolArgs, + Some(format!("{s_name}.{key}")), + at, + || Some(arrow(&show(&before[key]), &show(&after[key]), cap)), + ); + } + } + let mut added = sorted(after, |k| !before.contains_key(k)); + for key in sorted(before, |k| !after.contains_key(k)) { + // The same value under another key is a rename, not a loss. + if let Some(pos) = added.iter().position(|k| after[*k] == before[key]) { + let new = added.remove(pos); + acc.add( + LossKind::Changed, + Category::ToolArgs, + Some(format!("{s_name}.{key} -> {new}")), + at, + || Some(show(&before[key])), + ); + } else { + acc.add( + LossKind::Lost, + Category::ToolArgs, + Some(format!("{s_name}.{key}")), + at, + || Some(show(&before[key])), + ); + } + } + for key in added { + acc.add( + LossKind::Synthesized, + Category::ToolArgs, + Some(format!("{s_name}.{key}")), + None, + || Some(show(&after[key])), + ); + } + } + _ if s_in != r_in => acc.add( + LossKind::Changed, + Category::ToolArgs, + Some(s_name.clone()), + at, + || Some(arrow(&show(s_in), &show(r_in), cap)), + ), + _ => {} + } +} + +// ── accounting ───────────────────────────────────────────────────────── + +fn str_key(s: &str) -> u64 { + let mut h = DefaultHasher::new(); + s.hash(&mut h); + h.finish() +} + +fn stop_name(reason: &StopReason) -> String { + match reason { + StopReason::EndTurn => "end_turn".to_string(), + StopReason::ToolUse => "tool_use".to_string(), + StopReason::MaxTokens => "max_tokens".to_string(), + StopReason::StopSequence => "stop_sequence".to_string(), + StopReason::Aborted => "aborted".to_string(), + StopReason::Error => "error".to_string(), + StopReason::Other(other) => format!("other({other})"), + } +} + +/// Compare per-message values that targets may move between messages: equal +/// values cancel out; when the same number are left on each side they are +/// the same slots changed, otherwise the surplus is lost or synthesized. +fn compare_values(category: Category, src: &[(usize, String)], rt: &[String], acc: &mut Acc) { + let s_keys: Vec = src.iter().map(|(_, v)| str_key(v)).collect(); + let r_keys: Vec = rt.iter().map(|v| str_key(v)).collect(); + let (_, src_left, rt_left) = match_keys(&s_keys, &r_keys); + let cap = acc.cap; + if !src_left.is_empty() && src_left.len() == rt_left.len() { + for (&i, &j) in src_left.iter().zip(&rt_left) { + acc.add(LossKind::Changed, category, None, Some(src[i].0), || { + Some(arrow(&src[i].1, &rt[j], cap)) + }); + } + return; + } + for &i in &src_left { + acc.add(LossKind::Lost, category, None, Some(src[i].0), || { + Some(src[i].1.clone()) + }); + } + for &j in &rt_left { + acc.add(LossKind::Synthesized, category, None, None, || { + Some(rt[j].clone()) + }); + } +} + +/// A usage record's fields. An absent cache count and a zero one both mean +/// "nothing cached", so they compare equal. +fn usage_fields(u: &Usage) -> [(&'static str, u64); 4] { + [ + ("input_tokens", u.input_tokens), + ("output_tokens", u.output_tokens), + ( + "cache_read_input_tokens", + u.cache_read_input_tokens.unwrap_or(0), + ), + ( + "cache_creation_input_tokens", + u.cache_creation_input_tokens.unwrap_or(0), + ), + ] +} + +fn compare_usage(src: &[(usize, Usage)], rt: &[Usage], acc: &mut Acc) { + let key = |u: &Usage| { + let mut h = DefaultHasher::new(); + usage_fields(u).hash(&mut h); + h.finish() + }; + // The same turn with its cache counts dropped still has the same input + // and output counts. + let core = |u: &Usage| { + let mut h = DefaultHasher::new(); + (u.input_tokens, u.output_tokens).hash(&mut h); + h.finish() + }; + let s_keys: Vec = src.iter().map(|(_, u)| key(u)).collect(); + let r_keys: Vec = rt.iter().map(key).collect(); + let (_, src_left, rt_left) = match_keys(&s_keys, &r_keys); + let s_cores: Vec = src_left.iter().map(|&i| core(&src[i].1)).collect(); + let r_cores: Vec = rt_left.iter().map(|&j| core(&rt[j])).collect(); + let (pairs, s_gone, r_new) = match_keys(&s_cores, &r_cores); + let mut fieldwise = |i: usize, j: usize| { + let at = Some(src[i].0); + for ((name, was), (_, now)) in usage_fields(&src[i].1) + .into_iter() + .zip(usage_fields(&rt[j])) + { + let path = Some(name.to_string()); + match (was, now) { + (a, b) if a == b => {} + (_, 0) => acc.add(LossKind::Lost, Category::Usage, path, at, || None), + (0, _) => acc.add(LossKind::Synthesized, Category::Usage, path, None, || None), + (a, b) => acc.add(LossKind::Changed, Category::Usage, path, at, || { + Some(format!("{a} -> {b}")) + }), + } + } + }; + for (a, b) in pairs { + fieldwise(src_left[a], rt_left[b]); + } + // Nothing alike but as many records on each side: the same turns, + // counted differently. + if !s_gone.is_empty() && s_gone.len() == r_new.len() { + for (&a, &b) in s_gone.iter().zip(&r_new) { + fieldwise(src_left[a], rt_left[b]); + } + return; + } + for a in s_gone { + acc.add( + LossKind::Lost, + Category::Usage, + None, + Some(src[src_left[a]].0), + || None, + ); + } + for _ in r_new { + acc.add(LossKind::Synthesized, Category::Usage, None, None, || None); + } +} + +fn compare_accounting(src: &Transcript, rt: &Transcript, acc: &mut Acc) { + let usage: Vec<(usize, Usage)> = src + .body + .iter() + .enumerate() + .filter_map(|(at, m)| m.usage.map(|u| (at, u))) + .collect(); + let usage_rt: Vec = rt.body.iter().filter_map(|m| m.usage).collect(); + compare_usage(&usage, &usage_rt, acc); + + let stops = |t: &Transcript| -> Vec<(usize, String)> { + t.body + .iter() + .enumerate() + .filter_map(|(at, m)| m.stop_reason.as_ref().map(|r| (at, stop_name(r)))) + .collect() + }; + let rt_stops: Vec = stops(rt).into_iter().map(|(_, v)| v).collect(); + compare_values(Category::StopReason, &stops(src), &rt_stops, acc); + + let models = |t: &Transcript| -> Vec<(usize, String)> { + t.body + .iter() + .enumerate() + .filter_map(|(at, m)| { + let model = m.model.as_deref().map(str::trim).filter(|m| !m.is_empty()); + model.map(|model| (at, model.to_string())) + }) + .collect() + }; + let rt_models: Vec = models(rt).into_iter().map(|(_, v)| v).collect(); + compare_values(Category::MessageModel, &models(src), &rt_models, acc); + + // Targets that do not keep message times leave the source's seconds + // missing from the round trip. + let kept: HashSet = rt.body.iter().map(|m| m.timestamp.timestamp()).collect(); + for (at, m) in src.body.iter().enumerate() { + if !kept.contains(&m.timestamp.timestamp()) { + acc.add( + LossKind::Lost, + Category::Timestamp, + Some("message".to_string()), + Some(at), + || Some(m.timestamp.to_rfc3339()), + ); + } + } +} + +// ── metadata ─────────────────────────────────────────────────────────── + +fn compare_field(category: Category, was: Option<&str>, now: Option<&str>, acc: &mut Acc) { + // An empty value is no value. + let (was, now) = (was.filter(|s| !s.is_empty()), now.filter(|s| !s.is_empty())); + let cap = acc.cap; + match (was, now) { + (Some(a), None) => acc.add(LossKind::Lost, category, None, None, || Some(a.to_string())), + (Some(a), Some(b)) if a != b => { + acc.add(LossKind::Changed, category, None, None, || { + Some(arrow(a, b, cap)) + }); + } + (None, Some(b)) => acc.add(LossKind::Synthesized, category, None, None, || { + Some(b.to_string()) + }), + _ => {} + } +} + +fn compare_meta(src: &Transcript, rt: &Transcript, acc: &mut Acc) { + let (s, r) = (&src.meta, &rt.meta); + compare_field( + Category::Title, + s.title.as_deref().map(str::trim), + r.title.as_deref().map(str::trim), + acc, + ); + compare_field(Category::Model, s.model.as_deref(), r.model.as_deref(), acc); + compare_field( + Category::GitBranch, + s.git_branch.as_deref(), + r.git_branch.as_deref(), + acc, + ); + compare_field(Category::Cwd, s.cwd.as_deref(), r.cwd.as_deref(), acc); + let lineage = + |l: &crate::common::Lineage| format!("{:?} of {}", l.relation, l.parent).to_lowercase(); + compare_field( + Category::Lineage, + s.lineage.as_ref().map(lineage).as_deref(), + r.lineage.as_ref().map(lineage).as_deref(), + acc, + ); +} + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] + + use chrono::{DateTime, Utc}; + use serde_json::json; + + use super::*; + use crate::common::{ImageSource, Lineage, Message, Meta, Relation, Tool}; + + fn at(secs: i64) -> DateTime { + DateTime::from_timestamp(1_790_000_000 + secs, 0).unwrap() + } + + fn meta() -> Meta { + Meta { + id: "t".into(), + timestamp: at(0), + cwd: None, + git_branch: None, + title: None, + cli_version: None, + model: None, + lineage: None, + } + } + + fn msg(role: Role, content: Vec, secs: i64) -> Message { + Message { + role, + content, + timestamp: at(secs), + model: None, + stop_reason: None, + usage: None, + } + } + + fn text(s: &str) -> Block { + Block::Text { text: s.into() } + } + + fn call(id: &str, name: &str, input: Value) -> Block { + Block::ToolUse { + id: id.into(), + tool: Tool::from_canonical(name, input), + } + } + + fn result(id: &str, out: ToolOutput) -> Block { + Block::ToolResult { + tool_use_id: id.into(), + content: out, + is_error: false, + } + } + + fn thinking(s: &str, signature: Option<&str>) -> Block { + Block::Thinking { + text: s.into(), + signature: signature.map(Into::into), + encrypted: None, + } + } + + fn convo(messages: Vec) -> Transcript { + Transcript::new(meta(), messages) + } + + fn user(blocks: Vec) -> Message { + msg(Role::User, blocks, 1) + } + + fn assistant(blocks: Vec) -> Message { + msg(Role::Assistant, blocks, 2) + } + + fn run(src: &Transcript, rt: &Transcript) -> Report { + compare(HarnessId::Simple, src, rt, Options::default()) + } + + /// The `(kind, category, path, count)` of every item. + fn shape(report: &Report) -> Vec<(LossKind, Category, Option<&str>, usize)> { + report + .items + .iter() + .map(|i| (i.kind, i.category, i.path.as_deref(), i.count)) + .collect() + } + + /// Assert that a comparison found nothing. + fn assert_clean(report: &Report) { + assert_eq!(report.items.len(), 0, "{:?}", report.items); + } + + fn base() -> Transcript { + convo(vec![ + user(vec![text("fix it")]), + assistant(vec![ + thinking("hmm", Some("sig")), + text("on it"), + call( + "c1", + "Bash", + json!({"command": "ls", "description": "list"}), + ), + ]), + user(vec![result("c1", ToolOutput::Text("a\nb".into()))]), + ]) + } + + #[test] + fn identical_transcripts_have_no_items() { + let src = base(); + let report = run(&src, &src.clone()); + assert!(report.items.is_empty(), "{report:?}"); + assert_eq!(report.severity, Severity::None); + assert_eq!(report.warnings.len(), 0); + assert_eq!(report.messages_before, report.messages_after); + } + + #[test] + fn lost_text_is_content() { + let src = convo(vec![user(vec![text("one"), text("two")])]); + let rt = convo(vec![user(vec![text("one")])]); + let report = run(&src, &rt); + assert_eq!(shape(&report), [(LossKind::Lost, Category::Text, None, 1)]); + assert_eq!(report.items[0].example.as_deref(), Some("two")); + assert_eq!(report.items[0].first_at, Some(0)); + assert_eq!(report.severity, Severity::Content); + } + + #[test] + fn synthesized_text_is_listed_but_does_not_raise_severity() { + let src = convo(vec![user(vec![text("one")])]); + let rt = convo(vec![user(vec![text("one"), text("extra")])]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [(LossKind::Synthesized, Category::Text, None, 1)] + ); + assert_eq!(report.items[0].first_at, None); + assert_eq!(report.severity, Severity::None); + } + + #[test] + fn edited_text_is_changed_once_not_lost_and_synthesized() { + let src = convo(vec![user(vec![text("please fix the parser bug today")])]); + let rt = convo(vec![user(vec![text("please fix the parser bug")])]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [(LossKind::Changed, Category::Text, None, 1)] + ); + assert_eq!( + report.items[0].example.as_deref(), + Some("please fix the parser bug today -> please fix the parser bug") + ); + } + + #[test] + fn unrelated_replacement_text_is_lost_plus_synthesized() { + let src = convo(vec![user(vec![text("alpha beta gamma")])]); + let rt = convo(vec![user(vec![text("omega psi chi")])]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [ + (LossKind::Lost, Category::Text, None, 1), + (LossKind::Synthesized, Category::Text, None, 1) + ] + ); + } + + #[test] + fn surrounding_whitespace_is_not_a_change() { + let src = convo(vec![user(vec![text(" hello\n")])]); + let rt = convo(vec![user(vec![text("hello")])]); + assert_clean(&run(&src, &rt)); + } + + #[test] + fn empty_text_blocks_are_ignored() { + let src = convo(vec![user(vec![text("hello")])]); + let rt = convo(vec![user(vec![text("hello"), text(" ")])]); + assert_clean(&run(&src, &rt)); + } + + #[test] + fn a_text_changing_role_is_a_changed_text() { + let src = convo(vec![user(vec![text("same words")])]); + let rt = convo(vec![msg(Role::Assistant, vec![text("same words")], 1)]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [(LossKind::Changed, Category::Text, Some("role"), 1)] + ); + assert_eq!( + report.items[0].example.as_deref(), + Some("user -> assistant") + ); + } + + #[test] + fn thinking_lost_and_changed() { + let src = convo(vec![assistant(vec![thinking( + "deep thoughts about it", + None, + )])]); + let lost = run(&src, &convo(vec![assistant(vec![])])); + assert_eq!( + shape(&lost), + [(LossKind::Lost, Category::Thinking, None, 1)] + ); + let changed = run( + &src, + &convo(vec![assistant(vec![thinking("deep thoughts about", None)])]), + ); + assert_eq!( + shape(&changed), + [(LossKind::Changed, Category::Thinking, None, 1)] + ); + } + + #[test] + fn thinking_signature_lost_changed_synthesized() { + let signed = convo(vec![assistant(vec![thinking("hmm", Some("a"))])]); + let bare = convo(vec![assistant(vec![thinking("hmm", None)])]); + let other = convo(vec![assistant(vec![thinking("hmm", Some("b"))])]); + assert_eq!( + shape(&run(&signed, &bare)), + [(LossKind::Lost, Category::ThinkingSignature, None, 1)] + ); + assert_eq!( + shape(&run(&signed, &other)), + [(LossKind::Changed, Category::ThinkingSignature, None, 1)] + ); + assert_eq!( + shape(&run(&bare, &signed)), + [(LossKind::Synthesized, Category::ThinkingSignature, None, 1)] + ); + assert_eq!(run(&signed, &bare).severity, Severity::Accounting); + } + + #[test] + fn a_dropped_tool_call_is_lost_with_its_label() { + let src = base(); + let mut rt = base(); + rt.body[1].content.pop(); + let report = run(&src, &rt); + assert!( + report.items.iter().any(|i| i.kind == LossKind::Lost + && i.category == Category::ToolCall + && i.example.as_deref() == Some("Bash: ls")), + "{report:?}" + ); + } + + #[test] + fn a_renamed_tool_is_changed_by_name_only() { + let src = convo(vec![user(vec![call( + "c1", + "/release", + json!({"args": "patch"}), + )])]); + let rt = convo(vec![user(vec![call( + "c1", + "_release", + json!({"args": "patch"}), + )])]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [( + LossKind::Changed, + Category::ToolName, + Some("/release -> _release"), + 1 + )] + ); + } + + #[test] + fn dropped_rewritten_and_added_tool_arguments() { + let src = convo(vec![assistant(vec![call( + "c1", + "Bash", + json!({"command": "ls", "description": "list", "timeout_ms": 5}), + )])]); + let rt = convo(vec![assistant(vec![call( + "c1", + "Bash", + json!({"command": "ls", "timeout_ms": 9, "workdir": "/x"}), + )])]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [ + ( + LossKind::Lost, + Category::ToolArgs, + Some("Bash.description"), + 1 + ), + ( + LossKind::Changed, + Category::ToolArgs, + Some("Bash.timeout_ms"), + 1 + ), + ( + LossKind::Synthesized, + Category::ToolArgs, + Some("Bash.workdir"), + 1 + ), + ] + ); + assert_eq!(report.items[0].example.as_deref(), Some("list")); + assert_eq!(report.items[1].example.as_deref(), Some("5 -> 9")); + } + + #[test] + fn a_call_with_nothing_in_common_is_lost_not_changed() { + let src = convo(vec![assistant(vec![call( + "c1", + "Bash", + json!({"command": "ls"}), + )])]); + let rt = convo(vec![assistant(vec![call( + "c1", + "Bash", + json!({"command": "pwd"}), + )])]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [ + (LossKind::Lost, Category::ToolCall, None, 1), + (LossKind::Synthesized, Category::ToolCall, None, 1) + ] + ); + } + + #[test] + fn json_formatting_is_not_loss() { + // Key order and whitespace in a result held as text, or in a + // structured result, never count. + let src = convo(vec![ + assistant(vec![call( + "c1", + "mcp__x__y", + json!({"b": 1, "a": {"y": [1, 2], "x": null}}), + )]), + user(vec![result( + "c1", + ToolOutput::Text("{\"a\": 1,\n \"b\": [1,\n 2]}".into()), + )]), + ]); + let rt = convo(vec![ + assistant(vec![call( + "c9", + "mcp__x__y", + json!({"a": {"x": null, "y": [1, 2]}, "b": 1}), + )]), + user(vec![result( + "c9", + ToolOutput::Json(json!({"b": [1, 2], "a": 1})), + )]), + ]); + let report = run(&src, &rt); + assert!(report.items.is_empty(), "{report:?}"); + } + + #[test] + fn a_list_of_text_parts_equals_its_text() { + let src = convo(vec![user(vec![result( + "c1", + ToolOutput::Json(json!([{"type": "text", "text": "hello"}])), + )])]); + let rt = convo(vec![user(vec![result( + "c1", + ToolOutput::Text("hello".into()), + )])]); + assert_clean(&run(&src, &rt)); + } + + #[test] + fn a_changed_result_hides_its_body_unless_detail() { + let src = convo(vec![user(vec![result( + "c1", + ToolOutput::Text("secret output line one and more".into()), + )])]); + let rt = convo(vec![user(vec![result( + "c1", + ToolOutput::Text("secret output line one".into()), + )])]); + let plain = run(&src, &rt); + assert_eq!( + shape(&plain), + [(LossKind::Changed, Category::ToolResult, None, 1)] + ); + assert_eq!(plain.items[0].example, None); + let detailed = compare(HarnessId::Simple, &src, &rt, Options { detail: true }); + assert!( + detailed.items[0] + .example + .as_deref() + .is_some_and(|e| e.contains("secret")) + ); + } + + #[test] + fn a_lost_result_never_shows_its_body() { + let src = convo(vec![user(vec![result( + "c1", + ToolOutput::Text("password=hunter2".into()), + )])]); + let report = run(&src, &convo(vec![user(vec![])])); + assert_eq!( + shape(&report), + [(LossKind::Lost, Category::ToolResult, None, 1)] + ); + assert_eq!(report.items[0].example, None); + } + + #[test] + fn error_flag_lost_and_synthesized() { + let flagged = |is_error| { + convo(vec![user(vec![Block::ToolResult { + tool_use_id: "c1".into(), + content: ToolOutput::Text("boom".into()), + is_error, + }])]) + }; + assert_eq!( + shape(&run(&flagged(true), &flagged(false))), + [(LossKind::Lost, Category::IsError, None, 1)] + ); + assert_eq!( + shape(&run(&flagged(false), &flagged(true))), + [(LossKind::Synthesized, Category::IsError, None, 1)] + ); + } + + #[test] + fn images_and_artifacts() { + let image = |data: &str| Block::Image { + source: ImageSource { + source_type: "base64".into(), + media_type: "image/png".into(), + data: data.into(), + }, + }; + let art = |text: &str| Block::Artifact { + artifact: Artifact { + id: "a".into(), + name: "report.md".into(), + source: ArtifactSource::Text { + text: text.into(), + media_type: None, + }, + }, + }; + let src = convo(vec![user(vec![image("AAAA"), art("# one")])]); + let gone = run(&src, &convo(vec![user(vec![])])); + assert_eq!( + shape(&gone), + [ + (LossKind::Lost, Category::Image, None, 1), + (LossKind::Lost, Category::Artifact, None, 1) + ] + ); + assert_eq!(gone.items[0].example.as_deref(), Some("image/png")); + assert_eq!(gone.items[1].example.as_deref(), Some("report.md")); + // A re-encoded image is a change; an artifact is its name, since its + // content lives in a file of the store's making. + let edited = run(&src, &convo(vec![user(vec![image("BBBB"), art("# two")])])); + assert_eq!( + shape(&edited), + [(LossKind::Changed, Category::Image, None, 1)] + ); + } + + #[test] + fn an_artifact_that_comes_back_as_text_is_lost() { + // Harnesses without a file carrier keep only a mention in the text; + // claude_code keeps the artifact itself. + let src = convo(vec![assistant(vec![Block::Artifact { + artifact: Artifact { + id: "a".into(), + name: "report.md".into(), + source: ArtifactSource::Text { + text: "# one".into(), + media_type: None, + }, + }, + }])]); + let kept = assess(HarnessId::ClaudeCode, &src).unwrap(); + assert!( + kept.items.iter().all(|i| i.category != Category::Artifact), + "{:?}", + kept.items + ); + let mention = assess(HarnessId::Cursor, &src).unwrap(); + assert!( + mention + .items + .iter() + .any(|i| i.kind == LossKind::Lost && i.category == Category::Artifact), + "{:?}", + mention.items + ); + } + + #[test] + fn split_and_merged_messages_are_not_loss() { + let blocks = vec![ + text("first"), + call("c1", "Bash", json!({"command": "ls"})), + text("last"), + ]; + let merged = convo(vec![assistant(blocks.clone())]); + let split = convo(blocks.into_iter().map(|b| assistant(vec![b])).collect()); + assert_clean(&run(&merged, &split)); + let back = run(&split, &merged); + assert_clean(&back); + assert_eq!(back.messages_before, 3); + assert_eq!(back.messages_after, 1); + } + + #[test] + fn reordered_blocks_are_a_content_change() { + let src = convo(vec![assistant(vec![ + text("one"), + text("two"), + text("three"), + ])]); + let rt = convo(vec![assistant(vec![ + text("three"), + text("one"), + text("two"), + ])]); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [(LossKind::Changed, Category::Order, None, 1)] + ); + assert_eq!(report.severity, Severity::Content); + } + + fn with_usage(n: u64, cache: Option) -> Transcript { + let mut t = convo(vec![assistant(vec![text("hi")])]); + t.body[0].usage = Some(Usage { + input_tokens: n, + output_tokens: 5, + cache_read_input_tokens: cache, + cache_creation_input_tokens: None, + }); + t + } + + #[test] + fn usage_lost_changed_field_by_field_and_synthesized() { + let plain = convo(vec![assistant(vec![text("hi")])]); + assert_eq!( + shape(&run(&with_usage(10, None), &plain)), + [(LossKind::Lost, Category::Usage, None, 1)] + ); + assert_eq!( + shape(&run(&plain, &with_usage(10, None))), + [(LossKind::Synthesized, Category::Usage, None, 1)] + ); + assert_eq!( + shape(&run(&with_usage(10, Some(7)), &with_usage(10, None))), + [( + LossKind::Lost, + Category::Usage, + Some("cache_read_input_tokens"), + 1 + )] + ); + assert_eq!( + shape(&run(&with_usage(10, None), &with_usage(12, None))), + [(LossKind::Changed, Category::Usage, Some("input_tokens"), 1)] + ); + } + + #[test] + fn an_absent_cache_count_equals_zero() { + assert_clean(&run(&with_usage(10, None), &with_usage(10, Some(0)))); + assert_clean(&run(&with_usage(10, Some(0)), &with_usage(10, None))); + } + + fn with_stop(reason: Option) -> Transcript { + let mut t = convo(vec![assistant(vec![text("hi")])]); + t.body[0].stop_reason = reason; + t + } + + #[test] + fn stop_reason_lost_changed_synthesized() { + let ended = with_stop(Some(StopReason::EndTurn)); + let none = with_stop(None); + assert_eq!( + shape(&run(&ended, &none)), + [(LossKind::Lost, Category::StopReason, None, 1)] + ); + assert_eq!( + shape(&run(&none, &ended)), + [(LossKind::Synthesized, Category::StopReason, None, 1)] + ); + // Same slot, other representation: one change, not a loss plus a + // synthesis. + let aborted = with_stop(Some(StopReason::Aborted)); + let other = with_stop(Some(StopReason::Other("aborted".into()))); + let report = run(&aborted, &other); + assert_eq!( + shape(&report), + [(LossKind::Changed, Category::StopReason, None, 1)] + ); + assert_eq!( + report.items[0].example.as_deref(), + Some("aborted -> other(aborted)") + ); + assert_eq!(report.severity, Severity::Accounting); + } + + #[test] + fn unequal_stop_reason_leftovers_are_not_paired() { + let mut src = convo(vec![assistant(vec![text("a")]), assistant(vec![text("b")])]); + src.body[0].stop_reason = Some(StopReason::EndTurn); + src.body[1].stop_reason = Some(StopReason::Aborted); + let mut rt = src.clone(); + rt.body[0].stop_reason = Some(StopReason::ToolUse); + rt.body[1].stop_reason = None; + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [ + (LossKind::Lost, Category::StopReason, None, 2), + (LossKind::Synthesized, Category::StopReason, None, 1) + ] + ); + } + + #[test] + fn message_model_lost_changed_synthesized() { + let model = |m: Option<&str>| { + let mut t = convo(vec![assistant(vec![text("hi")])]); + t.body[0].model = m.map(Into::into); + t + }; + assert_eq!( + shape(&run(&model(Some("m1")), &model(None))), + [(LossKind::Lost, Category::MessageModel, None, 1)] + ); + assert_eq!( + shape(&run(&model(Some("m1")), &model(Some("m2")))), + [(LossKind::Changed, Category::MessageModel, None, 1)] + ); + assert_eq!( + shape(&run(&model(None), &model(Some("m1")))), + [(LossKind::Synthesized, Category::MessageModel, None, 1)] + ); + } + + #[test] + fn message_times_are_compared_by_second() { + let src = convo(vec![ + msg(Role::User, vec![text("a")], 1), + msg(Role::User, vec![text("b")], 2), + ]); + let mut rt = src.clone(); + rt.body[1].timestamp = at(50); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [(LossKind::Lost, Category::Timestamp, Some("message"), 1)] + ); + assert_eq!(report.items[0].first_at, Some(1)); + assert_eq!(report.severity, Severity::Metadata); + // Sub-second precision is not a loss. + let mut fine = src.clone(); + fine.body[0].timestamp += chrono::Duration::milliseconds(400); + assert_clean(&run(&src, &fine)); + } + + #[test] + fn session_metadata_lost_changed_synthesized() { + let mut src = base(); + src.meta.title = Some("A title".into()); + src.meta.model = Some("m1".into()); + src.meta.git_branch = Some("main".into()); + src.meta.cwd = Some("/work".into()); + src.meta.lineage = Some(Lineage { + parent: "p1".into(), + relation: Relation::Fork, + }); + let mut rt = base(); + rt.meta.title = Some("Another".into()); + rt.meta.model = None; + rt.meta.cwd = Some("/work".into()); + rt.meta.git_branch = None; + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [ + (LossKind::Changed, Category::Title, None, 1), + (LossKind::Lost, Category::Model, None, 1), + (LossKind::Lost, Category::GitBranch, None, 1), + (LossKind::Lost, Category::Lineage, None, 1), + ] + ); + assert_eq!(report.items[3].example.as_deref(), Some("fork of p1")); + assert_eq!(report.severity, Severity::Metadata); + + let mut gained = base(); + gained.meta.cwd = Some("/new".into()); + assert_eq!( + shape(&run(&base(), &gained)), + [(LossKind::Synthesized, Category::Cwd, None, 1)] + ); + // Titles compare trimmed. + let mut padded = base(); + padded.meta.title = Some("T ".into()); + let mut bare = base(); + bare.meta.title = Some("T".into()); + assert_clean(&run(&padded, &bare)); + } + + #[test] + fn the_session_start_is_not_reported() { + let src = base(); + let mut rt = base(); + rt.meta.timestamp = at(3600); + assert_clean(&run(&src, &rt)); + } + + #[test] + fn an_empty_session_warns() { + let src = convo(vec![]); + let report = run(&src, &src.clone()); + assert_clean(&report); + assert_eq!(report.severity, Severity::None); + assert_eq!(report.warnings.len(), 1); + assert!(report.warnings[0].contains("0 messages")); + } + + #[test] + fn severity_is_ordered_and_takes_the_worst_item() { + assert!(Severity::None < Severity::Metadata); + assert!(Severity::Metadata < Severity::Accounting); + assert!(Severity::Accounting < Severity::Content); + + let mut src = base(); + src.meta.title = Some("t".into()); + src.body[1].stop_reason = Some(StopReason::EndTurn); + let mut rt = src.clone(); + rt.meta.title = None; + assert_eq!(run(&src, &rt).severity, Severity::Metadata); + rt.body[1].stop_reason = None; + assert_eq!(run(&src, &rt).severity, Severity::Accounting); + rt.body[0].content.clear(); + assert_eq!(run(&src, &rt).severity, Severity::Content); + } + + #[test] + fn categories_map_to_their_severity_class() { + assert_eq!(Category::Text.severity(), Severity::Content); + assert_eq!(Category::IsError.severity(), Severity::Content); + assert_eq!(Category::Order.severity(), Severity::Content); + assert_eq!(Category::Usage.severity(), Severity::Accounting); + assert_eq!(Category::ThinkingSignature.severity(), Severity::Accounting); + assert_eq!(Category::Lineage.severity(), Severity::Metadata); + assert_eq!(Category::Timestamp.severity(), Severity::Metadata); + } + + #[test] + fn examples_are_capped_and_single_line() { + let long = format!("start\n\u{1b}[31m{}", "x".repeat(500)); + let src = convo(vec![user(vec![text(&long)])]); + let report = run(&src, &convo(vec![user(vec![])])); + let example = report.items[0].example.as_deref().unwrap(); + assert_eq!(example.chars().count(), EXAMPLE_CAP); + assert!(example.ends_with('…')); + assert!(!example.chars().any(char::is_control), "{example:?}"); + let detailed = compare( + HarnessId::Simple, + &src, + &convo(vec![user(vec![])]), + Options { detail: true }, + ); + let longer = detailed.items[0].example.as_deref().unwrap(); + assert_eq!(longer.chars().count(), DETAIL_CAP); + } + + #[test] + fn clip_handles_edges() { + assert_eq!(clip("abc", 80), "abc"); + assert_eq!(clip(" abc ", 80), "abc"); + assert_eq!(clip("abcd", 4), "abcd"); + assert_eq!(clip("abcde", 4), "abc…"); + assert_eq!(clip("", 4), ""); + assert_eq!(clip("日本語のコメントも直して", 5), "日本語の…"); + } + + #[test] + fn comparison_is_deterministic() { + let src = base(); + let mut rt = base(); + rt.body[1].content.truncate(1); + rt.meta.title = Some("x".into()); + let first = run(&src, &rt); + for _ in 0..20 { + assert_eq!(run(&src, &rt), first); + } + } + + #[test] + fn duplicate_blocks_match_one_to_one() { + let src = convo(vec![user(vec![text("ok"), text("ok"), text("ok")])]); + let rt = convo(vec![user(vec![text("ok")])]); + let report = run(&src, &rt); + assert_eq!(shape(&report), [(LossKind::Lost, Category::Text, None, 2)]); + } + + #[test] + fn assess_round_trips_through_the_real_codec() { + let src = base(); + let report = assess(HarnessId::Simple, &src).unwrap(); + assert!(report.items.is_empty(), "{report:?}"); + assert_eq!(report.target, HarnessId::Simple); + let serialized = serde_json::to_value(&report).unwrap(); + assert_eq!(serialized["target"], "simple"); + assert_eq!(serialized["severity"], "none"); + assert_eq!(serialized["items"], json!([])); + } + + #[test] + fn check_all_covers_every_target_and_survives_an_empty_session() { + let results = check_all(&convo(vec![])); + assert_eq!(results.len(), TARGETS.len()); + for (target, report) in results { + let report = report.unwrap_or_else(|e| panic!("{target}: {e}")); + assert_eq!(report.messages_before, 0); + assert_eq!(report.warnings.len(), 1); + } + } + + #[test] + fn read_only_sources_are_not_targets() { + for harness in [ + HarnessId::ClaudeChat, + HarnessId::CoworkRemote, + HarnessId::ChatGpt, + ] { + assert!(!TARGETS.contains(&harness)); + } + for harness in HarnessId::ALL { + let readonly = matches!( + harness, + HarnessId::ClaudeChat | HarnessId::CoworkRemote | HarnessId::ChatGpt + ); + assert_eq!(TARGETS.contains(&harness), !readonly, "{harness}"); + } + } + + #[test] + fn re_spaced_json_inside_text_parts_is_not_a_change() { + // Found by hand: a target re-serialized the JSON a result carried as + // the text of a text part, compactly; that is formatting, not loss. + let parts = |text: &str| ToolOutput::Json(json!([{"type": "text", "text": text}])); + let src = convo(vec![user(vec![result( + "c1", + parts("{\"a\": 1,\n \"b\": [1, 2]}"), + )])]); + let rt = convo(vec![user(vec![result( + "c1", + parts("{\"a\":1,\"b\":[1,2]}"), + )])]); + assert_clean(&run(&src, &rt)); + // A real difference inside it still counts. + let other = convo(vec![user(vec![result( + "c1", + parts("{\"a\":2,\"b\":[1,2]}"), + )])]); + assert_eq!( + shape(&run(&src, &other)), + [ + (LossKind::Lost, Category::ToolResult, None, 1), + (LossKind::Synthesized, Category::ToolResult, None, 1) + ] + ); + } + + #[test] + fn nested_stringified_json_is_bounded() { + let mut text = "{\"k\": 1}".to_string(); + // Each layer doubles the escaping, so a dozen is already deep. + for _ in 0..12 { + text = serde_json::to_string(&json!([text])).unwrap(); + } + let src = convo(vec![user(vec![result("c1", ToolOutput::Text(text))])]); + assert_clean(&run(&src, &src.clone())); + } + + #[test] + fn empty_models_and_titles_are_no_value() { + // Found by hand: pi writes an empty model on every turn it has none + // for, which is not a per-message model. + let mut src = convo(vec![assistant(vec![text("hi")])]); + let mut rt = src.clone(); + rt.body[0].model = Some(String::new()); + rt.meta.title = Some(String::new()); + assert_clean(&run(&src, &rt)); + src.body[0].model = Some(" ".into()); + assert_clean(&run(&src, &rt)); + } + + #[test] + fn a_cache_count_dropped_from_fewer_records_is_both_a_field_and_a_record_loss() { + // Found by hand: cursor_desktop drops cache counts and merges turns. + let record = |input: u64, cache: Option| Usage { + input_tokens: input, + output_tokens: 5, + cache_read_input_tokens: cache, + cache_creation_input_tokens: None, + }; + let mut src = convo(vec![assistant(vec![text("a")]); 3]); + for (n, m) in src.body.iter_mut().enumerate() { + m.usage = Some(record(10 + n as u64, Some(7))); + } + let mut rt = convo(vec![assistant(vec![text("a")]); 2]); + rt.body[0].usage = Some(record(10, None)); + rt.body[1].usage = Some(record(11, None)); + let report = run(&src, &rt); + assert_eq!( + shape(&report), + [ + (LossKind::Lost, Category::Text, None, 1), + (LossKind::Lost, Category::Usage, None, 1), + ( + LossKind::Lost, + Category::Usage, + Some("cache_read_input_tokens"), + 2 + ), + ] + ); + } + + #[test] + fn arrow_shows_whitespace_and_cuts_around_the_difference() { + assert_eq!(arrow("# notes\n", "# notes", 80), "# notes\\n -> # notes"); + let head = "x".repeat(300); + let a = format!("{head} alpha {}", "y".repeat(300)); + let b = format!("{head} beta {}", "y".repeat(300)); + let shown = arrow(&a, &b, 80); + assert!(shown.chars().count() <= 80, "{shown}"); + assert!(shown.contains("alpha") && shown.contains("beta"), "{shown}"); + assert!(shown.starts_with('…'), "{shown}"); + } + + #[test] + fn many_look_alike_leftovers_are_bounded_and_counted() { + // Results that open alike but are different never pair; each is + // compared against a bounded number of candidates. + let results = |tail: &str| -> Vec { + (0..2000) + .map(|n| { + user(vec![result( + "c", + ToolOutput::Text(format!("output{tail}{n}")), + )]) + }) + .collect() + }; + let report = run(&convo(results("-a-")), &convo(results("-b-"))); + assert_eq!( + shape(&report), + [ + (LossKind::Lost, Category::ToolResult, None, 2000), + (LossKind::Synthesized, Category::ToolResult, None, 2000), + ] + ); + } + + #[test] + fn loss_serializes_without_empty_fields() { + let src = convo(vec![user(vec![result("c1", ToolOutput::Text("x".into()))])]); + let report = run(&src, &convo(vec![user(vec![])])); + let value = serde_json::to_value(&report.items[0]).unwrap(); + assert_eq!( + value, + json!({"kind": "lost", "category": "tool_result", "count": 1, "first_at": 0}) + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index a340523..8fea8c5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,9 +17,15 @@ //! - [`Transcript`], [`Harness`], [`Codec`], [`Store`] — the generic type and //! the traits over it. //! - [`harness`] — one module per implemented harness. +//! - [`dispatch`] — parse or render a harness's native text, chosen by +//! [`HarnessId`] at run time. +//! - [`fidelity`] — what converting into a harness loses, measured before +//! anything is written. pub mod common; +pub mod dispatch; pub mod error; +pub mod fidelity; pub mod harness; #[cfg(not(target_arch = "wasm32"))] pub mod local; diff --git a/src/wasm.rs b/src/wasm.rs index e1bbb6f..2417b07 100644 --- a/src/wasm.rs +++ b/src/wasm.rs @@ -13,11 +13,8 @@ use serde::{Deserialize, Serialize}; use wasm_bindgen::prelude::*; use crate::common; -use crate::harness::{ - amp, antigravity, campfire, chatgpt, claude_chat, claude_code, codex, cowork, cowork_remote, - cursor, cursor_desktop, fx, grok, grok_bot, hermes, opencode, pi, simple, -}; -use crate::transcript::{Codec, Common, HarnessId, TextCodec, Transcript}; +use crate::dispatch::{parse_to_common, render_from_common}; +use crate::transcript::{HarnessId, Transcript}; /// Continue/convert a session from one harness's native text into another's. /// @@ -194,58 +191,6 @@ struct CommonJson { messages: Vec, } -fn parse_to_common(harness: HarnessId, text: &str) -> crate::Result> { - fn go(text: &str) -> crate::Result> { - H::to_common(&H::from_text(text)?) - } - match harness { - HarnessId::ClaudeCode => go::(text), - HarnessId::ClaudeChat => go::(text), - HarnessId::CoworkRemote => go::(text), - HarnessId::ChatGpt => go::(text), - HarnessId::Codex => go::(text), - HarnessId::OpenCode => go::(text), - HarnessId::Pi => go::(text), - HarnessId::Campfire => go::(text), - HarnessId::Cursor => go::(text), - HarnessId::CursorDesktop => go::(text), - HarnessId::Grok => go::(text), - HarnessId::GrokBot => go::(text), - HarnessId::Fx => go::(text), - HarnessId::Hermes => go::(text), - HarnessId::Amp => go::(text), - HarnessId::Antigravity => go::(text), - HarnessId::Simple => go::(text), - HarnessId::Cowork => go::(text), - } -} - -fn render_from_common(harness: HarnessId, common: &Transcript) -> crate::Result { - fn go(common: &Transcript) -> crate::Result { - H::to_text(&H::from_common(common)?) - } - match harness { - HarnessId::ClaudeCode => go::(common), - HarnessId::ClaudeChat => go::(common), - HarnessId::CoworkRemote => go::(common), - HarnessId::ChatGpt => go::(common), - HarnessId::Codex => go::(common), - HarnessId::OpenCode => go::(common), - HarnessId::Pi => go::(common), - HarnessId::Campfire => go::(common), - HarnessId::Cursor => go::(common), - HarnessId::CursorDesktop => go::(common), - HarnessId::Grok => go::(common), - HarnessId::GrokBot => go::(common), - HarnessId::Fx => go::(common), - HarnessId::Hermes => go::(common), - HarnessId::Amp => go::(common), - HarnessId::Antigravity => go::(common), - HarnessId::Simple => go::(common), - HarnessId::Cowork => go::(common), - } -} - fn parse_harness(name: &str) -> Result { name.parse().map_err(js) } diff --git a/tests/README.md b/tests/README.md index cb06c99..90b2dee 100644 --- a/tests/README.md +++ b/tests/README.md @@ -35,8 +35,11 @@ three contracts every harness must honor: extracted correctly. Plus cross-cutting modules: `cross_harness` (a conversation survives a hop -through every harness), `path_safety` (adversarial ids, symlinks, and -references cannot escape a store root), `store_delete`, `search`, and +through every harness), `fidelity` (what converting into each harness loses, +pinned in one golden table that also feeds `docs/formats/README.md`; update +both deliberately when a codec change moves it), `path_safety` (adversarial +ids, symlinks, and references cannot escape a store root), `store_delete`, +`search`, and `properties` — proptest sweeps asserting the codec fixpoint over generated conversations for every harness that writes (all but the pull-only Claude Chat and ChatGPT) at once. The generator in `properties` @@ -70,8 +73,9 @@ can compare across machines. - Every test file opens with `#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]`; the production lint set (workspace `Cargo.toml`) denies all three. -- CLI behavior that needs the real binary (id resolution, for one) is tested - in `cli/tests/`: the `txcript` binary runs against a synthetic store in a - temp dir, with every store environment variable pointed at an empty one. +- CLI behavior that needs the real binary (id resolution, `check`, the loss + note on `continue`) is tested in `cli/tests/`: the `txcript` binary runs + against a synthetic store in a temp dir, with every store environment + variable pointed at an empty one. - Feature-gated surfaces gate at the module declaration in `main.rs` (`search`) or on the test (`opencode` store). diff --git a/tests/integration/fidelity.rs b/tests/integration/fidelity.rs new file mode 100644 index 0000000..899351e --- /dev/null +++ b/tests/integration/fidelity.rs @@ -0,0 +1,741 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] + +//! Conversion fidelity (`txcript::fidelity`) measured against every harness +//! codec: a rich canonical session assessed per target with a golden result, +//! determinism, and no-panic sweeps over generated transcripts. +//! +//! The golden table below is the oracle for codec fidelity. A codec change +//! that loses less (or more) shows up as a line to add, change, or delete +//! here, which is the point: update it deliberately, in the same commit as the +//! codec change, and update the table in `docs/formats/README.md` that +//! `docs_table_matches_the_measurement` checks. Only losses and changes are +//! listed (synthesized items and per-message timestamps are not stable enough +//! to pin) and each line is one harness, kind, category and count on the +//! `rich` session. + +use chrono::{DateTime, Utc}; +use proptest::prelude::*; +use serde_json::json; +use txcript::common::{ + Block, ImageSource, Message, Meta, Role, StopReason, Tool, ToolOutput, Usage, +}; +use txcript::fidelity::{self, Category, LossKind, Severity}; +use txcript::{Common, HarnessId, Transcript}; + +fn ts(secs: i64) -> DateTime { + DateTime::from_timestamp(1_790_000_000 + secs, 0).unwrap() +} + +// One fixture with every kind of block; length grows with the kinds covered. +#[allow(clippy::too_many_lines)] +fn rich() -> Transcript { + let meta = Meta { + id: "fid-1".into(), + timestamp: ts(0), + cwd: Some("/work/repo".into()), + git_branch: Some("feat/clock".into()), + title: Some("Fix the flaky clock test".into()), + cli_version: Some("1.2.3".into()), + model: Some("claude-opus-4-8".into()), + lineage: None, + }; + let model = || Some("claude-opus-4-8".to_string()); + let usage = |n: u64| { + Some(Usage { + input_tokens: 1000 + n, + output_tokens: 100 + n, + cache_read_input_tokens: Some(900), + cache_creation_input_tokens: None, + }) + }; + let user = |blocks: Vec, t: i64| Message { + role: Role::User, + content: blocks, + timestamp: ts(t), + model: None, + stop_reason: None, + usage: None, + }; + let assistant = |blocks: Vec, t: i64, stop: StopReason| Message { + role: Role::Assistant, + content: blocks, + timestamp: ts(t), + model: model(), + stop_reason: Some(stop), + usage: usage(u64::try_from(t).unwrap()), + }; + let text = |s: &str| Block::Text { text: s.into() }; + let call = |id: &str, tool: Tool| Block::ToolUse { + id: id.into(), + tool, + }; + let result = |id: &str, out: ToolOutput, is_error: bool| Block::ToolResult { + tool_use_id: id.into(), + content: out, + is_error, + }; + let body = vec![ + user( + vec![ + text("Fix the flaky test in clock.rs. 日本語のコメントも直して 🚀"), + Block::Image { + source: ImageSource { + source_type: "base64".into(), + media_type: "image/png".into(), + data: "iVBORw0KGgo=".into(), + }, + }, + ], + 1, + ), + assistant( + vec![ + Block::Thinking { + text: "The test reads the wall clock.".into(), + signature: Some("sig-aaa".into()), + encrypted: None, + }, + text("Let me run it first."), + call( + "c1", + Tool::Bash { + command: "cargo test clock".into(), + workdir: None, + timeout_ms: None, + description: Some("Run the clock tests".into()), + run_in_background: false, + }, + ), + ], + 2, + StopReason::ToolUse, + ), + user( + vec![result( + "c1", + ToolOutput::Text("running 3 tests\n3 passed".into()), + false, + )], + 3, + ), + assistant( + vec![ + text("Pinning the clock."), + call( + "c2", + Tool::Edit { + file_path: "/work/repo/src/clock.rs".into(), + old_string: "Utc::now()".into(), + new_string: "self.now".into(), + replace_all: false, + }, + ), + ], + 4, + StopReason::ToolUse, + ), + user( + vec![result("c2", ToolOutput::Text("patched".into()), false)], + 5, + ), + assistant( + vec![call( + "c3", + Tool::Raw { + tool_name: "mcp__docs__lookup".into(), + input: json!({"q": "clock", "filters": {"b": 1, "a": 2}}), + }, + )], + 6, + StopReason::ToolUse, + ), + user( + vec![result( + "c3", + ToolOutput::Json(json!([{"title": "Clock"}, {"title": "Time"}])), + false, + )], + 7, + ), + assistant( + vec![call( + "c4", + Tool::Bash { + command: "false".into(), + workdir: None, + timeout_ms: None, + description: None, + run_in_background: false, + }, + )], + 8, + StopReason::ToolUse, + ), + user(vec![result("c4", ToolOutput::Text("boom".into()), true)], 9), + user( + vec![call( + "c5", + Tool::Command { + command: "/release".into(), + args: Some("patch".into()), + }, + )], + 10, + ), + user( + vec![result( + "c5", + ToolOutput::Text("released v1.2.3".into()), + false, + )], + 11, + ), + assistant( + vec![ + Block::Thinking { + text: "Everything passes now.".into(), + signature: None, + encrypted: None, + }, + text("All green."), + ], + 12, + StopReason::EndTurn, + ), + user(vec![text("thanks")], 13), + assistant(vec![text("Anytime.")], 14, StopReason::Aborted), + ]; + Transcript::new(meta, body) +} + +/// The targets `continue --with` writes, minus the pull-only and mint-only +/// ones: what `txcript check` compares by default. +const WRITABLE: [HarnessId; 11] = [ + HarnessId::ClaudeCode, + HarnessId::Codex, + HarnessId::OpenCode, + HarnessId::Pi, + HarnessId::Campfire, + HarnessId::Cursor, + HarnessId::CursorDesktop, + HarnessId::Grok, + HarnessId::Fx, + HarnessId::Antigravity, + HarnessId::Cowork, +]; + +/// One line per harness and severity, then one per loss, sorted. +const GOLDEN: &[&str] = &[ + "amp lost tool_call x1", + "amp severity content", + "antigravity changed title x1", + "antigravity lost message_model x6", + "antigravity lost model x1", + "antigravity lost tool_args Bash.description x1", + "antigravity lost tool_call x1", + "antigravity lost tool_result x1", + "antigravity severity content", + "campfire lost git_branch x1", + "campfire lost model x1", + "campfire lost thinking_signature x1", + "campfire lost title x1", + "campfire lost tool_call x1", + "campfire severity content", + "claude_code changed stop_reason x1", + "claude_code severity accounting", + "codex changed tool_name /release -> _release x1", + "codex lost is_error x1", + "codex lost model x1", + "codex lost stop_reason x6", + "codex lost thinking_signature x1", + "codex lost title x1", + "codex lost tool_args Bash.description x1", + "codex lost usage x2", + "codex severity content", + "cowork changed stop_reason x1", + "cowork severity accounting", + "cursor lost git_branch x1", + "cursor lost stop_reason x6", + "cursor lost tool_call x1", + "cursor lost usage x6", + "cursor severity content", + "cursor_desktop lost git_branch x1", + "cursor_desktop lost image x1", + "cursor_desktop lost stop_reason x6", + "cursor_desktop lost tool_args Bash.description x1", + "cursor_desktop lost tool_call x1", + "cursor_desktop lost tool_result x1", + "cursor_desktop lost usage cache_read_input_tokens x6", + "cursor_desktop severity content", + "fx lost git_branch x1", + "fx lost stop_reason x5", + "fx lost tool_args Bash.description x1", + "fx lost tool_call x1", + "fx lost usage x6", + "fx severity content", + "grok lost stop_reason x4", + "grok lost thinking_signature x1", + "grok lost tool_call x1", + "grok lost usage x6", + "grok severity content", + "grok_bot changed title x1", + "grok_bot lost cwd x1", + "grok_bot lost git_branch x1", + "grok_bot lost image x1", + "grok_bot lost message_model x6", + "grok_bot lost model x1", + "grok_bot lost stop_reason x6", + "grok_bot lost usage x6", + "grok_bot severity content", + "hermes lost thinking_signature x1", + "hermes lost tool_call x1", + "hermes lost usage x6", + "hermes severity content", + "opencode changed stop_reason x1", + "opencode lost git_branch x1", + "opencode lost thinking_signature x1", + "opencode lost tool_call x1", + "opencode lost tool_result x1", + "opencode severity content", + "pi lost git_branch x1", + "pi lost model x1", + "pi lost thinking_signature x1", + "pi lost title x1", + "pi lost tool_call x1", + "pi severity content", + "simple severity none", +]; + +/// The `rich` session assessed for `target`. +fn report(target: HarnessId) -> fidelity::Report { + fidelity::assess(target, &rich()).unwrap_or_else(|e| panic!("{target}: {e}")) +} + +fn golden_lines(target: HarnessId, report: &fidelity::Report) -> Vec { + let mut lines = vec![format!("{target} severity {}", report.severity.as_str())]; + for item in &report.items { + if item.kind == LossKind::Synthesized || item.category == Category::Timestamp { + continue; + } + let kind = match item.kind { + LossKind::Lost => "lost", + _ => "changed", + }; + let category = serde_json::to_value(item.category).unwrap(); + let path = item + .path + .as_deref() + .map(|p| format!(" {p}")) + .unwrap_or_default(); + lines.push(format!( + "{target} {kind} {}{path} x{}", + category.as_str().unwrap(), + item.count + )); + } + lines +} + +#[test] +fn every_target_matches_the_golden_measurement() { + let mut actual: Vec = fidelity::TARGETS + .into_iter() + .flat_map(|target| golden_lines(target, &report(target))) + .collect(); + actual.sort(); + let expected: Vec = GOLDEN.iter().map(ToString::to_string).collect(); + if actual != expected { + let table = actual + .iter() + .map(|l| format!(" \"{l}\",")) + .collect::>() + .join("\n"); + panic!( + "the measured fidelity differs from GOLDEN. If a codec change caused this on \ + purpose, replace the table with:\n{table}" + ); + } +} + +#[test] +fn claude_code_and_simple_lose_no_content() { + for target in [HarnessId::ClaudeCode, HarnessId::Simple, HarnessId::Cowork] { + let r = report(target); + assert!( + r.severity < Severity::Content, + "{target} lost content: {:?}", + r.items + ); + } + let simple = report(HarnessId::Simple); + assert!(simple.items.is_empty(), "{:?}", simple.items); + assert_eq!(simple.messages_before, simple.messages_after); +} + +#[test] +fn measurement_is_deterministic() { + let src = rich(); + for target in fidelity::TARGETS { + let first = fidelity::assess(target, &src).unwrap(); + for _ in 0..3 { + assert_eq!(fidelity::assess(target, &src).unwrap(), first, "{target}"); + } + } +} + +#[test] +fn check_all_reports_every_target_and_never_aborts() { + let results = fidelity::check_all(&rich()); + let targets: Vec = results.iter().map(|(t, _)| *t).collect(); + assert_eq!(targets, fidelity::TARGETS); + for (target, result) in results { + assert_eq!(result.unwrap().target, target); + } +} + +#[test] +fn read_only_sources_cannot_be_assessed_as_targets() { + // Not in `TARGETS`, and a direct call fails cleanly instead of panicking. + for target in [ + HarnessId::ClaudeChat, + HarnessId::CoworkRemote, + HarnessId::ChatGpt, + ] { + assert!(!fidelity::TARGETS.contains(&target)); + let _ = fidelity::assess(target, &rich()); + } +} + +fn status(report: &fidelity::Report, class: Severity) -> &'static str { + let any = |kind: LossKind| { + report.items.iter().any(|i| { + i.kind == kind && i.category.severity() == class && i.category != Category::Timestamp + }) + }; + if any(LossKind::Lost) { + "lost" + } else if any(LossKind::Changed) { + "changed" + } else { + "ok" + } +} + +/// One cell of the docs table: `lost: a, b; changed: c`, or a dash. +fn cell(report: &fidelity::Report, class: Severity) -> String { + let names = |kind: LossKind| -> Vec { + let mut names: Vec = Vec::new(); + for item in report.items.iter().filter(|i| { + i.kind == kind && i.category.severity() == class && i.category != Category::Timestamp + }) { + let name = serde_json::to_value(item.category).unwrap(); + let name = name.as_str().unwrap().to_string(); + if !names.contains(&name) { + names.push(name); + } + } + names + }; + let parts: Vec = [(LossKind::Lost, "lost"), (LossKind::Changed, "changed")] + .into_iter() + .filter_map(|(kind, label)| { + let names = names(kind); + (!names.is_empty()).then(|| format!("{label}: {}", names.join(", "))) + }) + .collect(); + if parts.is_empty() { + "ok".to_string() + } else { + parts.join("; ") + } +} + +/// The per-harness support table in `docs/formats/README.md` is this +/// measurement, rendered. When it drifts, the failure prints the rows to paste. +#[test] +fn docs_table_matches_the_measurement() { + let docs = include_str!("../../docs/formats/README.md"); + let rows: Vec = WRITABLE + .into_iter() + .map(|target| { + let r = report(target); + format!( + "| {target} | {} | {} | {} |", + cell(&r, Severity::Content), + cell(&r, Severity::Accounting), + cell(&r, Severity::Metadata) + ) + }) + .collect(); + let missing: Vec<&String> = rows + .iter() + .filter(|row| !docs.contains(row.as_str())) + .collect(); + assert!( + missing.is_empty(), + "docs/formats/README.md is out of date; its table rows should be:\n{}", + rows.join("\n") + ); + // The status words the CLI matrix would print agree with the cells. + for target in WRITABLE { + let r = report(target); + for class in [Severity::Content, Severity::Accounting, Severity::Metadata] { + let word = status(&r, class); + let c = cell(&r, class); + assert_eq!(word == "ok", c == "ok", "{target} {class:?}: {word} vs {c}"); + } + } +} + +// ── generated transcripts ────────────────────────────────────────────── + +#[derive(Debug, Clone)] +enum Step { + UserText(String), + UserImage, + Reply { + thinking: Option<(String, Option)>, + text: String, + usage: Option<(u64, u64)>, + stop: Option, + model: Option, + }, + Call { + tool: Tool, + result: ToolOutput, + is_error: bool, + }, + Command { + name: String, + args: Option, + output: String, + }, +} + +fn arb_text() -> impl Strategy { + prop_oneof![ + 3 => "[a-zA-Z0-9][a-zA-Z0-9 ,.!?<>&'/_-]{0,38}", + 1 => "[a-zA-Z0-9 日本語é🚀\n\t]{1,30}", + 1 => Just(" padded\n".to_string()), + 1 => Just(String::new()), + ] +} + +fn arb_stop() -> impl Strategy { + prop_oneof![ + Just(StopReason::EndTurn), + Just(StopReason::ToolUse), + Just(StopReason::MaxTokens), + Just(StopReason::StopSequence), + Just(StopReason::Aborted), + Just(StopReason::Error), + "[a-z_]{1,8}".prop_map(StopReason::Other), + ] +} + +fn arb_output() -> impl Strategy { + prop_oneof![ + arb_text().prop_map(ToolOutput::Text), + arb_text().prop_map(|t| ToolOutput::Json(json!([{"type": "text", "text": t}]))), + arb_text().prop_map(|t| ToolOutput::Json(json!({"k": t, "n": [1, 2, 3]}))), + ] +} + +fn arb_tool() -> impl Strategy { + prop_oneof![ + (arb_text(), proptest::option::of(arb_text())).prop_map(|(command, description)| { + Tool::Bash { + command, + workdir: None, + timeout_ms: None, + description, + run_in_background: false, + } + }), + ("/repo/[a-z]{1,8}\\.rs", arb_text(), arb_text()).prop_map( + |(file_path, old_string, new_string)| Tool::Edit { + file_path, + old_string, + new_string, + replace_all: false, + } + ), + "/repo/[a-z]{1,8}\\.rs".prop_map(|file_path| Tool::Read { + file_path, + offset: None, + limit: None, + }), + ("mcp__[a-z]{1,5}__[a-z]{1,6}", arb_text()).prop_map(|(tool_name, q)| Tool::Raw { + tool_name, + input: json!({"q": q, "opts": {"b": 1, "a": [true, null]}}), + }), + ] +} + +fn arb_step() -> impl Strategy { + prop_oneof![ + arb_text().prop_map(Step::UserText), + Just(Step::UserImage), + ( + proptest::option::of((arb_text(), proptest::option::of("[a-z0-9]{4,10}"))), + arb_text(), + proptest::option::of((0u64..5000, 0u64..500)), + proptest::option::of(arb_stop()), + proptest::option::of("[a-z0-9-]{3,12}"), + ) + .prop_map(|(thinking, text, usage, stop, model)| Step::Reply { + thinking, + text, + usage, + stop, + model, + }), + (arb_tool(), arb_output(), any::()).prop_map(|(tool, result, is_error)| { + Step::Call { + tool, + result, + is_error, + } + }), + ("/[a-z]{2,8}", proptest::option::of(arb_text()), arb_text()) + .prop_map(|(name, args, output)| Step::Command { name, args, output }), + ] +} + +fn message(role: Role, content: Vec, n: usize) -> Message { + Message { + role, + content, + timestamp: ts(i64::try_from(n).unwrap() + 1), + model: None, + stop_reason: None, + usage: None, + } +} + +fn build(steps: &[Step]) -> Transcript { + let mut t = rich(); + t.body.clear(); + for (i, step) in steps.iter().enumerate() { + let id = format!("c{i}"); + let n = t.body.len(); + match step { + Step::UserText(text) => t.body.push(message( + Role::User, + vec![Block::Text { text: text.clone() }], + n, + )), + Step::UserImage => t.body.push(message( + Role::User, + vec![Block::Image { + source: ImageSource { + source_type: "base64".into(), + media_type: "image/png".into(), + data: "iVBORw0KGgo=".into(), + }, + }], + n, + )), + Step::Reply { + thinking, + text, + usage, + stop, + model, + } => { + let mut content = Vec::new(); + if let Some((text, signature)) = thinking { + content.push(Block::Thinking { + text: text.clone(), + signature: signature.clone(), + encrypted: None, + }); + } + content.push(Block::Text { text: text.clone() }); + let mut reply = message(Role::Assistant, content, n); + reply.model.clone_from(model); + reply.stop_reason.clone_from(stop); + reply.usage = usage.map(|(input_tokens, output_tokens)| Usage { + input_tokens, + output_tokens, + cache_read_input_tokens: None, + cache_creation_input_tokens: None, + }); + t.body.push(reply); + } + Step::Call { + tool, + result, + is_error, + } => { + t.body.push(message( + Role::Assistant, + vec![Block::ToolUse { + id: id.clone(), + tool: tool.clone(), + }], + n, + )); + t.body.push(message( + Role::User, + vec![Block::ToolResult { + tool_use_id: id, + content: result.clone(), + is_error: *is_error, + }], + n + 1, + )); + } + Step::Command { name, args, output } => { + t.body.push(message( + Role::User, + vec![Block::ToolUse { + id: id.clone(), + tool: Tool::Command { + command: name.clone(), + args: args.clone(), + }, + }], + n, + )); + t.body.push(message( + Role::User, + vec![Block::ToolResult { + tool_use_id: id, + content: ToolOutput::Text(output.clone()), + is_error: false, + }], + n + 1, + )); + } + } + } + t +} + +proptest! { + /// Whatever the transcript, assessing it never panics for any target, + /// repeats exactly, and measures the lossless Simple codec as lossless: + /// a false positive from the comparator shows up here. + #[test] + fn assessing_generated_transcripts_is_total_and_stable( + steps in proptest::collection::vec(arb_step(), 0..8) + ) { + let src = build(&steps); + for target in fidelity::TARGETS { + let first = fidelity::assess(target, &src); + let second = fidelity::assess(target, &src); + match (first, second) { + (Ok(a), Ok(b)) => { + prop_assert_eq!(&a, &b); + prop_assert_eq!(a.messages_before, src.body.len()); + if target == HarnessId::Simple { + prop_assert!(a.items.is_empty(), "simple is lossless: {:?}", a.items); + } + } + (Err(a), Err(b)) => prop_assert_eq!(a.to_string(), b.to_string()), + _ => prop_assert!(false, "{target}: flaky result"), + } + } + } +} diff --git a/tests/integration/main.rs b/tests/integration/main.rs index 4eb9cce..47f0786 100644 --- a/tests/integration/main.rs +++ b/tests/integration/main.rs @@ -1,8 +1,8 @@ //! Integration tests: every module exercises public API against real backing //! stores (temp dirs, real `SQLite`) — no mocks. Per-harness modules cover //! store round-trip fidelity, codec fixpoints through Common, and discovery; -//! the rest cover cross-harness conversion, adversarial path handling, -//! search, and deletion. Property-based codec invariants live in +//! the rest cover cross-harness conversion, conversion fidelity, adversarial +//! path handling, search, and deletion. Property-based codec invariants live in //! `properties`. Tests that pin a specific shipped bug belong in //! `tests/regression/`, not here — see `tests/README.md`. @@ -16,6 +16,7 @@ mod cowork; mod cross_harness; mod cursor; mod cursor_desktop; +mod fidelity; mod fx; mod grok; mod grok_bot;