From 3a95c15d09ab8acbe1d434635d013478f87be1f1 Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 24 Aug 2026 11:09:03 +0700 Subject: [PATCH 1/8] fix(impact): a file argument reaches the symbols that depend on it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `impact ` seeded its traversal from the File node and walked inbound CALLS. Call edges attach to symbols, never to files, so a file argument could only ever return "no dependants found" — while `preflight` on the same file, using `symbols_in_file`, correctly reported 199. Two commands contradicting each other about one file is worse than either being silent, because an agent acts on the answer it was given. Seed a file's symbols alongside the file itself, and follow inbound IMPORTS so file-level importers are reported next to symbol-level callers. Symbol arguments are untouched. A file every module imports has hundreds of dependants, so the renderer leads with the count and shows the nearest few as evidence rather than spending an agent's budget to say "a lot". The corpus-wide `impact >= preflight` invariant is satisfiable by IMPORTS edges alone, so the test also pins what actually broke: a file's affected list must contain a caller, which fails without this change. --- crates/reify-cli/src/main.rs | 2 +- crates/reify-cli/src/mcp.rs | 7 +-- crates/reify-cli/src/render.rs | 13 ++++- crates/reify/src/query.rs | 90 +++++++++++++++++++++++++++++----- 4 files changed, 94 insertions(+), 18 deletions(-) diff --git a/crates/reify-cli/src/main.rs b/crates/reify-cli/src/main.rs index 25a5f5c..e4197a7 100644 --- a/crates/reify-cli/src/main.rs +++ b/crates/reify-cli/src/main.rs @@ -96,7 +96,7 @@ enum Command { /// What breaks if this changes. Impact { - /// A symbol name or a description of the change. + /// A symbol name, a file path, or a description of the change. query: String, }, diff --git a/crates/reify-cli/src/mcp.rs b/crates/reify-cli/src/mcp.rs index 91e9114..f29436a 100644 --- a/crates/reify-cli/src/mcp.rs +++ b/crates/reify-cli/src/mcp.rs @@ -117,12 +117,13 @@ fn tool_definitions() -> Vec { json!({ "name": "reify_impact", "description": - "List what depends on a symbol, including through shared database tables \ - where no call edge exists. Call this before changing shared logic.", + "List what depends on a symbol or a file — callers, importers, and \ + coupling through shared database tables where no call edge exists. \ + Call this before changing shared logic.", "inputSchema": { "type": "object", "properties": { - "query": {"type": "string", "description": "A symbol name or a described change"} + "query": {"type": "string", "description": "A symbol name, a file path, or a described change"} }, "required": ["query"] } diff --git a/crates/reify-cli/src/render.rs b/crates/reify-cli/src/render.rs index 2629f74..aff1195 100644 --- a/crates/reify-cli/src/render.rs +++ b/crates/reify-cli/src/render.rs @@ -532,7 +532,18 @@ pub fn impact(answer: &ImpactAnswer, json: bool) -> Result<()> { } } if !answer.affected.is_empty() { - heading("Affected"); + // A file every module imports has hundreds of dependants. Printing all of them + // spends an agent's budget to say one thing — "a lot" — so the count leads and + // the nearest few are the evidence for it. + let shown = answer.affected.len(); + if answer.affected_total > shown { + heading(&format!( + "Affected {} total, {shown} nearest shown", + answer.affected_total + )); + } else { + heading(&format!("Affected {shown}")); + } for item in &answer.affected { println!( " {} {} {} ({}, {} hop{})", diff --git a/crates/reify/src/query.rs b/crates/reify/src/query.rs index b32b6e0..0fd6b1d 100644 --- a/crates/reify/src/query.rs +++ b/crates/reify/src/query.rs @@ -229,7 +229,12 @@ pub struct ImpactAnswer { pub schema: &'static str, pub query: String, pub origins: Vec, + /// Everything found to depend on the origins. Truncated for presentation; + /// `affected_total` is what was actually found. pub affected: Vec, + /// How many dependants were found before the list was truncated. Naming sixty of + /// two hundred is a sample, and calling it the answer would be a lie about scope. + pub affected_total: usize, pub tables: Vec, pub co_changing_files: Vec, pub unknowns: Vec, @@ -247,6 +252,7 @@ pub fn impact(store: &Store, query: &str) -> Result { query: query.to_string(), origins: origins.iter().map(citation).collect(), affected: Vec::new(), + affected_total: 0, tables: Vec::new(), co_changing_files: Vec::new(), unknowns: Vec::new(), @@ -258,30 +264,52 @@ pub fn impact(store: &Store, query: &str) -> Result { return Ok(answer); } - let origin_ids: HashSet = origins.iter().map(|n| n.id).collect(); + // A file argument is the common case from an editor hook, and a file is not the + // node dependencies attach to: `CALLS` edges land on symbols, `IMPORTS` on files. + // Seeding a file's symbols alongside the file itself is what makes `impact ` + // agree with `preflight ` instead of contradicting it. + let mut seeds: Vec = origins.clone(); + for origin in &origins { + if origin.kind != NodeKind::File { + continue; + } + if let Some(path) = &origin.path { + seeds.extend(store.symbols_in_file(path)?); + } + } + let origin_ids: HashSet = seeds.iter().map(|n| n.id).collect(); let mut seen: HashSet = origin_ids.clone(); - let mut frontier: Vec<(Node, u32, String)> = origins - .iter() - .cloned() - .map(|n| (n, 0, String::new())) - .collect(); + let mut frontier: Vec<(Node, u32, String)> = + seeds.into_iter().map(|n| (n, 0, String::new())).collect(); while let Some((node, depth, _)) = frontier.pop() { - if depth >= IMPACT_MAX_DEPTH || answer.affected.len() >= IMPACT_MAX_NODES { + if depth >= IMPACT_MAX_DEPTH { continue; } - // Callers depend on this symbol. - for (dependant, _, confidence) in - store.neighbors(node.id, Direction::In, &[EdgeKind::Calls])? - { + // Past the display budget, keep counting but stop widening: the marginal + // second-hop name costs tokens without changing what an engineer decides. + let widen = answer.affected.len() < IMPACT_MAX_NODES; + // Callers depend on this symbol; importers depend on this file. + for (dependant, edge, confidence) in store.neighbors( + node.id, + Direction::In, + &[EdgeKind::Calls, EdgeKind::Imports], + )? { if !seen.insert(dependant.id) { continue; } - let reason = format!("calls {}", node.name); + let verb = if edge == EdgeKind::Imports { + "imports" + } else { + "calls" + }; + let reason = format!("{verb} {}", node.name); answer .affected .push(affected(&dependant, depth + 1, reason, confidence)); - frontier.push((dependant, depth + 1, String::new())); + if widen { + frontier.push((dependant, depth + 1, String::new())); + } } // Data coupling: anything else touching a table this symbol writes. for (table, _, _) in store.neighbors( @@ -332,6 +360,7 @@ pub fn impact(store: &Store, query: &str) -> Result { .then(b.confidence.total_cmp(&a.confidence)) .then(a.location.cmp(&b.location)) }); + answer.affected_total = answer.affected.len(); answer.affected.truncate(IMPACT_MAX_NODES); if answer.affected.is_empty() { @@ -1311,6 +1340,41 @@ class SalesOrder: let _ = fs::remove_dir_all(&root); } + #[test] + fn impact_on_a_file_never_reports_less_than_preflight_on_the_same_file() { + // Two commands contradicting each other about one file is worse than either + // being silent: an agent acts on the answer it was given. `impact` counts + // intra-file callers that `preflight` deliberately excludes, so it may report + // more — never fewer. + let (store, root) = indexed(); + let mut checked = 0; + for file in store.nodes_of_kind(NodeKind::File).unwrap() { + let path = file.path.as_deref().unwrap_or(&file.name); + let expected = preflight(&store, path).unwrap().dependants; + let found = impact(&store, path).unwrap().affected_total; + assert!( + found >= expected, + "impact({path}) found {found} affected, preflight found {expected} dependants" + ); + checked += 1; + } + assert!(checked > 0, "the fixture corpus indexed no files"); + + // The invariant above is satisfiable by file-level imports alone, so pin the + // thing that actually broke: a file argument must reach the symbols that call + // into it, the way a symbol argument does. + let answer = impact(&store, "app/order.py").unwrap(); + assert!( + answer + .affected + .iter() + .any(|a| a.reason.starts_with("calls")), + "a file's callers must be reported, not only its importers: {:?}", + answer.affected + ); + let _ = fs::remove_dir_all(&root); + } + #[test] fn impact_on_an_unknown_query_says_so_rather_than_inventing() { let (store, root) = indexed(); From 6aa58ba404a50e5fb0f57704929f48e02c293de0 Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 24 Aug 2026 11:09:18 +0700 Subject: [PATCH 2/8] fix(rules): match a rule's subject on whole words, as its polarity already does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `classify_phrase` tested the subject with a raw substring match while the polarity test on the next line used the word-boundaried `contains_word`. The asymmetry read `must-revalidate` as the `validation` subject — `validate` sits inside `revalidate` — with `must` supplying the polarity, so an HTTP cache header was mined as a business rule at 0.97 confidence. `contains_word` already falls back to substring for multi-word needles like "credit limit" and for non-ASCII ones, so the multilingual corpus is unaffected. Django's mined rules go 180 -> 152. All 30 losses were inspected: every one is a test method name, a mangled documentation snippet, or the same bug in French via premise/remise. Two genuine rules are newly gained. Known and deliberately out of scope: the `validation` subject lacks the inflections `approval` has, so prose like "the service validates the order" is now missed where the loose match caught it incidentally. Restoring those inflections would also restore the test-name noise this removes; that is a tuning decision about the subject table, not part of this bug. --- crates/reify/src/rules.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/reify/src/rules.rs b/crates/reify/src/rules.rs index 1dd4661..b0c5e94 100644 --- a/crates/reify/src/rules.rs +++ b/crates/reify/src/rules.rs @@ -688,7 +688,10 @@ fn subject_vocabulary(name: &str) -> BTreeSet { fn classify_phrase(phrase: &str) -> Option<(String, Polarity)> { let lowered = phrase.to_lowercase(); for subject in SUBJECTS { - if !subject.terms.iter().any(|t| lowered.contains(t)) { + // Word-boundaried, like the polarity test below. A raw substring match reads + // `must-revalidate` as the `validation` subject, because `validate` sits inside + // `revalidate` — and a cache header then arrives as a business rule at 0.97. + if !subject.terms.iter().any(|t| contains_word(&lowered, t)) { continue; } let signals = |words: &[&str], shared: &[&str]| { @@ -842,6 +845,23 @@ mod tests { assert_eq!(classify_phrase("returns a list of rows"), None); } + #[test] + fn a_subject_term_inside_a_longer_word_is_not_that_subject() { + // `validate` sits inside `revalidate`, and `must` supplies the polarity, so a + // substring subject test mined an HTTP cache header as a validation rule at + // 0.97 confidence. The subject must be word-boundaried like the polarity is. + assert_eq!(classify_phrase("adds a must-revalidate header"), None); + // A genuine claim is untouched. + assert_eq!( + classify_phrase("orders must require approval"), + Some(("approval".into(), Polarity::Require)) + ); + assert_eq!( + classify_phrase("the service rejects an order that fails validation"), + Some(("validation".into(), Polarity::Require)) + ); + } + #[test] fn agglutinative_and_unspaced_scripts_still_match() { // Korean attaches particles to the stem and Thai has no word boundaries, so From a9b32f7903e9f18f974c04593d96af9de63d7db8 Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 24 Aug 2026 11:09:18 +0700 Subject: [PATCH 3/8] fix(bench): the prompt names the repository the task came from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `prompt()` hardcoded "(ERPNext)" with no parameter, so every model-in-the-loop run against Medusa, OFBiz and OpenMRS told the model it was working on ERPNext while asking about a different codebase in a different language. It does not obviously favour one arm — every condition shared the same wrong name, and the prompt-differs-only-in-context test still holds — but it is a validity defect in three published tables. The name now comes from `TaskSet::repository`. The three affected reports carry a dated note stating the defect and its scope rather than being silently regenerated from runs that were not redone. --- benchmarks/REPORT-medusa.md | 18 +++++++++++++++++ benchmarks/REPORT-ofbiz.md | 18 +++++++++++++++++ benchmarks/REPORT-openmrs.md | 18 +++++++++++++++++ crates/reify-bench/src/agent.rs | 34 +++++++++++++++++++++++++++------ 4 files changed, 82 insertions(+), 6 deletions(-) diff --git a/benchmarks/REPORT-medusa.md b/benchmarks/REPORT-medusa.md index d88c1af..0dba335 100644 --- a/benchmarks/REPORT-medusa.md +++ b/benchmarks/REPORT-medusa.md @@ -1,5 +1,23 @@ # Reify Brownfield Benchmark +> **Correction, 2026-08-24** — added by hand; the rest of this file is generated. +> +> The *With a model in the loop* section below was produced with a prompt that named +> the wrong repository. `reify-bench`'s prompt template hard-coded the literal +> `(ERPNext)`, so every row of that table told the model it was working on ERPNext +> while asking it about **medusa**. Everything above that section is model-free and +> is unaffected. +> +> **Scope.** The misnaming sits in the preamble every condition shares, so it does +> not obviously favour one arm, and the test that the prompt differs between +> conditions only in its CONTEXT block still holds. It is a validity defect all the +> same: those hit rates are not what a correctly-prompted model would produce, and +> should not be read as such. The table is left exactly as measured rather than +> regenerated, because re-running it needs the provider and the pinned checkout. +> +> Fixed in `crates/reify-bench/src/agent.rs`: `prompt()` now takes the repository +> name from `TaskSet::repository`. + Generated by `reify-bench report`. Every number here is computed from `outcomes.json` in the same directory; nothing is entered by hand. diff --git a/benchmarks/REPORT-ofbiz.md b/benchmarks/REPORT-ofbiz.md index bfef115..eddbf9d 100644 --- a/benchmarks/REPORT-ofbiz.md +++ b/benchmarks/REPORT-ofbiz.md @@ -1,5 +1,23 @@ # Reify Brownfield Benchmark +> **Correction, 2026-08-24** — added by hand; the rest of this file is generated. +> +> The *With a model in the loop* section below was produced with a prompt that named +> the wrong repository. `reify-bench`'s prompt template hard-coded the literal +> `(ERPNext)`, so every row of that table told the model it was working on ERPNext +> while asking it about **ofbiz**. Everything above that section is model-free and +> is unaffected. +> +> **Scope.** The misnaming sits in the preamble every condition shares, so it does +> not obviously favour one arm, and the test that the prompt differs between +> conditions only in its CONTEXT block still holds. It is a validity defect all the +> same: those hit rates are not what a correctly-prompted model would produce, and +> should not be read as such. The table is left exactly as measured rather than +> regenerated, because re-running it needs the provider and the pinned checkout. +> +> Fixed in `crates/reify-bench/src/agent.rs`: `prompt()` now takes the repository +> name from `TaskSet::repository`. + Generated by `reify-bench report`. Every number here is computed from `outcomes.json` in the same directory; nothing is entered by hand. diff --git a/benchmarks/REPORT-openmrs.md b/benchmarks/REPORT-openmrs.md index 0f55612..9e23ce7 100644 --- a/benchmarks/REPORT-openmrs.md +++ b/benchmarks/REPORT-openmrs.md @@ -1,5 +1,23 @@ # Reify Brownfield Benchmark +> **Correction, 2026-08-24** — added by hand; the rest of this file is generated. +> +> The *With a model in the loop* section below was produced with a prompt that named +> the wrong repository. `reify-bench`'s prompt template hard-coded the literal +> `(ERPNext)`, so every row of that table told the model it was working on ERPNext +> while asking it about **openmrs**. Everything above that section is model-free and +> is unaffected. +> +> **Scope.** The misnaming sits in the preamble every condition shares, so it does +> not obviously favour one arm, and the test that the prompt differs between +> conditions only in its CONTEXT block still holds. It is a validity defect all the +> same: those hit rates are not what a correctly-prompted model would produce, and +> should not be read as such. The table is left exactly as measured rather than +> regenerated, because re-running it needs the provider and the pinned checkout. +> +> Fixed in `crates/reify-bench/src/agent.rs`: `prompt()` now takes the repository +> name from `TaskSet::repository`. + Generated by `reify-bench report`. Every number here is computed from `outcomes.json` in the same directory; nothing is entered by hand. diff --git a/crates/reify-bench/src/agent.rs b/crates/reify-bench/src/agent.rs index 17c396b..a6dc923 100644 --- a/crates/reify-bench/src/agent.rs +++ b/crates/reify-bench/src/agent.rs @@ -57,9 +57,14 @@ pub struct AgentOutcome { /// /// Identical across conditions except for the CONTEXT block, so any difference in /// outcome is attributable to the context and not to the wording of the question. -pub fn prompt(task: &Task, context_block: &str) -> String { +/// +/// `repository` comes from the task set rather than from a literal. It was a literal +/// — `ERPNext` — until 2026-08-24, which meant every Medusa, OFBiz and OpenMRS run +/// told the model it was working on a different codebase than the one it was asked +/// about. See the dated notes at the top of the three affected reports. +pub fn prompt(repository: &str, task: &Task, context_block: &str) -> String { format!( - "You are helping a developer change a large existing codebase (ERPNext).\n\ + "You are helping a developer change a large existing codebase ({repository}).\n\ \n\ TASK: {}\n\ \n\ @@ -111,11 +116,12 @@ pub fn oracle_block(task: &Task) -> String { pub fn run( provider: &Provider, root: &Path, + repository: &str, task: &Task, condition: &str, context_block: &str, ) -> AgentOutcome { - let text = prompt(task, context_block); + let text = prompt(repository, task, context_block); let started = std::time::Instant::now(); let mut outcome = AgentOutcome { task: task.id.clone(), @@ -268,8 +274,8 @@ mod tests { #[test] fn the_prompt_differs_between_conditions_only_in_its_context() { - let a = prompt(&task(), "context A"); - let b = prompt(&task(), "context B"); + let a = prompt("erpnext", &task(), "context A"); + let b = prompt("erpnext", &task(), "context B"); let strip = |s: &str| s.replace("context A", "@").replace("context B", "@"); assert_eq!( strip(&a), @@ -278,9 +284,25 @@ mod tests { ); } + #[test] + fn the_prompt_names_the_repository_the_task_came_from() { + // Until 2026-08-24 this was the literal `ERPNext`, so three published + // model-in-the-loop tables asked about one codebase while naming another. + let set = crate::tasks::TaskSet { + repository: ".bench/medusa".into(), + head: "a".repeat(40), + generated_from_commits: 1, + base: None, + tasks: vec![task()], + }; + let text = prompt(set.repository_name(), &task(), "context"); + assert!(text.contains("medusa"), "{text}"); + assert!(!text.contains("ERPNext"), "{text}"); + } + #[test] fn an_empty_context_is_stated_rather_than_left_blank() { - assert!(prompt(&task(), "").contains("(none provided)")); + assert!(prompt("erpnext", &task(), "").contains("(none provided)")); } #[test] From c011eb82c05a4da6ada0e8f8863696b6d4e783a0 Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 24 Aug 2026 11:10:46 +0700 Subject: [PATCH 4/8] feat(bench): a model-free benchmark for detecting an incomplete patch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `reify verify` — a post-flight check that reads an agent's diff and reports what the patch missed — is only worth building if the call graph can support it. This measures that before the feature is written. For each qualifying merged commit the parent tree is indexed, one file's only hunk is withheld, and the truncated patch goes to the checker. The same commit then goes through complete, where every finding is a false positive by construction. That negative control is what stops the metric rewarding a checker that simply shouts. The checker is the shipped graph query reached through `query::impact`, not a new one, so this measures the substrate the decision needs. The pre-registered condition — recall below 0.25 or false alarms above 0.1 per commit means do not build — was written into metrics.rs before the first run. It fires on all three repositories. Recall is 0.50 on Rust, 0.10 on Python and 0.40 on Go, but false alarms run 4.4 to 23.5 per already-complete commit. It fails on noise, not blindness: a CALLS edge says a caller exists, not that the caller needed changing, and nothing in the graph separates a changed signature from an edit inside a body. No rewrite around that edge removes it. `reify verify` is therefore not built. 116s, no model, no network. AGENTS.md records the verdict so it is not rebuilt without beating these numbers first. --- AGENTS.md | 39 + CLAUDE.md | 2 + benchmarks/REPORT-verify.md | 144 + .../verify-django/verify-environment.json | 25 + .../verify-django/verify-outcomes.json | 663 ++++ .../results/verify-django/verify-summary.json | 36 + .../results/verify-django/verify-tasks.json | 2386 ++++++++++++++ .../verify-gh-cli/verify-environment.json | 25 + .../verify-gh-cli/verify-outcomes.json | 560 ++++ .../results/verify-gh-cli/verify-summary.json | 36 + .../results/verify-gh-cli/verify-tasks.json | 2881 +++++++++++++++++ .../verify-reify/verify-environment.json | 25 + .../results/verify-reify/verify-outcomes.json | 270 ++ .../results/verify-reify/verify-summary.json | 33 + .../results/verify-reify/verify-tasks.json | 852 +++++ crates/reify-bench/src/conditions.rs | 275 +- crates/reify-bench/src/main.rs | 666 +++- crates/reify-bench/src/metrics.rs | 392 ++- crates/reify-bench/src/tasks.rs | 586 +++- docs/metrics.md | 22 + 20 files changed, 9862 insertions(+), 56 deletions(-) create mode 100644 AGENTS.md create mode 100644 CLAUDE.md create mode 100644 benchmarks/REPORT-verify.md create mode 100644 benchmarks/results/verify-django/verify-environment.json create mode 100644 benchmarks/results/verify-django/verify-outcomes.json create mode 100644 benchmarks/results/verify-django/verify-summary.json create mode 100644 benchmarks/results/verify-django/verify-tasks.json create mode 100644 benchmarks/results/verify-gh-cli/verify-environment.json create mode 100644 benchmarks/results/verify-gh-cli/verify-outcomes.json create mode 100644 benchmarks/results/verify-gh-cli/verify-summary.json create mode 100644 benchmarks/results/verify-gh-cli/verify-tasks.json create mode 100644 benchmarks/results/verify-reify/verify-environment.json create mode 100644 benchmarks/results/verify-reify/verify-outcomes.json create mode 100644 benchmarks/results/verify-reify/verify-summary.json create mode 100644 benchmarks/results/verify-reify/verify-tasks.json diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..9094603 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,39 @@ +# Project agent memory + +This file is the project's committed home for project-intrinsic agent knowledge: build, test, release, architecture, and sharp-edge notes that should travel with the code. + +## Checks + +`cargo fmt --all`, `cargo clippy --workspace --all-targets` (CI treats warnings as +errors) and `cargo test --workspace` must all be clean. CI additionally runs the test +suite with network egress blocked — `crates/reify/tests/offline.rs` fails the build if a +networking crate enters the dependency tree. + +## The standard this repository is held to + +`crates/reify-bench` is the most load-bearing thing here, and its value is its +intellectual honesty rather than its numbers: steel-manned baselines, falsification +conditions written down *before* the run, Wilson intervals reported next to the +admission that they overlap, provider failures excluded rather than scored as misses, +and one rule from `metrics.rs` — *a metric that cannot be defined precisely does not get +reported*. Metric definitions live in `docs/metrics.md`. Reports are **generated**, never +hand-edited; a number that was not re-measured is corrected with a dated note rather than +silently regenerated (see the top of `benchmarks/REPORT-medusa.md`). + +When a fitted parameter fails held-out validation, the fit is published and the default +reverts — `HISTORY_PRIOR_WEIGHT` in `crates/reify/src/context.rs` is the worked example. + +## Decisions already measured + +`reify verify` — a post-flight check reporting what an agent's patch missed — was +measured before being written and **failed** its pre-registered condition on Rust, Python +and Go. `benchmarks/REPORT-verify.md` has the numbers; `reify-bench verify-eval` +reproduces them in about two minutes with no model. Do not build it on the `CALLS` graph +alone without re-running that benchmark and beating those numbers. + +## Maintaining this file + +Keep this file for knowledge useful to almost every future agent session in this project. +Do not repeat what the codebase already shows; point to the authoritative file or command instead. +Prefer rewriting or pruning existing entries over appending new ones. +When updating this file, preserve this bar for all agents and keep entries concise. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..a9d4d26 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,2 @@ + +@AGENTS.md diff --git a/benchmarks/REPORT-verify.md b/benchmarks/REPORT-verify.md new file mode 100644 index 0000000..a781d4c --- /dev/null +++ b/benchmarks/REPORT-verify.md @@ -0,0 +1,144 @@ +# Can the graph tell that a patch is incomplete? + +Generated by `reify-bench verify-report`. Every number is computed from the `verify-summary.json` files named below; nothing is entered by hand. + +This benchmark exists to decide one thing: whether `reify verify` — a post-flight check that reads an agent's diff and reports what the patch missed — is worth building on Reify's call graph. It is model-free, deterministic, and costs nothing per run. + +## Construction + +For each merged commit that passes the retrieval benchmark's filters and touches at least two indexable files: + +1. the parent tree is extracted and indexed, so the change is absent from the index by construction; +2. one file's **only** hunk is withheld — the *omission*. Removing it removes that file from the patch entirely, so a citation of it cannot be an echo of a hunk still present. Among the files with exactly one hunk, the last by path order is chosen; the choice is arbitrary, fixed, and made before any checker runs; +3. the truncated patch goes to the checker; +4. **the same commit goes to the checker complete.** A merged commit is complete by definition, so every finding there is a false positive. This control is not optional: without it the metric would reward a checker that simply shouts. + +The checker is not `reify verify`, which does not exist. It is the shipped graph query — *symbols changed by this diff, minus symbols present in the diff, where an inbound `CALLS` edge exists at distance 1* — reached through `reify::query::impact`. That deliberately measures the **substrate**, which is the number the decision needs. + +## Pre-registered falsification condition + +> If `omission_recall` on this substrate is below **0.25**, or `false_alarm_rate` is above **0.1 per commit**, the `reify verify` feature does not get built on this substrate. + +Stated in `crates/reify-bench/src/metrics.rs` before the first run and not moved since. A result that kills the feature is a result. + +## Results + +| Metric | reify | django | gh-cli | +|---|---:|---:|---:| +| Most indexed language | rust | python | go | +| Trials | 4 | 20 | 20 | +| `omission_recall` | **0.50** (0.15–0.85) | **0.10** (0.03–0.30) | **0.40** (0.22–0.61) | +| …attributable to the omission | 0.00 (0.00–0.49) | 0.05 (0.01–0.24) | 0.15 (0.05–0.36) | +| `omission_recall_symbol` | — (0 scorable) | 0.12 (0.03–0.36) over 16 | 0.31 (0.14–0.56) over 16 | +| Omitted files a caller query *could* cite | 3/4 | 20/20 | 19/20 | +| `false_alarm_rate` (per complete commit) | **23.5** | **6.9** | **4.4** | +| Complete commits with ≥1 false alarm | 4/4 (0.51–1.00) | 18/20 (0.70–0.97) | 15/20 (0.53–0.89) | +| `findings_per_diff` (median) | 19 | 1 | 2 | +| `verify_tokens` (median) | 422 | 36 | 48 | +| `verify_latency_ms` (median) | 1 | 1 | 0 | +| Index per trial, ms (median) | 151 | 4732 | 809 | +| Whole run, wall clock | 1s | 97s | 18s | +| Pre-registered verdict | **do not build** | **do not build** | **do not build** | + +## What the numbers say + +**reify** — false_alarm_rate 23.50 > 0.10 + +**django** — omission_recall 0.10 < 0.25; false_alarm_rate 6.90 > 0.10 + +**gh-cli** — false_alarm_rate 4.45 > 0.10 + +Every repository fails the pre-registered condition, so **`reify verify` does not get built on this substrate**. The condition was written down before the first run precisely so this outcome could not be argued away afterwards. + +**It fails on noise, not on blindness.** 3 of 3 repositories exceed the false-alarm ceiling; 1 of 3 fall below the recall floor (a repository can fail both). The graph does find the omitted file often enough to be interesting; what it cannot do is stay quiet about a patch that is already complete. + +**The negative control takes most of the headline back.** `omission_recall` counts a citation of the omitted file whether or not the complete commit is cited too. The attributable row counts only citations the complete commit does *not* produce, and it is the smaller number in every repository here. The gap is the checker citing a file it would have cited anyway — which is not detection, however it reads next to the label. + +**The ceiling is not what binds.** A finding is a caller, so the omitted file can only be cited if something in it calls out of itself. In the least favourable repository here that holds for 75% of omissions, so the edges mostly exist and `omission_recall` is not capped by their absence. The gap between that row and the recall row is a *ranking* gap, not a coverage one. + +**The noise is structural, not marginal.** `false_alarm_rate` is findings per commit that is complete by construction. A `CALLS` edge says a caller exists; it does not say the caller needed changing. Nothing in the graph distinguishes a changed signature from an edit inside a body, so every caller of every touched symbol is a candidate. That is a property of the edge, and no rewriting of the query around the same edge removes it. + +## Cost and determinism + +No model, no network, no provider key: the whole run is a git extract, an index and a graph query. Total wall clock for everything in this report is **116s**, dominated by re-indexing one parent tree per trial. The query itself is the `verify_latency_ms` row — single-digit milliseconds. + +Each run is deterministic given a fixed `HEAD`: task selection, the omission rule and the query contain no randomness and no tunable threshold. A run against a repository whose history is still moving — this one, for instance — should pin the window with `--until `, or the trial set moves with the branch. + +```bash +reify-bench verify-eval --repo --out results/verify- --until +reify-bench verify-report --results "name=results/verify-" --out benchmarks/REPORT-verify.md +``` + +## Limitations + +1. **Small samples.** The intervals are wide and are printed beside every rate. Where two repositories differ by less than their intervals, they have not been shown to differ. +2. **The omission-selection rule has a direction.** "Last by path order, among files with exactly one hunk" is arbitrary but not neutral: in a repository laid out as `src/` and `tests/`, path order lands on `tests/`. Counted across every run here, 17 of 44 omissions sit under a path segment named `test` or `tests`. The rule was fixed before any run and has not been changed since; every omitted file is named in the appendix, so the effect is checkable rather than described. +3. **`CALLS` at distance 1 only.** `impact` also propagates two hops and crosses into the data layer. Widening the query would raise recall and raise the false-alarm rate with it — the trade this benchmark measures rather than pre-empts. +4. **A checker, not the feature.** `reify verify` could use a signature diff, type information, or the model. This measures the substrate those would all stand on. +5. **Parent trees are extracted with `git archive`**, so the indexed tree has no git history and no co-change edges. The checker uses neither; a checker that did would need re-measuring. +6. **Ground truth is one commit's hunks.** A change that could correctly have been made elsewhere scores as a miss. +7. **`impact`'s own bounds are inherited, not bypassed.** It stops at 60 affected nodes and walks depth-first to two hops, so on a widely-called symbol some direct callers can be crowded out by second-hop ones. That is the shipped query's behaviour and measuring around it would measure something that does not exist. + +## Appendix: every trial + +`could cite` is whether the omitted file calls out of itself at all — the ceiling for that trial. `cited` is findings on the truncated patch, `noise` is findings on the same commit complete. + +### reify (`git@github.com:lambiengcode/reify.git`, commit `0b0bcf5cf5fc25f4c7325f108a0474e1d2895cda`) + +| Trial | Omitted file | could cite | hit | attributable | cited | noise | +|---|---|---|---|---|---:|---:| +| `v-9af59e47` | `crates/reify/tests/fixtures.rs` | yes | yes | no | 19 | 19 | +| `v-2b7bad4c` | `assets/make-logo.py` | no | no | no | 1 | 1 | +| `v-deb46ef8` | `crates/reify-cli/src/render.rs` | yes | no | no | 4 | 4 | +| `v-7dd36dae` | `crates/reify-bench/src/conditions.rs` | yes | yes | no | 70 | 70 | + +### django (`git@github.com:django/django`, commit `0b40210e4808937a7c0922e8b7502bff4752faa3`) + +| Trial | Omitted file | could cite | hit | attributable | cited | noise | +|---|---|---|---|---|---:|---:| +| `v-d992705f` | `tests/basic/models.py` | yes | no | no | 1 | 1 | +| `v-be6cf832` | `django/db/models/sql/query.py` | yes | no | no | 1 | 13 | +| `v-c72f5fb4` | `tests/validators/tests.py` | yes | no | no | 5 | 5 | +| `v-1a001208` | `tests/ordering/tests.py` | yes | no | no | 0 | 0 | +| `v-07d4f69c` | `tests/migrations/test_operations.py` | yes | no | no | 0 | 0 | +| `v-febefb17` | `django/db/models/base.py` | yes | no | no | 1 | 2 | +| `v-082b3df4` | `tests/admin_changelist/tests.py` | yes | no | no | 1 | 1 | +| `v-6df8fe3b` | `tests/admin_changelist/tests.py` | yes | no | no | 3 | 3 | +| `v-616e8c52` | `tests/admin_views/tests.py` | yes | no | no | 1 | 1 | +| `v-89e82866` | `tests/admin_views/tests.py` | yes | no | no | 1 | 1 | +| `v-47511a21` | `tests/admin_utils/tests.py` | yes | yes | no | 15 | 14 | +| `v-27137e65` | `django/test/signals.py` | yes | no | no | 6 | 6 | +| `v-94653491` | `tests/field_defaults/models.py` | yes | no | no | 2 | 2 | +| `v-ca14173f` | `tests/urlpatterns/tests.py` | yes | no | no | 1 | 1 | +| `v-c9ff757a` | `tests/bulk_create/tests.py` | yes | no | no | 1 | 1 | +| `v-2936a0a9` | `tests/admin_views/tests.py` | yes | no | no | 10 | 10 | +| `v-92e1d9e3` | `tests/admin_views/tests.py` | yes | no | no | 1 | 1 | +| `v-92470ad3` | `tests/admin_utils/tests.py` | yes | yes | yes | 11 | 10 | +| `v-4ea38d54` | `django/contrib/admin/options.py` | yes | no | no | 61 | 61 | +| `v-6fc81500` | `django/core/handlers/exception.py` | yes | no | no | 5 | 5 | + +### gh-cli (`git@github.com:cli/cli`, commit `5d3c4817f1619213951dbf15031bad04acb88392`) + +| Trial | Omitted file | could cite | hit | attributable | cited | noise | +|---|---|---|---|---|---:|---:| +| `v-e4efbc42` | `pkg/cmd/copilot/copilot_test.go` | yes | yes | yes | 2 | 1 | +| `v-1e04dab8` | `pkg/cmd/copilot/copilot.go` | yes | no | no | 0 | 1 | +| `v-2e9fedd3` | `git/client_test.go` | yes | yes | no | 7 | 7 | +| `v-a6bcd08d` | `pkg/cmd/project/item-add/item_add.go` | yes | no | no | 0 | 1 | +| `v-efe3f165` | `pkg/cmd/project/shared/queries/resolve_fields_test.go` | yes | yes | yes | 29 | 28 | +| `v-688751de` | `pkg/cmd/pr/checkout/checkout_test.go` | yes | yes | no | 2 | 2 | +| `v-9f14d1ac` | `pkg/cmd/pr/checkout/checkout_test.go` | yes | yes | no | 3 | 2 | +| `v-d5f4bed3` | `pkg/cmd/skills/update/update.go` | yes | no | no | 2 | 5 | +| `v-74e77914` | `internal/codespaces/connection/connection.go` | yes | no | no | 14 | 14 | +| `v-f1d11210` | `pkg/cmd/skills/install/install_test.go` | yes | no | no | 2 | 2 | +| `v-751dc5e0` | `pkg/cmd/release/shared/fetch.go` | yes | no | no | 0 | 6 | +| `v-517dae6a` | `internal/skills/registry/registry_test.go` | yes | no | no | 0 | 0 | +| `v-8d2b059e` | `pkg/cmd/discussion/view/view.go` | yes | no | no | 0 | 0 | +| `v-2618999b` | `pkg/cmd/discussion/client/client_impl_test.go` | yes | no | no | 0 | 0 | +| `v-e2d150da` | `pkg/cmd/discussion/edit/edit.go` | yes | no | no | 1 | 2 | +| `v-c1f3c1a1` | `pkg/cmd/discussion/view/view.go` | yes | no | no | 0 | 0 | +| `v-b1029009` | `pkg/cmd/discussion/edit/edit.go` | yes | yes | no | 5 | 5 | +| `v-16a20347` | `pkg/cmd/skills/update/update_test.go` | yes | yes | yes | 2 | 1 | +| `v-fb748cb2` | `pkg/cmd/skills/preview/preview_test.go` | yes | yes | no | 13 | 12 | +| `v-a44721d2` | `internal/prompter/echo_linux_test.go` | no | no | no | 0 | 0 | + diff --git a/benchmarks/results/verify-django/verify-environment.json b/benchmarks/results/verify-django/verify-environment.json new file mode 100644 index 0000000..7147bf0 --- /dev/null +++ b/benchmarks/results/verify-django/verify-environment.json @@ -0,0 +1,25 @@ +{ + "after": null, + "candidates_rejected": 3, + "checker": "symbols changed by this diff, minus symbols present in the diff, where an inbound CALLS edge exists at distance 1, via reify::query::impact", + "count": 20, + "head": "0b40210e4808937a7c0922e8b7502bff4752faa3", + "languages": [ + [ + "python", + 2014 + ], + [ + "javascript", + 16 + ] + ], + "origin": "git@github.com:django/django", + "reify_version": "0.2.2", + "repository": "/private/tmp/claude-501/-Users-lambiengcode--treehouse-reify-2e416f-2-reify/a9d50d0b-cce2-4027-a129-e64262e800a7/scratchpad/django", + "scan": 400, + "token_counts": "estimated by reify heuristic-v1", + "trials": 20, + "until": null, + "wall_clock_ms": 96781 +} \ No newline at end of file diff --git a/benchmarks/results/verify-django/verify-outcomes.json b/benchmarks/results/verify-django/verify-outcomes.json new file mode 100644 index 0000000..6adbaa0 --- /dev/null +++ b/benchmarks/results/verify-django/verify-outcomes.json @@ -0,0 +1,663 @@ +[ + { + "task": "v-d992705f", + "commit": "d992705f9eb56199dc474b77af16474e5ce3d2ab", + "omission_file": "tests/basic/models.py", + "omission_symbol": null, + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": null, + "findings": 1, + "false_alarms": 1, + "changed_symbols": 5, + "verify_tokens": 23, + "verify_latency_ms": 1, + "index_ms": 4264, + "cited": [ + "django/__init__.py:8" + ], + "cited_on_complete": [ + "django/__init__.py:8" + ] + }, + { + "task": "v-be6cf832", + "commit": "be6cf832293779d8aeaeddca8c47a37ba1530898", + "omission_file": "django/db/models/sql/query.py", + "omission_symbol": "django/db/models/sql/query.py:1775", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 1, + "false_alarms": 13, + "changed_symbols": 3, + "verify_tokens": 28, + "verify_latency_ms": 1, + "index_ms": 4761, + "cited": [ + "django/db/models/sql/compiler.py:757" + ], + "cited_on_complete": [ + "django/db/models/sql/compiler.py:757", + "django/db/models/sql/query.py:1344", + "django/db/models/sql/query.py:1891", + "django/db/models/sql/query.py:2327", + "django/db/models/sql/query.py:2588", + "tests/composite_pk/test_names_to_path.py:114", + "tests/composite_pk/test_names_to_path.py:18", + "tests/composite_pk/test_names_to_path.py:27", + "tests/composite_pk/test_names_to_path.py:51", + "tests/composite_pk/test_names_to_path.py:78", + "tests/composite_pk/test_names_to_path.py:9", + "tests/queries/test_query.py:193", + "tests/queries/test_query.py:203" + ] + }, + { + "task": "v-c72f5fb4", + "commit": "c72f5fb4793f2cac66d76e3cbd590499ad85b89e", + "omission_file": "tests/validators/tests.py", + "omission_symbol": null, + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": null, + "findings": 5, + "false_alarms": 5, + "changed_symbols": 1, + "verify_tokens": 121, + "verify_latency_ms": 0, + "index_ms": 4727, + "cited": [ + "django/contrib/admin/utils.py:433", + "django/db/models/fields/__init__.py:2720", + "django/forms/fields.py:772", + "tests/forms_tests/tests/test_validators.py:127", + "tests/forms_tests/tests/test_validators.py:75" + ], + "cited_on_complete": [ + "django/contrib/admin/utils.py:433", + "django/db/models/fields/__init__.py:2720", + "django/forms/fields.py:772", + "tests/forms_tests/tests/test_validators.py:127", + "tests/forms_tests/tests/test_validators.py:75" + ] + }, + { + "task": "v-1a001208", + "commit": "1a001208b0b9d79b15b27ca04d94f31f3d55d5ea", + "omission_file": "tests/ordering/tests.py", + "omission_symbol": "tests/ordering/tests.py:729", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 0, + "false_alarms": 0, + "changed_symbols": 2, + "verify_tokens": 17, + "verify_latency_ms": 0, + "index_ms": 4732, + "cited": [], + "cited_on_complete": [] + }, + { + "task": "v-07d4f69c", + "commit": "07d4f69c94a0e32c583b3aee5daf48fd81b4cd69", + "omission_file": "tests/migrations/test_operations.py", + "omission_symbol": "tests/migrations/test_operations.py:2576", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 0, + "false_alarms": 0, + "changed_symbols": 4, + "verify_tokens": 17, + "verify_latency_ms": 1, + "index_ms": 5098, + "cited": [], + "cited_on_complete": [] + }, + { + "task": "v-febefb17", + "commit": "febefb175e03352e5aeb2ed827024bacab96cf16", + "omission_file": "django/db/models/base.py", + "omission_symbol": "django/db/models/base.py:1553", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 1, + "false_alarms": 2, + "changed_symbols": 3, + "verify_tokens": 36, + "verify_latency_ms": 1, + "index_ms": 4756, + "cited": [ + "tests/validation/test_unique.py:85" + ], + "cited_on_complete": [ + "django/db/models/base.py:1469", + "tests/validation/test_unique.py:85" + ] + }, + { + "task": "v-082b3df4", + "commit": "082b3df4067c3899dd4d57e8c2eca5baea9d07bb", + "omission_file": "tests/admin_changelist/tests.py", + "omission_symbol": "tests/admin_changelist/tests.py:1810", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 1, + "false_alarms": 1, + "changed_symbols": 2, + "verify_tokens": 33, + "verify_latency_ms": 0, + "index_ms": 4691, + "cited": [ + "django/contrib/admin/templatetags/admin_list.py:334" + ], + "cited_on_complete": [ + "django/contrib/admin/templatetags/admin_list.py:334" + ] + }, + { + "task": "v-6df8fe3b", + "commit": "6df8fe3bc1879265958b8e59c637a4145995e93c", + "omission_file": "tests/admin_changelist/tests.py", + "omission_symbol": "tests/admin_changelist/tests.py:1796", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 3, + "false_alarms": 3, + "changed_symbols": 1, + "verify_tokens": 65, + "verify_latency_ms": 0, + "index_ms": 5047, + "cited": [ + "tests/model_inheritance/tests.py:217", + "tests/model_inheritance/tests.py:667", + "tests/model_inheritance/tests.py:677" + ], + "cited_on_complete": [ + "tests/model_inheritance/tests.py:217", + "tests/model_inheritance/tests.py:667", + "tests/model_inheritance/tests.py:677" + ] + }, + { + "task": "v-616e8c52", + "commit": "616e8c52ded7f4c7b00cae5a95f5a5d12a6a39b9", + "omission_file": "tests/admin_views/tests.py", + "omission_symbol": "tests/admin_views/tests.py:576", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 1, + "false_alarms": 1, + "changed_symbols": 1, + "verify_tokens": 31, + "verify_latency_ms": 0, + "index_ms": 4650, + "cited": [ + "django/contrib/admin/options.py:2048" + ], + "cited_on_complete": [ + "django/contrib/admin/options.py:2048" + ] + }, + { + "task": "v-89e82866", + "commit": "89e82866dc2746383c336c7b10e050b9da3ae1ef", + "omission_file": "tests/admin_views/tests.py", + "omission_symbol": "tests/admin_views/tests.py:3144", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 1, + "false_alarms": 1, + "changed_symbols": 1, + "verify_tokens": 32, + "verify_latency_ms": 0, + "index_ms": 4735, + "cited": [ + "django/contrib/admin/options.py:2048" + ], + "cited_on_complete": [ + "django/contrib/admin/options.py:2048" + ] + }, + { + "task": "v-47511a21", + "commit": "47511a21026cdd721d8fbf8571cc079bc38bb46d", + "omission_file": "tests/admin_utils/tests.py", + "omission_symbol": "tests/admin_utils/tests.py:243", + "omission_file_reachable": true, + "file_hit": true, + "file_hit_attributable": false, + "symbol_hit": true, + "findings": 15, + "false_alarms": 14, + "changed_symbols": 1, + "verify_tokens": 388, + "verify_latency_ms": 0, + "index_ms": 4693, + "cited": [ + "django/contrib/admin/helpers.py:269", + "django/contrib/admin/templatetags/admin_list.py:205", + "tests/admin_utils/tests.py:132", + "tests/admin_utils/tests.py:181", + "tests/admin_utils/tests.py:199", + "tests/admin_utils/tests.py:205", + "tests/admin_utils/tests.py:210", + "tests/admin_utils/tests.py:220", + "tests/admin_utils/tests.py:235", + "tests/admin_utils/tests.py:243", + "tests/admin_utils/tests.py:260", + "tests/admin_utils/tests.py:277", + "tests/admin_utils/tests.py:285", + "tests/postgres_tests/test_array.py:1550", + "tests/postgres_tests/test_array.py:1559" + ], + "cited_on_complete": [ + "django/contrib/admin/helpers.py:269", + "django/contrib/admin/templatetags/admin_list.py:205", + "tests/admin_utils/tests.py:132", + "tests/admin_utils/tests.py:181", + "tests/admin_utils/tests.py:199", + "tests/admin_utils/tests.py:205", + "tests/admin_utils/tests.py:210", + "tests/admin_utils/tests.py:220", + "tests/admin_utils/tests.py:235", + "tests/admin_utils/tests.py:260", + "tests/admin_utils/tests.py:277", + "tests/admin_utils/tests.py:285", + "tests/postgres_tests/test_array.py:1550", + "tests/postgres_tests/test_array.py:1559" + ] + }, + { + "task": "v-27137e65", + "commit": "27137e655e442e81095f1f8f77ff3870d9fdf169", + "omission_file": "django/test/signals.py", + "omission_symbol": "django/test/signals.py:146", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 6, + "false_alarms": 6, + "changed_symbols": 3, + "verify_tokens": 140, + "verify_latency_ms": 1, + "index_ms": 4656, + "cited": [ + "django/utils/formats.py:62", + "django/utils/translation/trans_real.py:495", + "django/utils/translation/trans_real.py:564", + "django/views/i18n.py:30", + "tests/i18n/tests.py:2066", + "tests/i18n/tests.py:2168" + ], + "cited_on_complete": [ + "django/utils/formats.py:62", + "django/utils/translation/trans_real.py:495", + "django/utils/translation/trans_real.py:564", + "django/views/i18n.py:30", + "tests/i18n/tests.py:2066", + "tests/i18n/tests.py:2168" + ] + }, + { + "task": "v-94653491", + "commit": "9465349120ef8a0b0689e12bcbfd05f3d173ebdf", + "omission_file": "tests/field_defaults/models.py", + "omission_symbol": null, + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": null, + "findings": 2, + "false_alarms": 2, + "changed_symbols": 2, + "verify_tokens": 44, + "verify_latency_ms": 0, + "index_ms": 5012, + "cited": [ + "django/db/models/base.py:1022", + "django/db/models/base.py:950" + ], + "cited_on_complete": [ + "django/db/models/base.py:1022", + "django/db/models/base.py:950" + ] + }, + { + "task": "v-ca14173f", + "commit": "ca14173f968cf36115f22d6c6785f738de4391ed", + "omission_file": "tests/urlpatterns/tests.py", + "omission_symbol": "tests/urlpatterns/tests.py:443", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 1, + "false_alarms": 1, + "changed_symbols": 1, + "verify_tokens": 30, + "verify_latency_ms": 0, + "index_ms": 4749, + "cited": [ + "django/urls/utils.py:199" + ], + "cited_on_complete": [ + "django/urls/utils.py:199" + ] + }, + { + "task": "v-c9ff757a", + "commit": "c9ff757a55392b1f50968eb89fe775f6155168d8", + "omission_file": "tests/bulk_create/tests.py", + "omission_symbol": "tests/bulk_create/tests.py:442", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 1, + "false_alarms": 1, + "changed_symbols": 1, + "verify_tokens": 31, + "verify_latency_ms": 1, + "index_ms": 4688, + "cited": [ + "django/db/models/query.py:817" + ], + "cited_on_complete": [ + "django/db/models/query.py:817" + ] + }, + { + "task": "v-2936a0a9", + "commit": "2936a0a99719e3c3777039a0d6968deecb55c752", + "omission_file": "tests/admin_views/tests.py", + "omission_symbol": "tests/admin_views/tests.py:802", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 10, + "false_alarms": 10, + "changed_symbols": 5, + "verify_tokens": 225, + "verify_latency_ms": 1, + "index_ms": 4754, + "cited": [ + "django/contrib/admin/helpers.py:204", + "django/contrib/admin/helpers.py:381", + "django/contrib/admin/templatetags/admin_list.py:88", + "django/contrib/admin/views/main.py:375", + "django/contrib/admin/views/main.py:424", + "tests/admin_utils/tests.py:353", + "tests/admin_utils/tests.py:399", + "tests/admin_utils/tests.py:420", + "tests/admin_utils/tests.py:436", + "tests/admin_utils/tests.py:458" + ], + "cited_on_complete": [ + "django/contrib/admin/helpers.py:204", + "django/contrib/admin/helpers.py:381", + "django/contrib/admin/templatetags/admin_list.py:88", + "django/contrib/admin/views/main.py:375", + "django/contrib/admin/views/main.py:424", + "tests/admin_utils/tests.py:353", + "tests/admin_utils/tests.py:399", + "tests/admin_utils/tests.py:420", + "tests/admin_utils/tests.py:436", + "tests/admin_utils/tests.py:458" + ] + }, + { + "task": "v-92e1d9e3", + "commit": "92e1d9e3619ae5274a64b38f26177064486892f2", + "omission_file": "tests/admin_views/tests.py", + "omission_symbol": null, + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": null, + "findings": 1, + "false_alarms": 1, + "changed_symbols": 3, + "verify_tokens": 34, + "verify_latency_ms": 0, + "index_ms": 4666, + "cited": [ + "django/contrib/admin/templatetags/admin_list.py:348" + ], + "cited_on_complete": [ + "django/contrib/admin/templatetags/admin_list.py:348" + ] + }, + { + "task": "v-92470ad3", + "commit": "92470ad3742524902b29769d2c822dbe791630db", + "omission_file": "tests/admin_utils/tests.py", + "omission_symbol": "tests/admin_utils/tests.py:132", + "omission_file_reachable": true, + "file_hit": true, + "file_hit_attributable": true, + "symbol_hit": true, + "findings": 11, + "false_alarms": 10, + "changed_symbols": 3, + "verify_tokens": 217, + "verify_latency_ms": 1, + "index_ms": 4680, + "cited": [ + "django/contrib/admin/helpers.py:269", + "django/contrib/admin/templatetags/admin_list.py:326", + "django/contrib/sites/management.py:11", + "tests/admin_utils/tests.py:132", + "tests/contenttypes_tests/test_views.py:29", + "tests/gis_tests/geoapp/test_feeds.py:16", + "tests/gis_tests/geoapp/test_sitemaps.py:18", + "tests/sites_tests/tests.py:143", + "tests/sites_tests/tests.py:196", + "tests/sites_tests/tests.py:24", + "tests/sites_tests/tests.py:311" + ], + "cited_on_complete": [ + "django/contrib/admin/helpers.py:269", + "django/contrib/admin/templatetags/admin_list.py:326", + "django/contrib/sites/management.py:11", + "tests/contenttypes_tests/test_views.py:29", + "tests/gis_tests/geoapp/test_feeds.py:16", + "tests/gis_tests/geoapp/test_sitemaps.py:18", + "tests/sites_tests/tests.py:143", + "tests/sites_tests/tests.py:196", + "tests/sites_tests/tests.py:24", + "tests/sites_tests/tests.py:311" + ] + }, + { + "task": "v-4ea38d54", + "commit": "4ea38d54c10e0f44e189605c217a55cdfe9fdde8", + "omission_file": "django/contrib/admin/options.py", + "omission_symbol": "django/contrib/admin/options.py:2558", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 61, + "false_alarms": 61, + "changed_symbols": 4, + "verify_tokens": 1315, + "verify_latency_ms": 2, + "index_ms": 4639, + "cited": [ + "django/contrib/admin/sites.py:325", + "tests/admin_checks/tests.py:1024", + "tests/admin_checks/tests.py:1042", + "tests/admin_checks/tests.py:1049", + "tests/admin_checks/tests.py:294", + "tests/admin_checks/tests.py:309", + "tests/admin_checks/tests.py:324", + "tests/admin_checks/tests.py:340", + "tests/admin_checks/tests.py:365", + "tests/admin_checks/tests.py:382", + "tests/admin_checks/tests.py:398", + "tests/admin_checks/tests.py:405", + "tests/admin_checks/tests.py:413", + "tests/admin_checks/tests.py:435", + "tests/admin_checks/tests.py:456", + "tests/admin_checks/tests.py:474", + "tests/admin_checks/tests.py:489", + "tests/admin_checks/tests.py:508", + "tests/admin_checks/tests.py:533", + "tests/admin_checks/tests.py:548", + "tests/admin_checks/tests.py:570", + "tests/admin_checks/tests.py:594", + "tests/admin_checks/tests.py:618", + "tests/admin_checks/tests.py:642", + "tests/admin_checks/tests.py:666", + "tests/admin_checks/tests.py:681", + "tests/admin_checks/tests.py:699", + "tests/admin_checks/tests.py:718", + "tests/admin_checks/tests.py:729", + "tests/admin_checks/tests.py:741", + "tests/admin_checks/tests.py:748", + "tests/admin_checks/tests.py:759", + "tests/admin_checks/tests.py:770", + "tests/admin_checks/tests.py:787", + "tests/admin_checks/tests.py:794", + "tests/admin_checks/tests.py:810", + "tests/admin_checks/tests.py:827", + "tests/admin_checks/tests.py:842", + "tests/admin_checks/tests.py:853", + "tests/admin_checks/tests.py:860", + "tests/admin_checks/tests.py:881", + "tests/admin_checks/tests.py:900", + "tests/admin_checks/tests.py:907", + "tests/admin_checks/tests.py:914", + "tests/admin_checks/tests.py:930", + "tests/admin_checks/tests.py:946", + "tests/admin_checks/tests.py:967", + "tests/admin_checks/tests.py:982", + "tests/admin_checks/tests.py:999", + "tests/admin_registration/tests.py:21", + "tests/admin_views/test_adminsite.py:106", + "tests/admin_views/tests.py:9338", + "tests/generic_inline_admin/tests.py:314", + "tests/generic_inline_admin/tests.py:344", + "tests/generic_inline_admin/tests.py:419", + "tests/generic_inline_admin/tests.py:422", + "tests/modeladmin/test_actions.py:30", + "tests/modeladmin/test_actions.py:67", + "tests/modeladmin/test_actions.py:89", + "tests/modeladmin/test_checks.py:18", + "tests/modeladmin/test_checks.py:1811" + ], + "cited_on_complete": [ + "django/contrib/admin/sites.py:325", + "tests/admin_checks/tests.py:1024", + "tests/admin_checks/tests.py:1042", + "tests/admin_checks/tests.py:1049", + "tests/admin_checks/tests.py:294", + "tests/admin_checks/tests.py:309", + "tests/admin_checks/tests.py:324", + "tests/admin_checks/tests.py:340", + "tests/admin_checks/tests.py:365", + "tests/admin_checks/tests.py:382", + "tests/admin_checks/tests.py:398", + "tests/admin_checks/tests.py:405", + "tests/admin_checks/tests.py:413", + "tests/admin_checks/tests.py:435", + "tests/admin_checks/tests.py:456", + "tests/admin_checks/tests.py:474", + "tests/admin_checks/tests.py:489", + "tests/admin_checks/tests.py:508", + "tests/admin_checks/tests.py:533", + "tests/admin_checks/tests.py:548", + "tests/admin_checks/tests.py:570", + "tests/admin_checks/tests.py:594", + "tests/admin_checks/tests.py:618", + "tests/admin_checks/tests.py:642", + "tests/admin_checks/tests.py:666", + "tests/admin_checks/tests.py:681", + "tests/admin_checks/tests.py:699", + "tests/admin_checks/tests.py:718", + "tests/admin_checks/tests.py:729", + "tests/admin_checks/tests.py:741", + "tests/admin_checks/tests.py:748", + "tests/admin_checks/tests.py:759", + "tests/admin_checks/tests.py:770", + "tests/admin_checks/tests.py:787", + "tests/admin_checks/tests.py:794", + "tests/admin_checks/tests.py:810", + "tests/admin_checks/tests.py:827", + "tests/admin_checks/tests.py:842", + "tests/admin_checks/tests.py:853", + "tests/admin_checks/tests.py:860", + "tests/admin_checks/tests.py:881", + "tests/admin_checks/tests.py:900", + "tests/admin_checks/tests.py:907", + "tests/admin_checks/tests.py:914", + "tests/admin_checks/tests.py:930", + "tests/admin_checks/tests.py:946", + "tests/admin_checks/tests.py:967", + "tests/admin_checks/tests.py:982", + "tests/admin_checks/tests.py:999", + "tests/admin_registration/tests.py:21", + "tests/admin_views/test_adminsite.py:106", + "tests/admin_views/tests.py:9338", + "tests/generic_inline_admin/tests.py:314", + "tests/generic_inline_admin/tests.py:344", + "tests/generic_inline_admin/tests.py:419", + "tests/generic_inline_admin/tests.py:422", + "tests/modeladmin/test_actions.py:30", + "tests/modeladmin/test_actions.py:67", + "tests/modeladmin/test_actions.py:89", + "tests/modeladmin/test_checks.py:18", + "tests/modeladmin/test_checks.py:1811" + ] + }, + { + "task": "v-6fc81500", + "commit": "6fc8150005256db2052b01812d65dff737563a1b", + "omission_file": "django/core/handlers/exception.py", + "omission_symbol": "django/core/handlers/exception.py:41", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 5, + "false_alarms": 5, + "changed_symbols": 6, + "verify_tokens": 113, + "verify_latency_ms": 1, + "index_ms": 4772, + "cited": [ + "django/contrib/staticfiles/handlers.py:103", + "django/core/handlers/asgi.py:228", + "django/test/client.py:220", + "tests/asgi/urls.py:60", + "tests/staticfiles_tests/test_handlers.py:18" + ], + "cited_on_complete": [ + "django/contrib/staticfiles/handlers.py:103", + "django/core/handlers/asgi.py:228", + "django/test/client.py:220", + "tests/asgi/urls.py:60", + "tests/staticfiles_tests/test_handlers.py:18" + ] + } +] \ No newline at end of file diff --git a/benchmarks/results/verify-django/verify-summary.json b/benchmarks/results/verify-django/verify-summary.json new file mode 100644 index 0000000..ec5f410 --- /dev/null +++ b/benchmarks/results/verify-django/verify-summary.json @@ -0,0 +1,36 @@ +{ + "tasks": 20, + "omission_recall": 0.1, + "omission_recall_ci": [ + 0.027865902, + 0.3010382 + ], + "omission_recall_attributable": 0.05, + "omission_recall_attributable_ci": [ + 0.008881226, + 0.2361359 + ], + "reachable_omissions": 20, + "omission_recall_reachable": 0.1, + "omission_recall_reachable_ci": [ + 0.027865902, + 0.3010382 + ], + "symbol_scorable": 16, + "omission_recall_symbol": 0.125, + "omission_recall_symbol_ci": [ + 0.034976766, + 0.3602333 + ], + "false_alarm_rate": 6.9, + "commits_with_a_false_alarm": 18, + "false_alarm_share_ci": [ + 0.69896173, + 0.9721341 + ], + "median_findings_per_diff": 1, + "median_verify_tokens": 36, + "median_verify_latency_ms": 1, + "median_index_ms": 4732, + "diffs_resolving_to_nothing": 0 +} \ No newline at end of file diff --git a/benchmarks/results/verify-django/verify-tasks.json b/benchmarks/results/verify-django/verify-tasks.json new file mode 100644 index 0000000..e61c2ec --- /dev/null +++ b/benchmarks/results/verify-django/verify-tasks.json @@ -0,0 +1,2386 @@ +{ + "repository": "/private/tmp/claude-501/-Users-lambiengcode--treehouse-reify-2e416f-2-reify/a9d50d0b-cce2-4027-a129-e64262e800a7/scratchpad/django", + "head": "0b40210e4808937a7c0922e8b7502bff4752faa3", + "generated_from_commits": 400, + "rejected": [ + [ + "cef2346abf8d6e9e61a5a3599fbcf72163e6a6e5", + "no file changed by exactly one hunk" + ], + [ + "f1949c1f9758947ade984c895ff16bef46f56520", + "no file changed by exactly one hunk" + ], + [ + "60121939f6b225c7a719dd561e372e1d8e5e2c4a", + "no file changed by exactly one hunk" + ] + ], + "tasks": [ + { + "id": "v-d992705f", + "commit": "d992705f9eb56199dc474b77af16474e5ce3d2ab", + "parent": "504d1f12cd5879177295fecd2ba4da1001a0930f", + "date": "2026-08-08", + "prompt": "Fixed #37259 -- Restored support for old-signature Model.from_db overrides.", + "complete": { + "files": [ + { + "path": "django/db/models/query.py", + "created": false, + "hunks": [ + { + "old_start": 6, + "old_len": 7, + "changed_lines": [ + 9 + ] + }, + { + "old_start": 52, + "old_len": 6, + "changed_lines": [ + 55 + ] + }, + { + "old_start": 63, + "old_len": 6, + "changed_lines": [ + 66 + ] + }, + { + "old_start": 123, + "old_len": 6, + "changed_lines": [ + 126 + ] + }, + { + "old_start": 142, + "old_len": 11, + "changed_lines": [ + 145, + 149 + ] + }, + { + "old_start": 213, + "old_len": 13, + "changed_lines": [ + 216, + 220, + 221, + 222 + ] + }, + { + "old_start": 3044, + "old_len": 6, + "changed_lines": [ + 3047 + ] + }, + { + "old_start": 3063, + "old_len": 11, + "changed_lines": [ + 3066, + 3070 + ] + } + ] + }, + { + "path": "tests/basic/models.py", + "created": false, + "hunks": [ + { + "old_start": 61, + "old_len": 3, + "changed_lines": [ + 64 + ] + } + ] + }, + { + "path": "tests/basic/tests.py", + "created": false, + "hunks": [ + { + "old_start": 24, + "old_len": 6, + "changed_lines": [ + 27 + ] + }, + { + "old_start": 31, + "old_len": 6, + "changed_lines": [ + 34 + ] + }, + { + "old_start": 1120, + "old_len": 3, + "changed_lines": [ + 1123 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/db/models/query.py", + "created": false, + "hunks": [ + { + "old_start": 6, + "old_len": 7, + "changed_lines": [ + 9 + ] + }, + { + "old_start": 52, + "old_len": 6, + "changed_lines": [ + 55 + ] + }, + { + "old_start": 63, + "old_len": 6, + "changed_lines": [ + 66 + ] + }, + { + "old_start": 123, + "old_len": 6, + "changed_lines": [ + 126 + ] + }, + { + "old_start": 142, + "old_len": 11, + "changed_lines": [ + 145, + 149 + ] + }, + { + "old_start": 213, + "old_len": 13, + "changed_lines": [ + 216, + 220, + 221, + 222 + ] + }, + { + "old_start": 3044, + "old_len": 6, + "changed_lines": [ + 3047 + ] + }, + { + "old_start": 3063, + "old_len": 11, + "changed_lines": [ + 3066, + 3070 + ] + } + ] + }, + { + "path": "tests/basic/tests.py", + "created": false, + "hunks": [ + { + "old_start": 24, + "old_len": 6, + "changed_lines": [ + 27 + ] + }, + { + "old_start": 31, + "old_len": 6, + "changed_lines": [ + 34 + ] + }, + { + "old_start": 1120, + "old_len": 3, + "changed_lines": [ + 1123 + ] + } + ] + } + ] + }, + "omission_file": "tests/basic/models.py", + "omission_line": 64 + }, + { + "id": "v-be6cf832", + "commit": "be6cf832293779d8aeaeddca8c47a37ba1530898", + "parent": "c72f5fb4793f2cac66d76e3cbd590499ad85b89e", + "date": "2026-08-13", + "prompt": "Fixed #37274 -- Allowed transforms in order_by after alias.", + "complete": { + "files": [ + { + "path": "django/db/models/sql/compiler.py", + "created": false, + "hunks": [ + { + "old_start": 1052, + "old_len": 6, + "changed_lines": [ + 1055 + ] + } + ] + }, + { + "path": "django/db/models/sql/query.py", + "created": false, + "hunks": [ + { + "old_start": 1800, + "old_len": 8, + "changed_lines": [ + 1803, + 1804 + ] + } + ] + }, + { + "path": "tests/annotations/tests.py", + "created": false, + "hunks": [ + { + "old_start": 1481, + "old_len": 6, + "changed_lines": [ + 1484 + ] + }, + { + "old_start": 1536, + "old_len": 9, + "changed_lines": [ + 1539, + 1541 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/db/models/sql/compiler.py", + "created": false, + "hunks": [ + { + "old_start": 1052, + "old_len": 6, + "changed_lines": [ + 1055 + ] + } + ] + }, + { + "path": "tests/annotations/tests.py", + "created": false, + "hunks": [ + { + "old_start": 1481, + "old_len": 6, + "changed_lines": [ + 1484 + ] + }, + { + "old_start": 1536, + "old_len": 9, + "changed_lines": [ + 1539, + 1541 + ] + } + ] + } + ] + }, + "omission_file": "django/db/models/sql/query.py", + "omission_line": 1803 + }, + { + "id": "v-c72f5fb4", + "commit": "c72f5fb4793f2cac66d76e3cbd590499ad85b89e", + "parent": "f7610bda78afb13ee395dfee2445805d8c7ad0f6", + "date": "2026-08-14", + "prompt": "Fixed #37279 -- Rejected null characters in URLValidator.", + "complete": { + "files": [ + { + "path": "django/core/validators.py", + "created": false, + "hunks": [ + { + "old_start": 152, + "old_len": 7, + "changed_lines": [ + 155 + ] + } + ] + }, + { + "path": "tests/validators/tests.py", + "created": false, + "hunks": [ + { + "old_start": 265, + "old_len": 6, + "changed_lines": [ + 268 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/core/validators.py", + "created": false, + "hunks": [ + { + "old_start": 152, + "old_len": 7, + "changed_lines": [ + 155 + ] + } + ] + } + ] + }, + "omission_file": "tests/validators/tests.py", + "omission_line": 268 + }, + { + "id": "v-1a001208", + "commit": "1a001208b0b9d79b15b27ca04d94f31f3d55d5ea", + "parent": "3436cf9bce84bb1f6877ad96819637366b27b719", + "date": "2026-08-14", + "prompt": "Fixed #37278 -- Made QuerySet.totally_ordered understand aliases to pure Col/ColPairs.", + "complete": { + "files": [ + { + "path": "django/db/models/query.py", + "created": false, + "hunks": [ + { + "old_start": 26, + "old_len": 7, + "changed_lines": [ + 29 + ] + }, + { + "old_start": 2048, + "old_len": 7, + "changed_lines": [ + 2051 + ] + }, + { + "old_start": 2058, + "old_len": 7, + "changed_lines": [ + 2061 + ] + }, + { + "old_start": 2068, + "old_len": 18, + "changed_lines": [ + 2071, + 2072, + 2073, + 2074, + 2075, + 2076, + 2077, + 2078, + 2079, + 2082 + ] + }, + { + "old_start": 2097, + "old_len": 7, + "changed_lines": [ + 2100 + ] + } + ] + }, + { + "path": "tests/composite_pk/test_order_by.py", + "created": false, + "hunks": [ + { + "old_start": 70, + "old_len": 3, + "changed_lines": [ + 73 + ] + } + ] + }, + { + "path": "tests/ordering/models.py", + "created": false, + "hunks": [ + { + "old_start": 85, + "old_len": 10, + "changed_lines": [ + 88, + 92 + ] + } + ] + }, + { + "path": "tests/ordering/tests.py", + "created": false, + "hunks": [ + { + "old_start": 726, + "old_len": 6, + "changed_lines": [ + 729 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/db/models/query.py", + "created": false, + "hunks": [ + { + "old_start": 26, + "old_len": 7, + "changed_lines": [ + 29 + ] + }, + { + "old_start": 2048, + "old_len": 7, + "changed_lines": [ + 2051 + ] + }, + { + "old_start": 2058, + "old_len": 7, + "changed_lines": [ + 2061 + ] + }, + { + "old_start": 2068, + "old_len": 18, + "changed_lines": [ + 2071, + 2072, + 2073, + 2074, + 2075, + 2076, + 2077, + 2078, + 2079, + 2082 + ] + }, + { + "old_start": 2097, + "old_len": 7, + "changed_lines": [ + 2100 + ] + } + ] + }, + { + "path": "tests/composite_pk/test_order_by.py", + "created": false, + "hunks": [ + { + "old_start": 70, + "old_len": 3, + "changed_lines": [ + 73 + ] + } + ] + }, + { + "path": "tests/ordering/models.py", + "created": false, + "hunks": [ + { + "old_start": 85, + "old_len": 10, + "changed_lines": [ + 88, + 92 + ] + } + ] + } + ] + }, + "omission_file": "tests/ordering/tests.py", + "omission_line": 729 + }, + { + "id": "v-07d4f69c", + "commit": "07d4f69c94a0e32c583b3aee5daf48fd81b4cd69", + "parent": "4ee04972e7f9163dbdf5a7c36330e3379187e187", + "date": "2026-08-08", + "prompt": "Fixed #37260 -- Made alterations between Python on_delete options noops.", + "complete": { + "files": [ + { + "path": "django/db/models/fields/related.py", + "created": false, + "hunks": [ + { + "old_start": 601, + "old_len": 6, + "changed_lines": [ + 604 + ] + } + ] + }, + { + "path": "tests/migrations/test_operations.py", + "created": false, + "hunks": [ + { + "old_start": 2573, + "old_len": 6, + "changed_lines": [ + 2576 + ] + } + ] + }, + { + "path": "tests/schema/tests.py", + "created": false, + "hunks": [ + { + "old_start": 635, + "old_len": 10, + "changed_lines": [ + 638, + 642 + ] + }, + { + "old_start": 4926, + "old_len": 7, + "changed_lines": [ + 4929 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/db/models/fields/related.py", + "created": false, + "hunks": [ + { + "old_start": 601, + "old_len": 6, + "changed_lines": [ + 604 + ] + } + ] + }, + { + "path": "tests/schema/tests.py", + "created": false, + "hunks": [ + { + "old_start": 635, + "old_len": 10, + "changed_lines": [ + 638, + 642 + ] + }, + { + "old_start": 4926, + "old_len": 7, + "changed_lines": [ + 4929 + ] + } + ] + } + ] + }, + "omission_file": "tests/migrations/test_operations.py", + "omission_line": 2576 + }, + { + "id": "v-febefb17", + "commit": "febefb175e03352e5aeb2ed827024bacab96cf16", + "parent": "812c08bd4e9da7b74ab9ee0db83da58a6da48d19", + "date": "2026-08-14", + "prompt": "Fixed #37248 -- Skipped unique validation of a dynamic DatabaseDefault expression.", + "complete": { + "files": [ + { + "path": "django/db/models/base.py", + "created": false, + "hunks": [ + { + "old_start": 1562, + "old_len": 9, + "changed_lines": [ + 1565, + 1566, + 1567 + ] + } + ] + }, + { + "path": "django/db/models/constraints.py", + "created": false, + "hunks": [ + { + "old_start": 5, + "old_len": 7, + "changed_lines": [ + 8 + ] + }, + { + "old_start": 605, + "old_len": 6, + "changed_lines": [ + 608 + ] + } + ] + }, + { + "path": "tests/constraints/models.py", + "created": false, + "hunks": [ + { + "old_start": 1, + "old_len": 5, + "changed_lines": [ + 2 + ] + }, + { + "old_start": 175, + "old_len": 3, + "changed_lines": [ + 178 + ] + } + ] + }, + { + "path": "tests/constraints/tests.py", + "created": false, + "hunks": [ + { + "old_start": 16, + "old_len": 6, + "changed_lines": [ + 19 + ] + }, + { + "old_start": 1503, + "old_len": 3, + "changed_lines": [ + 1506 + ] + } + ] + }, + { + "path": "tests/validation/models.py", + "created": false, + "hunks": [ + { + "old_start": 2, + "old_len": 7, + "changed_lines": [ + 5 + ] + }, + { + "old_start": 52, + "old_len": 6, + "changed_lines": [ + 55 + ] + } + ] + }, + { + "path": "tests/validation/test_unique.py", + "created": false, + "hunks": [ + { + "old_start": 4, + "old_len": 17, + "changed_lines": [ + 7, + 14, + 18 + ] + }, + { + "old_start": 160, + "old_len": 6, + "changed_lines": [ + 163 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/db/models/constraints.py", + "created": false, + "hunks": [ + { + "old_start": 5, + "old_len": 7, + "changed_lines": [ + 8 + ] + }, + { + "old_start": 605, + "old_len": 6, + "changed_lines": [ + 608 + ] + } + ] + }, + { + "path": "tests/constraints/models.py", + "created": false, + "hunks": [ + { + "old_start": 1, + "old_len": 5, + "changed_lines": [ + 2 + ] + }, + { + "old_start": 175, + "old_len": 3, + "changed_lines": [ + 178 + ] + } + ] + }, + { + "path": "tests/constraints/tests.py", + "created": false, + "hunks": [ + { + "old_start": 16, + "old_len": 6, + "changed_lines": [ + 19 + ] + }, + { + "old_start": 1503, + "old_len": 3, + "changed_lines": [ + 1506 + ] + } + ] + }, + { + "path": "tests/validation/models.py", + "created": false, + "hunks": [ + { + "old_start": 2, + "old_len": 7, + "changed_lines": [ + 5 + ] + }, + { + "old_start": 52, + "old_len": 6, + "changed_lines": [ + 55 + ] + } + ] + }, + { + "path": "tests/validation/test_unique.py", + "created": false, + "hunks": [ + { + "old_start": 4, + "old_len": 17, + "changed_lines": [ + 7, + 14, + 18 + ] + }, + { + "old_start": 160, + "old_len": 6, + "changed_lines": [ + 163 + ] + } + ] + } + ] + }, + "omission_file": "django/db/models/base.py", + "omission_line": 1565 + }, + { + "id": "v-082b3df4", + "commit": "082b3df4067c3899dd4d57e8c2eca5baea9d07bb", + "parent": "6df8fe3bc1879265958b8e59c637a4145995e93c", + "date": "2026-08-10", + "prompt": "Fixed #37270 -- Fixed incorrect values for second-degree relations in ModelAdmin.list_display.", + "complete": { + "files": [ + { + "path": "django/contrib/admin/templatetags/admin_list.py", + "created": false, + "hunks": [ + { + "old_start": 228, + "old_len": 7, + "changed_lines": [ + 231 + ] + }, + { + "old_start": 246, + "old_len": 11, + "changed_lines": [ + 249, + 250, + 253 + ] + } + ] + }, + { + "path": "tests/admin_changelist/models.py", + "created": false, + "hunks": [ + { + "old_start": 47, + "old_len": 6, + "changed_lines": [ + 50 + ] + } + ] + }, + { + "path": "tests/admin_changelist/tests.py", + "created": false, + "hunks": [ + { + "old_start": 1807, + "old_len": 6, + "changed_lines": [ + 1810 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/contrib/admin/templatetags/admin_list.py", + "created": false, + "hunks": [ + { + "old_start": 228, + "old_len": 7, + "changed_lines": [ + 231 + ] + }, + { + "old_start": 246, + "old_len": 11, + "changed_lines": [ + 249, + 250, + 253 + ] + } + ] + }, + { + "path": "tests/admin_changelist/models.py", + "created": false, + "hunks": [ + { + "old_start": 47, + "old_len": 6, + "changed_lines": [ + 50 + ] + } + ] + } + ] + }, + "omission_file": "tests/admin_changelist/tests.py", + "omission_line": 1810 + }, + { + "id": "v-6df8fe3b", + "commit": "6df8fe3bc1879265958b8e59c637a4145995e93c", + "parent": "616e8c52ded7f4c7b00cae5a95f5a5d12a6a39b9", + "date": "2026-08-10", + "prompt": "Fixed #24580 -- Tested FK values with __html__ in ModelAdmin.list_display.", + "complete": { + "files": [ + { + "path": "tests/admin_changelist/models.py", + "created": false, + "hunks": [ + { + "old_start": 23, + "old_len": 6, + "changed_lines": [ + 26 + ] + } + ] + }, + { + "path": "tests/admin_changelist/tests.py", + "created": false, + "hunks": [ + { + "old_start": 1793, + "old_len": 6, + "changed_lines": [ + 1796 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "tests/admin_changelist/models.py", + "created": false, + "hunks": [ + { + "old_start": 23, + "old_len": 6, + "changed_lines": [ + 26 + ] + } + ] + } + ] + }, + "omission_file": "tests/admin_changelist/tests.py", + "omission_line": 1796 + }, + { + "id": "v-616e8c52", + "commit": "616e8c52ded7f4c7b00cae5a95f5a5d12a6a39b9", + "parent": "2b4c88b2ce753a44b5ba867e5e52a20e07b258e6", + "date": "2026-08-08", + "prompt": "Fixed #37264 -- Handled further malformed _source_model values in admin popups.", + "complete": { + "files": [ + { + "path": "django/contrib/admin/options.py", + "created": false, + "hunks": [ + { + "old_start": 1594, + "old_len": 10, + "changed_lines": [ + 1597, + 1599, + 1600 + ] + } + ] + }, + { + "path": "tests/admin_views/tests.py", + "created": false, + "hunks": [ + { + "old_start": 576, + "old_len": 23, + "changed_lines": [ + 579, + 581, + 582, + 583, + 584, + 585, + 586, + 587, + 588, + 589, + 590, + 591, + 592, + 593, + 594, + 595 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/contrib/admin/options.py", + "created": false, + "hunks": [ + { + "old_start": 1594, + "old_len": 10, + "changed_lines": [ + 1597, + 1599, + 1600 + ] + } + ] + } + ] + }, + "omission_file": "tests/admin_views/tests.py", + "omission_line": 579 + }, + { + "id": "v-89e82866", + "commit": "89e82866dc2746383c336c7b10e050b9da3ae1ef", + "parent": "dfc52e53f1d19a2730854d68b602fb4dba8bf0c5", + "date": "2026-03-01", + "prompt": "Fixed #29969 -- Omitted inlines without add permission on save-as-new.", + "complete": { + "files": [ + { + "path": "django/contrib/admin/options.py", + "created": false, + "hunks": [ + { + "old_start": 2646, + "old_len": 6, + "changed_lines": [ + 2649 + ] + } + ] + }, + { + "path": "tests/admin_views/tests.py", + "created": false, + "hunks": [ + { + "old_start": 3141, + "old_len": 6, + "changed_lines": [ + 3144 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/contrib/admin/options.py", + "created": false, + "hunks": [ + { + "old_start": 2646, + "old_len": 6, + "changed_lines": [ + 2649 + ] + } + ] + } + ] + }, + "omission_file": "tests/admin_views/tests.py", + "omission_line": 3144 + }, + { + "id": "v-47511a21", + "commit": "47511a21026cdd721d8fbf8571cc079bc38bb46d", + "parent": "d2e59b77fe18de318a8272c2a7bbc798d84d1d0d", + "date": "2026-07-13", + "prompt": "Fixed CVE-2026-15920 -- Made display_for_field validate URLs before rendering admin links.", + "complete": { + "files": [ + { + "path": "django/contrib/admin/utils.py", + "created": false, + "hunks": [ + { + "old_start": 7, + "old_len": 8, + "changed_lines": [ + 10, + 11 + ] + }, + { + "old_start": 464, + "old_len": 6, + "changed_lines": [ + 467 + ] + } + ] + }, + { + "path": "tests/admin_utils/tests.py", + "created": false, + "hunks": [ + { + "old_start": 240, + "old_len": 6, + "changed_lines": [ + 243 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/contrib/admin/utils.py", + "created": false, + "hunks": [ + { + "old_start": 7, + "old_len": 8, + "changed_lines": [ + 10, + 11 + ] + }, + { + "old_start": 464, + "old_len": 6, + "changed_lines": [ + 467 + ] + } + ] + } + ] + }, + "omission_file": "tests/admin_utils/tests.py", + "omission_line": 243 + }, + { + "id": "v-27137e65", + "commit": "27137e655e442e81095f1f8f77ff3870d9fdf169", + "parent": "f1949c1f9758947ade984c895ff16bef46f56520", + "date": "2026-07-10", + "prompt": "Fixed CVE-2026-15337 -- Mitigated potential DoS in check_for_language.", + "complete": { + "files": [ + { + "path": "django/test/signals.py", + "created": false, + "hunks": [ + { + "old_start": 153, + "old_len": 7, + "changed_lines": [ + 156 + ] + } + ] + }, + { + "path": "django/utils/translation/trans_real.py", + "created": false, + "hunks": [ + { + "old_start": 31, + "old_len": 9, + "changed_lines": [ + 34, + 35, + 36 + ] + }, + { + "old_start": 65, + "old_len": 7, + "changed_lines": [ + 68 + ] + }, + { + "old_start": 462, + "old_len": 19, + "changed_lines": [ + 465, + 472, + 473, + 477 + ] + } + ] + }, + { + "path": "tests/i18n/tests.py", + "created": false, + "hunks": [ + { + "old_start": 59, + "old_len": 7, + "changed_lines": [ + 62 + ] + }, + { + "old_start": 2081, + "old_len": 6, + "changed_lines": [ + 2084 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/utils/translation/trans_real.py", + "created": false, + "hunks": [ + { + "old_start": 31, + "old_len": 9, + "changed_lines": [ + 34, + 35, + 36 + ] + }, + { + "old_start": 65, + "old_len": 7, + "changed_lines": [ + 68 + ] + }, + { + "old_start": 462, + "old_len": 19, + "changed_lines": [ + 465, + 472, + 473, + 477 + ] + } + ] + }, + { + "path": "tests/i18n/tests.py", + "created": false, + "hunks": [ + { + "old_start": 59, + "old_len": 7, + "changed_lines": [ + 62 + ] + }, + { + "old_start": 2081, + "old_len": 6, + "changed_lines": [ + 2084 + ] + } + ] + } + ] + }, + "omission_file": "django/test/signals.py", + "omission_line": 156 + }, + { + "id": "v-94653491", + "commit": "9465349120ef8a0b0689e12bcbfd05f3d173ebdf", + "parent": "8c83e9c0ea39099b213478a893cbebe9faa837ba", + "date": "2026-07-28", + "prompt": "Fixed #37238 -- Prevented fallback to python default for a pk with a db_default.", + "complete": { + "files": [ + { + "path": "django/db/models/base.py", + "created": false, + "hunks": [ + { + "old_start": 1094, + "old_len": 7, + "changed_lines": [ + 1097 + ] + } + ] + }, + { + "path": "tests/field_defaults/models.py", + "created": false, + "hunks": [ + { + "old_start": 68, + "old_len": 3, + "changed_lines": [ + 71 + ] + } + ] + }, + { + "path": "tests/field_defaults/tests.py", + "created": false, + "hunks": [ + { + "old_start": 23, + "old_len": 6, + "changed_lines": [ + 26 + ] + }, + { + "old_start": 137, + "old_len": 6, + "changed_lines": [ + 140 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/db/models/base.py", + "created": false, + "hunks": [ + { + "old_start": 1094, + "old_len": 7, + "changed_lines": [ + 1097 + ] + } + ] + }, + { + "path": "tests/field_defaults/tests.py", + "created": false, + "hunks": [ + { + "old_start": 23, + "old_len": 6, + "changed_lines": [ + 26 + ] + }, + { + "old_start": 137, + "old_len": 6, + "changed_lines": [ + 140 + ] + } + ] + } + ] + }, + "omission_file": "tests/field_defaults/models.py", + "omission_line": 71 + }, + { + "id": "v-ca14173f", + "commit": "ca14173f968cf36115f22d6c6785f738de4391ed", + "parent": "1c5927f04a853c79ac9b098eab92fb328ff9e4ad", + "date": "2026-07-30", + "prompt": "Fixed #37240 -- Fixed simplify_regex with multiple unnamed groups.", + "complete": { + "files": [ + { + "path": "django/urls/utils.py", + "created": false, + "hunks": [ + { + "old_start": 136, + "old_len": 12, + "changed_lines": [ + 139, + 142, + 143, + 144 + ] + } + ] + }, + { + "path": "tests/urlpatterns/tests.py", + "created": false, + "hunks": [ + { + "old_start": 454, + "old_len": 6, + "changed_lines": [ + 457 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/urls/utils.py", + "created": false, + "hunks": [ + { + "old_start": 136, + "old_len": 12, + "changed_lines": [ + 139, + 142, + 143, + 144 + ] + } + ] + } + ] + }, + "omission_file": "tests/urlpatterns/tests.py", + "omission_line": 457 + }, + { + "id": "v-c9ff757a", + "commit": "c9ff757a55392b1f50968eb89fe775f6155168d8", + "parent": "2936a0a99719e3c3777039a0d6968deecb55c752", + "date": "2026-07-27", + "prompt": "Fixed #37234 -- Fixed bulk_create for late-saved related primary keys.", + "complete": { + "files": [ + { + "path": "django/db/models/query.py", + "created": false, + "hunks": [ + { + "old_start": 747, + "old_len": 6, + "changed_lines": [ + 750 + ] + }, + { + "old_start": 757, + "old_len": 7, + "changed_lines": [ + 760 + ] + } + ] + }, + { + "path": "tests/bulk_create/tests.py", + "created": false, + "hunks": [ + { + "old_start": 439, + "old_len": 6, + "changed_lines": [ + 442 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/db/models/query.py", + "created": false, + "hunks": [ + { + "old_start": 747, + "old_len": 6, + "changed_lines": [ + 750 + ] + }, + { + "old_start": 757, + "old_len": 7, + "changed_lines": [ + 760 + ] + } + ] + } + ] + }, + "omission_file": "tests/bulk_create/tests.py", + "omission_line": 442 + }, + { + "id": "v-2936a0a9", + "commit": "2936a0a99719e3c3777039a0d6968deecb55c752", + "parent": "e1feeee45ea8bcd4325554c9b94fcd75fcd8dbdc", + "date": "2026-01-03", + "prompt": "Fixed #27752 -- Fixed ordering by Model.__str__ in the admin.", + "complete": { + "files": [ + { + "path": "django/contrib/admin/utils.py", + "created": false, + "hunks": [ + { + "old_start": 372, + "old_len": 7, + "changed_lines": [ + 375 + ] + } + ] + }, + { + "path": "django/contrib/admin/views/main.py", + "created": false, + "hunks": [ + { + "old_start": 359, + "old_len": 7, + "changed_lines": [ + 362 + ] + } + ] + }, + { + "path": "tests/admin_utils/models.py", + "created": false, + "hunks": [ + { + "old_start": 47, + "old_len": 6, + "changed_lines": [ + 50 + ] + } + ] + }, + { + "path": "tests/admin_utils/tests.py", + "created": false, + "hunks": [ + { + "old_start": 445, + "old_len": 6, + "changed_lines": [ + 448 + ] + } + ] + }, + { + "path": "tests/admin_views/models.py", + "created": false, + "hunks": [ + { + "old_start": 47, + "old_len": 6, + "changed_lines": [ + 50 + ] + } + ] + }, + { + "path": "tests/admin_views/tests.py", + "created": false, + "hunks": [ + { + "old_start": 799, + "old_len": 6, + "changed_lines": [ + 802 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/contrib/admin/utils.py", + "created": false, + "hunks": [ + { + "old_start": 372, + "old_len": 7, + "changed_lines": [ + 375 + ] + } + ] + }, + { + "path": "django/contrib/admin/views/main.py", + "created": false, + "hunks": [ + { + "old_start": 359, + "old_len": 7, + "changed_lines": [ + 362 + ] + } + ] + }, + { + "path": "tests/admin_utils/models.py", + "created": false, + "hunks": [ + { + "old_start": 47, + "old_len": 6, + "changed_lines": [ + 50 + ] + } + ] + }, + { + "path": "tests/admin_utils/tests.py", + "created": false, + "hunks": [ + { + "old_start": 445, + "old_len": 6, + "changed_lines": [ + 448 + ] + } + ] + }, + { + "path": "tests/admin_views/models.py", + "created": false, + "hunks": [ + { + "old_start": 47, + "old_len": 6, + "changed_lines": [ + 50 + ] + } + ] + } + ] + }, + "omission_file": "tests/admin_views/tests.py", + "omission_line": 802 + }, + { + "id": "v-92e1d9e3", + "commit": "92e1d9e3619ae5274a64b38f26177064486892f2", + "parent": "50e5264a458961134d34d6340a00d9a1b269df7a", + "date": "2026-07-27", + "prompt": "Fixed #37233 -- Prevented sort controls for unordered __str__ admin columns.", + "complete": { + "files": [ + { + "path": "django/contrib/admin/templatetags/admin_list.py", + "created": false, + "hunks": [ + { + "old_start": 116, + "old_len": 7, + "changed_lines": [ + 119 + ] + } + ] + }, + { + "path": "tests/admin_changelist/tests.py", + "created": false, + "hunks": [ + { + "old_start": 4, + "old_len": 7, + "changed_lines": [ + 7 + ] + }, + { + "old_start": 115, + "old_len": 6, + "changed_lines": [ + 118 + ] + }, + { + "old_start": 1726, + "old_len": 6, + "changed_lines": [ + 1729 + ] + } + ] + }, + { + "path": "tests/admin_views/tests.py", + "created": false, + "hunks": [ + { + "old_start": 4323, + "old_len": 6, + "changed_lines": [ + 4326 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/contrib/admin/templatetags/admin_list.py", + "created": false, + "hunks": [ + { + "old_start": 116, + "old_len": 7, + "changed_lines": [ + 119 + ] + } + ] + }, + { + "path": "tests/admin_changelist/tests.py", + "created": false, + "hunks": [ + { + "old_start": 4, + "old_len": 7, + "changed_lines": [ + 7 + ] + }, + { + "old_start": 115, + "old_len": 6, + "changed_lines": [ + 118 + ] + }, + { + "old_start": 1726, + "old_len": 6, + "changed_lines": [ + 1729 + ] + } + ] + } + ] + }, + "omission_file": "tests/admin_views/tests.py", + "omission_line": 4326 + }, + { + "id": "v-92470ad3", + "commit": "92470ad3742524902b29769d2c822dbe791630db", + "parent": "09c8b50bc8e59f7ec2d97df1bfb3fbd3ae0d4522", + "date": "2026-07-26", + "prompt": "Fixed #37230 -- Fixed a crash for second-degree relations in ModelAdmin.list_display.", + "complete": { + "files": [ + { + "path": "django/contrib/admin/templatetags/admin_list.py", + "created": false, + "hunks": [ + { + "old_start": 16, + "old_len": 7, + "changed_lines": [ + 19 + ] + }, + { + "old_start": 226, + "old_len": 6, + "changed_lines": [ + 229 + ] + } + ] + }, + { + "path": "django/contrib/admin/utils.py", + "created": false, + "hunks": [ + { + "old_start": 314, + "old_len": 9, + "changed_lines": [ + 317, + 318, + 319 + ] + } + ] + }, + { + "path": "tests/admin_utils/models.py", + "created": false, + "hunks": [ + { + "old_start": 5, + "old_len": 6, + "changed_lines": [ + 8 + ] + } + ] + }, + { + "path": "tests/admin_utils/tests.py", + "created": false, + "hunks": [ + { + "old_start": 164, + "old_len": 6, + "changed_lines": [ + 167 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/contrib/admin/templatetags/admin_list.py", + "created": false, + "hunks": [ + { + "old_start": 16, + "old_len": 7, + "changed_lines": [ + 19 + ] + }, + { + "old_start": 226, + "old_len": 6, + "changed_lines": [ + 229 + ] + } + ] + }, + { + "path": "django/contrib/admin/utils.py", + "created": false, + "hunks": [ + { + "old_start": 314, + "old_len": 9, + "changed_lines": [ + 317, + 318, + 319 + ] + } + ] + }, + { + "path": "tests/admin_utils/models.py", + "created": false, + "hunks": [ + { + "old_start": 5, + "old_len": 6, + "changed_lines": [ + 8 + ] + } + ] + } + ] + }, + "omission_file": "tests/admin_utils/tests.py", + "omission_line": 167 + }, + { + "id": "v-4ea38d54", + "commit": "4ea38d54c10e0f44e189605c217a55cdfe9fdde8", + "parent": "8a162076e1988ddf9453edfb8329d5df1573dc38", + "date": "2026-06-10", + "prompt": "Fixed #37160 -- Made admin views raise PermissionDenied consistently.", + "complete": { + "files": [ + { + "path": "django/contrib/admin/options.py", + "created": false, + "hunks": [ + { + "old_start": 2563, + "old_len": 14, + "changed_lines": [ + 2566, + 2571, + 2572, + 2573 + ] + } + ] + }, + { + "path": "django/contrib/admin/sites.py", + "created": false, + "hunks": [ + { + "old_start": 9, + "old_len": 7, + "changed_lines": [ + 12 + ] + }, + { + "old_start": 256, + "old_len": 10, + "changed_lines": [ + 259, + 260, + 261, + 262 + ] + }, + { + "old_start": 290, + "old_len": 7, + "changed_lines": [ + 293 + ] + }, + { + "old_start": 451, + "old_len": 6, + "changed_lines": [ + 454 + ] + } + ] + }, + { + "path": "tests/admin_views/tests.py", + "created": false, + "hunks": [ + { + "old_start": 3380, + "old_len": 6, + "changed_lines": [ + 3383 + ] + }, + { + "old_start": 3527, + "old_len": 6, + "changed_lines": [ + 3530 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/contrib/admin/sites.py", + "created": false, + "hunks": [ + { + "old_start": 9, + "old_len": 7, + "changed_lines": [ + 12 + ] + }, + { + "old_start": 256, + "old_len": 10, + "changed_lines": [ + 259, + 260, + 261, + 262 + ] + }, + { + "old_start": 290, + "old_len": 7, + "changed_lines": [ + 293 + ] + }, + { + "old_start": 451, + "old_len": 6, + "changed_lines": [ + 454 + ] + } + ] + }, + { + "path": "tests/admin_views/tests.py", + "created": false, + "hunks": [ + { + "old_start": 3380, + "old_len": 6, + "changed_lines": [ + 3383 + ] + }, + { + "old_start": 3527, + "old_len": 6, + "changed_lines": [ + 3530 + ] + } + ] + } + ] + }, + "omission_file": "django/contrib/admin/options.py", + "omission_line": 2566 + }, + { + "id": "v-6fc81500", + "commit": "6fc8150005256db2052b01812d65dff737563a1b", + "parent": "2a5da9d00555beef8e5f6307cfcbfc029d45491e", + "date": "2026-07-18", + "prompt": "Fixed #36027 -- Made error response rendering thread-sensitive.", + "complete": { + "files": [ + { + "path": "django/contrib/staticfiles/handlers.py", + "created": false, + "hunks": [ + { + "old_start": 59, + "old_len": 7, + "changed_lines": [ + 62 + ] + } + ] + }, + { + "path": "django/core/handlers/exception.py", + "created": false, + "hunks": [ + { + "old_start": 43, + "old_len": 7, + "changed_lines": [ + 46 + ] + } + ] + }, + { + "path": "tests/asgi/tests.py", + "created": false, + "hunks": [ + { + "old_start": 30, + "old_len": 7, + "changed_lines": [ + 33 + ] + }, + { + "old_start": 497, + "old_len": 6, + "changed_lines": [ + 500 + ] + } + ] + }, + { + "path": "tests/asgi/urls.py", + "created": false, + "hunks": [ + { + "old_start": 2, + "old_len": 6, + "changed_lines": [ + 5 + ] + }, + { + "old_start": 51, + "old_len": 6, + "changed_lines": [ + 54 + ] + }, + { + "old_start": 73, + "old_len": 5, + "changed_lines": [ + 76 + ] + } + ] + }, + { + "path": "tests/staticfiles_tests/test_handlers.py", + "created": false, + "hunks": [ + { + "old_start": 1, + "old_len": 6, + "changed_lines": [ + 1, + 3 + ] + }, + { + "old_start": 12, + "old_len": 6, + "changed_lines": [ + 15 + ] + }, + { + "old_start": 23, + "old_len": 11, + "changed_lines": [ + 26, + 28, + 31 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "django/contrib/staticfiles/handlers.py", + "created": false, + "hunks": [ + { + "old_start": 59, + "old_len": 7, + "changed_lines": [ + 62 + ] + } + ] + }, + { + "path": "tests/asgi/tests.py", + "created": false, + "hunks": [ + { + "old_start": 30, + "old_len": 7, + "changed_lines": [ + 33 + ] + }, + { + "old_start": 497, + "old_len": 6, + "changed_lines": [ + 500 + ] + } + ] + }, + { + "path": "tests/asgi/urls.py", + "created": false, + "hunks": [ + { + "old_start": 2, + "old_len": 6, + "changed_lines": [ + 5 + ] + }, + { + "old_start": 51, + "old_len": 6, + "changed_lines": [ + 54 + ] + }, + { + "old_start": 73, + "old_len": 5, + "changed_lines": [ + 76 + ] + } + ] + }, + { + "path": "tests/staticfiles_tests/test_handlers.py", + "created": false, + "hunks": [ + { + "old_start": 1, + "old_len": 6, + "changed_lines": [ + 1, + 3 + ] + }, + { + "old_start": 12, + "old_len": 6, + "changed_lines": [ + 15 + ] + }, + { + "old_start": 23, + "old_len": 11, + "changed_lines": [ + 26, + 28, + 31 + ] + } + ] + } + ] + }, + "omission_file": "django/core/handlers/exception.py", + "omission_line": 46 + } + ] +} \ No newline at end of file diff --git a/benchmarks/results/verify-gh-cli/verify-environment.json b/benchmarks/results/verify-gh-cli/verify-environment.json new file mode 100644 index 0000000..71e5b3c --- /dev/null +++ b/benchmarks/results/verify-gh-cli/verify-environment.json @@ -0,0 +1,25 @@ +{ + "after": null, + "candidates_rejected": 23, + "checker": "symbols changed by this diff, minus symbols present in the diff, where an inbound CALLS edge exists at distance 1, via reify::query::impact", + "count": 20, + "head": "5d3c4817f1619213951dbf15031bad04acb88392", + "languages": [ + [ + "go", + 891 + ], + [ + "javascript", + 3 + ] + ], + "origin": "git@github.com:cli/cli", + "reify_version": "0.2.2", + "repository": "/private/tmp/claude-501/-Users-lambiengcode--treehouse-reify-2e416f-2-reify/a9d50d0b-cce2-4027-a129-e64262e800a7/scratchpad/gh-cli", + "scan": 400, + "token_counts": "estimated by reify heuristic-v1", + "trials": 20, + "until": null, + "wall_clock_ms": 17946 +} \ No newline at end of file diff --git a/benchmarks/results/verify-gh-cli/verify-outcomes.json b/benchmarks/results/verify-gh-cli/verify-outcomes.json new file mode 100644 index 0000000..ae0f482 --- /dev/null +++ b/benchmarks/results/verify-gh-cli/verify-outcomes.json @@ -0,0 +1,560 @@ +[ + { + "task": "v-e4efbc42", + "commit": "e4efbc42ccfb1f50c2b97e7b864eb5fc5bcc97f0", + "omission_file": "pkg/cmd/copilot/copilot_test.go", + "omission_symbol": "pkg/cmd/copilot/copilot_test.go:597", + "omission_file_reachable": true, + "file_hit": true, + "file_hit_attributable": true, + "symbol_hit": true, + "findings": 2, + "false_alarms": 1, + "changed_symbols": 1, + "verify_tokens": 47, + "verify_latency_ms": 0, + "index_ms": 778, + "cited": [ + "pkg/cmd/copilot/copilot.go:42", + "pkg/cmd/copilot/copilot_test.go:597" + ], + "cited_on_complete": [ + "pkg/cmd/copilot/copilot.go:42" + ] + }, + { + "task": "v-1e04dab8", + "commit": "1e04dab89cf10a2eab5b238d27fb1d7cb94b8af4", + "omission_file": "pkg/cmd/copilot/copilot.go", + "omission_symbol": "pkg/cmd/copilot/copilot.go:134", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 0, + "false_alarms": 1, + "changed_symbols": 1, + "verify_tokens": 17, + "verify_latency_ms": 0, + "index_ms": 840, + "cited": [], + "cited_on_complete": [ + "pkg/cmd/copilot/copilot.go:42" + ] + }, + { + "task": "v-2e9fedd3", + "commit": "2e9fedd3aaeb9bc5f044f2d825f565d98083fc56", + "omission_file": "git/client_test.go", + "omission_symbol": "git/client_test.go:1429", + "omission_file_reachable": true, + "file_hit": true, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 7, + "false_alarms": 7, + "changed_symbols": 5, + "verify_tokens": 141, + "verify_latency_ms": 1, + "index_ms": 886, + "cited": [ + "git/client_test.go:1316", + "pkg/cmd/issue/develop/develop.go:178", + "pkg/cmd/issue/issue.go:23", + "pkg/cmd/pr/close/close.go:65", + "pkg/cmd/pr/merge/merge.go:380", + "pkg/cmd/pr/merge/merge.go:505", + "pkg/cmd/repo/sync/sync.go:221" + ], + "cited_on_complete": [ + "git/client_test.go:1316", + "pkg/cmd/issue/develop/develop.go:178", + "pkg/cmd/issue/issue.go:23", + "pkg/cmd/pr/close/close.go:65", + "pkg/cmd/pr/merge/merge.go:380", + "pkg/cmd/pr/merge/merge.go:505", + "pkg/cmd/repo/sync/sync.go:221" + ] + }, + { + "task": "v-a6bcd08d", + "commit": "a6bcd08d07d1cbcb17cfce497c0cf261a966f703", + "omission_file": "pkg/cmd/project/item-add/item_add.go", + "omission_symbol": "pkg/cmd/project/item-add/item_add.go:126", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 0, + "false_alarms": 1, + "changed_symbols": 0, + "verify_tokens": 17, + "verify_latency_ms": 0, + "index_ms": 832, + "cited": [], + "cited_on_complete": [ + "pkg/cmd/project/item-add/item_add.go:84" + ] + }, + { + "task": "v-efe3f165", + "commit": "efe3f165dd297c85fff11473dbf586f2d39fbf86", + "omission_file": "pkg/cmd/project/shared/queries/resolve_fields_test.go", + "omission_symbol": "pkg/cmd/project/shared/queries/resolve_fields_test.go:31", + "omission_file_reachable": true, + "file_hit": true, + "file_hit_attributable": true, + "symbol_hit": true, + "findings": 29, + "false_alarms": 28, + "changed_symbols": 7, + "verify_tokens": 618, + "verify_latency_ms": 2, + "index_ms": 809, + "cited": [ + "pkg/cmd/project/field-list/field_list_test.go:186", + "pkg/cmd/project/field-list/field_list_test.go:278", + "pkg/cmd/project/field-list/field_list_test.go:370", + "pkg/cmd/project/field-list/field_list_test.go:452", + "pkg/cmd/project/field-list/field_list_test.go:517", + "pkg/cmd/project/field-list/field_list_test.go:91", + "pkg/cmd/project/item-edit/item_edit.go:263", + "pkg/cmd/project/item-list/item_list_test.go:102", + "pkg/cmd/project/item-list/item_list_test.go:212", + "pkg/cmd/project/item-list/item_list_test.go:319", + "pkg/cmd/project/item-list/item_list_test.go:426", + "pkg/cmd/project/item-list/item_list_test.go:523", + "pkg/cmd/project/item-list/item_list_test.go:632", + "pkg/cmd/project/item-list/item_list_test.go:722", + "pkg/cmd/project/list/list_test.go:174", + "pkg/cmd/project/list/list_test.go:252", + "pkg/cmd/project/list/list_test.go:332", + "pkg/cmd/project/list/list_test.go:402", + "pkg/cmd/project/list/list_test.go:470", + "pkg/cmd/project/list/list_test.go:549", + "pkg/cmd/project/list/list_test.go:599", + "pkg/cmd/project/list/list_test.go:678", + "pkg/cmd/project/list/list_test.go:725", + "pkg/cmd/project/list/list_test.go:772", + "pkg/cmd/project/list/list_test.go:812", + "pkg/cmd/project/list/list_test.go:851", + "pkg/cmd/project/list/list_test.go:891", + "pkg/cmd/project/list/list_test.go:95", + "pkg/cmd/project/shared/queries/resolve_fields_test.go:31" + ], + "cited_on_complete": [ + "pkg/cmd/project/field-list/field_list_test.go:186", + "pkg/cmd/project/field-list/field_list_test.go:278", + "pkg/cmd/project/field-list/field_list_test.go:370", + "pkg/cmd/project/field-list/field_list_test.go:452", + "pkg/cmd/project/field-list/field_list_test.go:517", + "pkg/cmd/project/field-list/field_list_test.go:91", + "pkg/cmd/project/item-edit/item_edit.go:263", + "pkg/cmd/project/item-list/item_list_test.go:102", + "pkg/cmd/project/item-list/item_list_test.go:212", + "pkg/cmd/project/item-list/item_list_test.go:319", + "pkg/cmd/project/item-list/item_list_test.go:426", + "pkg/cmd/project/item-list/item_list_test.go:523", + "pkg/cmd/project/item-list/item_list_test.go:632", + "pkg/cmd/project/item-list/item_list_test.go:722", + "pkg/cmd/project/list/list_test.go:174", + "pkg/cmd/project/list/list_test.go:252", + "pkg/cmd/project/list/list_test.go:332", + "pkg/cmd/project/list/list_test.go:402", + "pkg/cmd/project/list/list_test.go:470", + "pkg/cmd/project/list/list_test.go:549", + "pkg/cmd/project/list/list_test.go:599", + "pkg/cmd/project/list/list_test.go:678", + "pkg/cmd/project/list/list_test.go:725", + "pkg/cmd/project/list/list_test.go:772", + "pkg/cmd/project/list/list_test.go:812", + "pkg/cmd/project/list/list_test.go:851", + "pkg/cmd/project/list/list_test.go:891", + "pkg/cmd/project/list/list_test.go:95" + ] + }, + { + "task": "v-688751de", + "commit": "688751de2ce8d610ce76cd0608930e7912509ed3", + "omission_file": "pkg/cmd/pr/checkout/checkout_test.go", + "omission_symbol": null, + "omission_file_reachable": true, + "file_hit": true, + "file_hit_attributable": false, + "symbol_hit": null, + "findings": 2, + "false_alarms": 2, + "changed_symbols": 1, + "verify_tokens": 50, + "verify_latency_ms": 0, + "index_ms": 812, + "cited": [ + "pkg/cmd/pr/checkout/checkout.go:40", + "pkg/cmd/pr/checkout/checkout_test.go:179" + ], + "cited_on_complete": [ + "pkg/cmd/pr/checkout/checkout.go:40", + "pkg/cmd/pr/checkout/checkout_test.go:179" + ] + }, + { + "task": "v-9f14d1ac", + "commit": "9f14d1ac675f25a75d4b940dc88e733f06398e76", + "omission_file": "pkg/cmd/pr/checkout/checkout_test.go", + "omission_symbol": null, + "omission_file_reachable": true, + "file_hit": true, + "file_hit_attributable": false, + "symbol_hit": null, + "findings": 3, + "false_alarms": 2, + "changed_symbols": 2, + "verify_tokens": 72, + "verify_latency_ms": 0, + "index_ms": 811, + "cited": [ + "pkg/cmd/pr/checkout/checkout.go:40", + "pkg/cmd/pr/checkout/checkout_test.go:1075", + "pkg/cmd/pr/checkout/checkout_test.go:179" + ], + "cited_on_complete": [ + "pkg/cmd/pr/checkout/checkout.go:40", + "pkg/cmd/pr/checkout/checkout_test.go:179" + ] + }, + { + "task": "v-d5f4bed3", + "commit": "d5f4bed3f49dbbc5d5a2e7bb76ba9f25ba7ba574", + "omission_file": "pkg/cmd/skills/update/update.go", + "omission_symbol": "pkg/cmd/skills/update/update.go:66", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 2, + "false_alarms": 5, + "changed_symbols": 3, + "verify_tokens": 50, + "verify_latency_ms": 0, + "index_ms": 804, + "cited": [ + "pkg/cmd/skills/install/install_test.go:30", + "pkg/cmd/skills/skills.go:17" + ], + "cited_on_complete": [ + "pkg/cmd/skills/install/install_test.go:30", + "pkg/cmd/skills/skills.go:17", + "pkg/cmd/skills/update/update_test.go:25", + "pkg/cmd/skills/update/update_test.go:43", + "pkg/cmd/skills/update/update_test.go:54" + ] + }, + { + "task": "v-74e77914", + "commit": "74e779140c472bc380ce57978853480805dc16b7", + "omission_file": "internal/codespaces/connection/connection.go", + "omission_symbol": "internal/codespaces/connection/connection.go:26", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 14, + "false_alarms": 14, + "changed_symbols": 10, + "verify_tokens": 296, + "verify_latency_ms": 2, + "index_ms": 803, + "cited": [ + "internal/codespaces/portforwarder/port_forwarder.go:63", + "internal/codespaces/states.go:40", + "pkg/cmd/codespace/jupyter.go:32", + "pkg/cmd/codespace/logs.go:35", + "pkg/cmd/codespace/ports.go:233", + "pkg/cmd/codespace/ports.go:312", + "pkg/cmd/codespace/ports.go:53", + "pkg/cmd/codespace/ports_test.go:104", + "pkg/cmd/codespace/ports_test.go:14", + "pkg/cmd/codespace/ports_test.go:69", + "pkg/cmd/codespace/ports_test.go:91", + "pkg/cmd/codespace/rebuild.go:43", + "pkg/cmd/codespace/ssh.go:165", + "pkg/cmd/codespace/ssh.go:552" + ], + "cited_on_complete": [ + "internal/codespaces/portforwarder/port_forwarder.go:63", + "internal/codespaces/states.go:40", + "pkg/cmd/codespace/jupyter.go:32", + "pkg/cmd/codespace/logs.go:35", + "pkg/cmd/codespace/ports.go:233", + "pkg/cmd/codespace/ports.go:312", + "pkg/cmd/codespace/ports.go:53", + "pkg/cmd/codespace/ports_test.go:104", + "pkg/cmd/codespace/ports_test.go:14", + "pkg/cmd/codespace/ports_test.go:69", + "pkg/cmd/codespace/ports_test.go:91", + "pkg/cmd/codespace/rebuild.go:43", + "pkg/cmd/codespace/ssh.go:165", + "pkg/cmd/codespace/ssh.go:552" + ] + }, + { + "task": "v-f1d11210", + "commit": "f1d112104821b055bb0c5656f2989f9213db71f6", + "omission_file": "pkg/cmd/skills/install/install_test.go", + "omission_symbol": "pkg/cmd/skills/install/install_test.go:303", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 2, + "false_alarms": 2, + "changed_symbols": 1, + "verify_tokens": 50, + "verify_latency_ms": 0, + "index_ms": 820, + "cited": [ + "pkg/cmd/skills/install/install.go:250", + "pkg/cmd/skills/install/install.go:482" + ], + "cited_on_complete": [ + "pkg/cmd/skills/install/install.go:250", + "pkg/cmd/skills/install/install.go:482" + ] + }, + { + "task": "v-751dc5e0", + "commit": "751dc5e0383f08d1d6a211c97d0479aedb39726b", + "omission_file": "pkg/cmd/release/shared/fetch.go", + "omission_symbol": "pkg/cmd/release/shared/fetch.go:187", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 0, + "false_alarms": 6, + "changed_symbols": 1, + "verify_tokens": 17, + "verify_latency_ms": 0, + "index_ms": 821, + "cited": [], + "cited_on_complete": [ + "pkg/cmd/release/delete-asset/delete_asset.go:59", + "pkg/cmd/release/delete/delete.go:67", + "pkg/cmd/release/download/download.go:136", + "pkg/cmd/release/edit/edit.go:92", + "pkg/cmd/release/upload/upload.go:77", + "pkg/cmd/release/view/view.go:75" + ] + }, + { + "task": "v-517dae6a", + "commit": "517dae6a938d4efe73f1219873ebcc74cf4febe1", + "omission_file": "internal/skills/registry/registry_test.go", + "omission_symbol": "internal/skills/registry/registry_test.go:40", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 0, + "false_alarms": 0, + "changed_symbols": 0, + "verify_tokens": 17, + "verify_latency_ms": 0, + "index_ms": 816, + "cited": [], + "cited_on_complete": [] + }, + { + "task": "v-8d2b059e", + "commit": "8d2b059e07f71c17068f2286617f23286e05e0c0", + "omission_file": "pkg/cmd/discussion/view/view.go", + "omission_symbol": "pkg/cmd/discussion/view/view.go:96", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 0, + "false_alarms": 0, + "changed_symbols": 2, + "verify_tokens": 17, + "verify_latency_ms": 0, + "index_ms": 810, + "cited": [], + "cited_on_complete": [] + }, + { + "task": "v-2618999b", + "commit": "2618999bcb6c85d4554638937dc80c322a76d593", + "omission_file": "pkg/cmd/discussion/client/client_impl_test.go", + "omission_symbol": null, + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": null, + "findings": 0, + "false_alarms": 0, + "changed_symbols": 2, + "verify_tokens": 17, + "verify_latency_ms": 0, + "index_ms": 777, + "cited": [], + "cited_on_complete": [] + }, + { + "task": "v-e2d150da", + "commit": "e2d150da420b8bf84f3097f6fad3bbb715ea1cb4", + "omission_file": "pkg/cmd/discussion/edit/edit.go", + "omission_symbol": "pkg/cmd/discussion/edit/edit.go:125", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 1, + "false_alarms": 2, + "changed_symbols": 2, + "verify_tokens": 30, + "verify_latency_ms": 0, + "index_ms": 777, + "cited": [ + "pkg/cmd/discussion/create/create.go:34" + ], + "cited_on_complete": [ + "pkg/cmd/discussion/create/create.go:34", + "pkg/cmd/discussion/edit/edit.go:42" + ] + }, + { + "task": "v-c1f3c1a1", + "commit": "c1f3c1a164ab67436d525769adbce0fd67dd5e70", + "omission_file": "pkg/cmd/discussion/view/view.go", + "omission_symbol": "pkg/cmd/discussion/view/view.go:95", + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": false, + "findings": 0, + "false_alarms": 0, + "changed_symbols": 1, + "verify_tokens": 17, + "verify_latency_ms": 0, + "index_ms": 801, + "cited": [], + "cited_on_complete": [] + }, + { + "task": "v-b1029009", + "commit": "b1029009dbfbcf4240472097e12614dbc3cdcd19", + "omission_file": "pkg/cmd/discussion/edit/edit.go", + "omission_symbol": "pkg/cmd/discussion/edit/edit.go:125", + "omission_file_reachable": true, + "file_hit": true, + "file_hit_attributable": false, + "symbol_hit": true, + "findings": 5, + "false_alarms": 5, + "changed_symbols": 6, + "verify_tokens": 115, + "verify_latency_ms": 1, + "index_ms": 776, + "cited": [ + "pkg/cmd/auth/shared/git_credential.go:80", + "pkg/cmd/auth/shared/gitcredentials/updater_test.go:43", + "pkg/cmd/auth/shared/gitcredentials/updater_test.go:64", + "pkg/cmd/discussion/create/create.go:34", + "pkg/cmd/discussion/edit/edit.go:125" + ], + "cited_on_complete": [ + "pkg/cmd/auth/shared/git_credential.go:80", + "pkg/cmd/auth/shared/gitcredentials/updater_test.go:43", + "pkg/cmd/auth/shared/gitcredentials/updater_test.go:64", + "pkg/cmd/discussion/create/create.go:34", + "pkg/cmd/discussion/edit/edit.go:42" + ] + }, + { + "task": "v-16a20347", + "commit": "16a20347dd33b8f67abd8990ab0940d35f522233", + "omission_file": "pkg/cmd/skills/update/update_test.go", + "omission_symbol": "pkg/cmd/skills/update/update_test.go:313", + "omission_file_reachable": true, + "file_hit": true, + "file_hit_attributable": true, + "symbol_hit": true, + "findings": 2, + "false_alarms": 1, + "changed_symbols": 1, + "verify_tokens": 48, + "verify_latency_ms": 0, + "index_ms": 753, + "cited": [ + "pkg/cmd/skills/update/update.go:66", + "pkg/cmd/skills/update/update_test.go:313" + ], + "cited_on_complete": [ + "pkg/cmd/skills/update/update.go:66" + ] + }, + { + "task": "v-fb748cb2", + "commit": "fb748cb2bf3a434ff12f5268c9983a65310c6520", + "omission_file": "pkg/cmd/skills/preview/preview_test.go", + "omission_symbol": "pkg/cmd/skills/preview/preview_test.go:112", + "omission_file_reachable": true, + "file_hit": true, + "file_hit_attributable": false, + "symbol_hit": true, + "findings": 13, + "false_alarms": 12, + "changed_symbols": 4, + "verify_tokens": 306, + "verify_latency_ms": 1, + "index_ms": 776, + "cited": [ + "pkg/cmd/root/root.go:63", + "pkg/cmd/skills/install/install.go:232", + "pkg/cmd/skills/install/install_test.go:2134", + "pkg/cmd/skills/preview/preview_test.go:112", + "pkg/cmd/skills/preview/preview_test.go:1157", + "pkg/cmd/skills/preview/preview_test.go:24", + "pkg/cmd/skills/preview/preview_test.go:408", + "pkg/cmd/skills/preview/preview_test.go:419", + "pkg/cmd/skills/preview/preview_test.go:484", + "pkg/cmd/skills/preview/preview_test.go:696", + "pkg/cmd/skills/preview/preview_test.go:871", + "pkg/cmd/skills/preview/preview_test.go:948", + "pkg/cmd/skills/skills.go:16" + ], + "cited_on_complete": [ + "pkg/cmd/root/root.go:63", + "pkg/cmd/skills/install/install.go:232", + "pkg/cmd/skills/install/install_test.go:2134", + "pkg/cmd/skills/preview/preview_test.go:1157", + "pkg/cmd/skills/preview/preview_test.go:24", + "pkg/cmd/skills/preview/preview_test.go:408", + "pkg/cmd/skills/preview/preview_test.go:419", + "pkg/cmd/skills/preview/preview_test.go:484", + "pkg/cmd/skills/preview/preview_test.go:696", + "pkg/cmd/skills/preview/preview_test.go:871", + "pkg/cmd/skills/preview/preview_test.go:948", + "pkg/cmd/skills/skills.go:16" + ] + }, + { + "task": "v-a44721d2", + "commit": "a44721d233be9a2f6f0b5ee5c4f71274acb8d296", + "omission_file": "internal/prompter/echo_linux_test.go", + "omission_symbol": null, + "omission_file_reachable": false, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": null, + "findings": 0, + "false_alarms": 0, + "changed_symbols": 0, + "verify_tokens": 17, + "verify_latency_ms": 0, + "index_ms": 768, + "cited": [], + "cited_on_complete": [] + } +] \ No newline at end of file diff --git a/benchmarks/results/verify-gh-cli/verify-summary.json b/benchmarks/results/verify-gh-cli/verify-summary.json new file mode 100644 index 0000000..5332877 --- /dev/null +++ b/benchmarks/results/verify-gh-cli/verify-summary.json @@ -0,0 +1,36 @@ +{ + "tasks": 20, + "omission_recall": 0.4, + "omission_recall_ci": [ + 0.21880396, + 0.6134221 + ], + "omission_recall_attributable": 0.15, + "omission_recall_attributable_ci": [ + 0.05236779, + 0.36042333 + ], + "reachable_omissions": 19, + "omission_recall_reachable": 0.42105263, + "omission_recall_reachable_ci": [ + 0.2314162, + 0.63724446 + ], + "symbol_scorable": 16, + "omission_recall_symbol": 0.3125, + "omission_recall_symbol_ci": [ + 0.14164433, + 0.55596066 + ], + "false_alarm_rate": 4.45, + "commits_with_a_false_alarm": 15, + "false_alarm_share_ci": [ + 0.53129494, + 0.88813996 + ], + "median_findings_per_diff": 2, + "median_verify_tokens": 48, + "median_verify_latency_ms": 0, + "median_index_ms": 809, + "diffs_resolving_to_nothing": 3 +} \ No newline at end of file diff --git a/benchmarks/results/verify-gh-cli/verify-tasks.json b/benchmarks/results/verify-gh-cli/verify-tasks.json new file mode 100644 index 0000000..c36e535 --- /dev/null +++ b/benchmarks/results/verify-gh-cli/verify-tasks.json @@ -0,0 +1,2881 @@ +{ + "repository": "/private/tmp/claude-501/-Users-lambiengcode--treehouse-reify-2e416f-2-reify/a9d50d0b-cce2-4027-a129-e64262e800a7/scratchpad/gh-cli", + "head": "5d3c4817f1619213951dbf15031bad04acb88392", + "generated_from_commits": 399, + "rejected": [ + [ + "92ae1de0355368b3d6d1c361362dc524fe98498b", + "no file changed by exactly one hunk" + ], + [ + "5e6aa5ab275ad25a8999871f7f86aeda6349099b", + "no file changed by exactly one hunk" + ], + [ + "c2ad3b0eb7ead66eec3a8a61239e10a765332ec7", + "no file changed by exactly one hunk" + ], + [ + "b130a9be5b0f0db2e41ddd82044ef6c256c93625", + "no file changed by exactly one hunk" + ], + [ + "7b681a4e8b67d203ccdaab5ff54570bdf6ca3669", + "fewer than two indexable files" + ], + [ + "954ffc37e5931a9621868a50f05c8fb17e288f19", + "no file changed by exactly one hunk" + ], + [ + "cce391b663aa488b8b4f681616b52dc8f2e3d531", + "fewer than two indexable files" + ], + [ + "70bb306bd25eb407f90eabefd98824aed62cf519", + "no file changed by exactly one hunk" + ], + [ + "01bcd474447b5034da20686c838ae4b0cf2b23f5", + "no file changed by exactly one hunk" + ], + [ + "d63ab8ddd8696194f3fd49a726eb6d60dd3cdfa5", + "no file changed by exactly one hunk" + ], + [ + "5d77247a5a77b4ee1e9516f83da010dd9a0d08c3", + "no file changed by exactly one hunk" + ], + [ + "9f2da1186132c0ba891767939c81fb6e785b67dc", + "no file changed by exactly one hunk" + ], + [ + "da68cb8f6f597cfc3838cf40f89ecc01f4e53233", + "fewer than two indexable files" + ], + [ + "797effe0295914e03f4b40cb873276da1d3b7e93", + "fewer than two indexable files" + ], + [ + "7bd67a840456436a1edc0f80dc85687f29b277af", + "fewer than two indexable files" + ], + [ + "57008e797097e9b275ba7d1849ac750e762743e1", + "fewer than two indexable files" + ], + [ + "8b73951ca75c6495a776e36da9fb48cc93000866", + "fewer than two indexable files" + ], + [ + "d3a153872b9774009d620d0f3e12f993a21765eb", + "fewer than two indexable files" + ], + [ + "51b765381870dbf62390e0f7a297cc57ea771db7", + "fewer than two indexable files" + ], + [ + "97d1cbd9fc5499a2f804d991b353821e17b19eb7", + "no file changed by exactly one hunk" + ], + [ + "00fc8c923ab3f321a412e1c89e425c107122f0bb", + "no file changed by exactly one hunk" + ], + [ + "d9eb0627dceeb49b2943fa992414eb185787d02e", + "no file changed by exactly one hunk" + ], + [ + "601dd346b00b357a0541239fb80b34c3795e7c33", + "no file changed by exactly one hunk" + ] + ], + "tasks": [ + { + "id": "v-e4efbc42", + "commit": "e4efbc42ccfb1f50c2b97e7b864eb5fc5bcc97f0", + "parent": "046c222048e780a707bacea18fc6dc0d0865c7a5", + "date": "2026-08-21", + "prompt": "Narrow Copilot newline fix scope", + "complete": { + "files": [ + { + "path": "pkg/cmd/copilot/copilot.go", + "created": false, + "hunks": [ + { + "old_start": 156, + "old_len": 7, + "changed_lines": [ + 159 + ] + } + ] + }, + { + "path": "pkg/cmd/copilot/copilot_test.go", + "created": false, + "hunks": [ + { + "old_start": 620, + "old_len": 14, + "changed_lines": [ + 623, + 624, + 625, + 626, + 627, + 628, + 629, + 630 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "pkg/cmd/copilot/copilot.go", + "created": false, + "hunks": [ + { + "old_start": 156, + "old_len": 7, + "changed_lines": [ + 159 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/copilot/copilot_test.go", + "omission_line": 623 + }, + { + "id": "v-1e04dab8", + "commit": "1e04dab89cf10a2eab5b238d27fb1d7cb94b8af4", + "parent": "a255baf71d13fe5947a4eb7ad521ffd412d64cee", + "date": "2026-08-20", + "prompt": "Fix Copilot install warning newlines", + "complete": { + "files": [ + { + "path": "pkg/cmd/copilot/copilot.go", + "created": false, + "hunks": [ + { + "old_start": 152, + "old_len": 11, + "changed_lines": [ + 155, + 159 + ] + } + ] + }, + { + "path": "pkg/cmd/copilot/copilot_test.go", + "created": false, + "hunks": [ + { + "old_start": 16, + "old_len": 6, + "changed_lines": [ + 19 + ] + }, + { + "old_start": 591, + "old_len": 30, + "changed_lines": [ + 594, + 595, + 596, + 597, + 598, + 601, + 602, + 603, + 604, + 605, + 607, + 608, + 609, + 610, + 611, + 612, + 614, + 615, + 616, + 617 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "pkg/cmd/copilot/copilot_test.go", + "created": false, + "hunks": [ + { + "old_start": 16, + "old_len": 6, + "changed_lines": [ + 19 + ] + }, + { + "old_start": 591, + "old_len": 30, + "changed_lines": [ + 594, + 595, + 596, + 597, + 598, + 601, + 602, + 603, + 604, + 605, + 607, + 608, + 609, + 610, + 611, + 612, + 614, + 615, + 616, + 617 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/copilot/copilot.go", + "omission_line": 155 + }, + { + "id": "v-2e9fedd3", + "commit": "2e9fedd3aaeb9bc5f044f2d825f565d98083fc56", + "parent": "a526307b621c90dca18734bfedaa5533318edd1a", + "date": "2026-08-12", + "prompt": "Add worktree checkout to issue develop", + "complete": { + "files": [ + { + "path": "git/client.go", + "created": false, + "hunks": [ + { + "old_start": 642, + "old_len": 6, + "changed_lines": [ + 645 + ] + } + ] + }, + { + "path": "git/client_test.go", + "created": false, + "hunks": [ + { + "old_start": 1426, + "old_len": 6, + "changed_lines": [ + 1429 + ] + } + ] + }, + { + "path": "pkg/cmd/issue/develop/develop.go", + "created": false, + "hunks": [ + { + "old_start": 5, + "old_len": 6, + "changed_lines": [ + 8 + ] + }, + { + "old_start": 32, + "old_len": 6, + "changed_lines": [ + 35 + ] + }, + { + "old_start": 66, + "old_len": 6, + "changed_lines": [ + 69 + ] + }, + { + "old_start": 91, + "old_len": 6, + "changed_lines": [ + 94 + ] + }, + { + "old_start": 120, + "old_len": 6, + "changed_lines": [ + 123 + ] + }, + { + "old_start": 133, + "old_len": 6, + "changed_lines": [ + 136 + ] + }, + { + "old_start": 353, + "old_len": 16, + "changed_lines": [ + 356, + 357, + 360, + 364, + 365 + ] + } + ] + }, + { + "path": "pkg/cmd/issue/develop/develop_test.go", + "created": false, + "hunks": [ + { + "old_start": 4, + "old_len": 6, + "changed_lines": [ + 7 + ] + }, + { + "old_start": 24, + "old_len": 6, + "changed_lines": [ + 27 + ] + }, + { + "old_start": 58, + "old_len": 6, + "changed_lines": [ + 61 + ] + }, + { + "old_start": 106, + "old_len": 6, + "changed_lines": [ + 109 + ] + }, + { + "old_start": 138, + "old_len": 6, + "changed_lines": [ + 141 + ] + }, + { + "old_start": 767, + "old_len": 3, + "changed_lines": [ + 770 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "git/client.go", + "created": false, + "hunks": [ + { + "old_start": 642, + "old_len": 6, + "changed_lines": [ + 645 + ] + } + ] + }, + { + "path": "pkg/cmd/issue/develop/develop.go", + "created": false, + "hunks": [ + { + "old_start": 5, + "old_len": 6, + "changed_lines": [ + 8 + ] + }, + { + "old_start": 32, + "old_len": 6, + "changed_lines": [ + 35 + ] + }, + { + "old_start": 66, + "old_len": 6, + "changed_lines": [ + 69 + ] + }, + { + "old_start": 91, + "old_len": 6, + "changed_lines": [ + 94 + ] + }, + { + "old_start": 120, + "old_len": 6, + "changed_lines": [ + 123 + ] + }, + { + "old_start": 133, + "old_len": 6, + "changed_lines": [ + 136 + ] + }, + { + "old_start": 353, + "old_len": 16, + "changed_lines": [ + 356, + 357, + 360, + 364, + 365 + ] + } + ] + }, + { + "path": "pkg/cmd/issue/develop/develop_test.go", + "created": false, + "hunks": [ + { + "old_start": 4, + "old_len": 6, + "changed_lines": [ + 7 + ] + }, + { + "old_start": 24, + "old_len": 6, + "changed_lines": [ + 27 + ] + }, + { + "old_start": 58, + "old_len": 6, + "changed_lines": [ + 61 + ] + }, + { + "old_start": 106, + "old_len": 6, + "changed_lines": [ + 109 + ] + }, + { + "old_start": 138, + "old_len": 6, + "changed_lines": [ + 141 + ] + }, + { + "old_start": 767, + "old_len": 3, + "changed_lines": [ + 770 + ] + } + ] + } + ] + }, + "omission_file": "git/client_test.go", + "omission_line": 1429 + }, + { + "id": "v-a6bcd08d", + "commit": "a6bcd08d07d1cbcb17cfce497c0cf261a966f703", + "parent": "e83adbc0642994fae7c39a9a012eb34b8c81f4f1", + "date": "2026-08-03", + "prompt": "Fix item-add output for non-TTY", + "complete": { + "files": [ + { + "path": "pkg/cmd/project/item-add/item_add.go", + "created": false, + "hunks": [ + { + "old_start": 124, + "old_len": 10, + "changed_lines": [ + 127, + 128, + 131 + ] + } + ] + }, + { + "path": "pkg/cmd/project/item-add/item_add_test.go", + "created": false, + "hunks": [ + { + "old_start": 8, + "old_len": 6, + "changed_lines": [ + 11 + ] + }, + { + "old_start": 539, + "old_len": 3, + "changed_lines": [ + 542 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "pkg/cmd/project/item-add/item_add_test.go", + "created": false, + "hunks": [ + { + "old_start": 8, + "old_len": 6, + "changed_lines": [ + 11 + ] + }, + { + "old_start": 539, + "old_len": 3, + "changed_lines": [ + 542 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/project/item-add/item_add.go", + "omission_line": 127 + }, + { + "id": "v-efe3f165", + "commit": "efe3f165dd297c85fff11473dbf586f2d39fbf86", + "parent": "ae66a1c02e08366858f3070664f493afbe0cdf18", + "date": "2026-07-22", + "prompt": "Add named field columns to gh project item-list", + "complete": { + "files": [ + { + "path": "pkg/cmd/project/item-list/item_list.go", + "created": false, + "hunks": [ + { + "old_start": 21, + "old_len": 6, + "changed_lines": [ + 24 + ] + }, + { + "old_start": 55, + "old_len": 6, + "changed_lines": [ + 58 + ] + }, + { + "old_start": 71, + "old_len": 6, + "changed_lines": [ + 74 + ] + }, + { + "old_start": 101, + "old_len": 6, + "changed_lines": [ + 104 + ] + }, + { + "old_start": 142, + "old_len": 15, + "changed_lines": [ + 145, + 148, + 153 + ] + }, + { + "old_start": 162, + "old_len": 8, + "changed_lines": [ + 165, + 170 + ] + } + ] + }, + { + "path": "pkg/cmd/project/item-list/item_list_test.go", + "created": false, + "hunks": [ + { + "old_start": 61, + "old_len": 6, + "changed_lines": [ + 64 + ] + }, + { + "old_start": 95, + "old_len": 6, + "changed_lines": [ + 98 + ] + }, + { + "old_start": 734, + "old_len": 3, + "changed_lines": [ + 737 + ] + } + ] + }, + { + "path": "pkg/cmd/project/shared/queries/queries.go", + "created": false, + "hunks": [ + { + "old_start": 6, + "old_len": 6, + "changed_lines": [ + 9 + ] + }, + { + "old_start": 472, + "old_len": 6, + "changed_lines": [ + 475 + ] + } + ] + }, + { + "path": "pkg/cmd/project/shared/queries/queries_test.go", + "created": false, + "hunks": [ + { + "old_start": 680, + "old_len": 3, + "changed_lines": [ + 683 + ] + } + ] + }, + { + "path": "pkg/cmd/project/shared/queries/resolve_fields.go", + "created": false, + "hunks": [ + { + "old_start": 85, + "old_len": 6, + "changed_lines": [ + 88 + ] + } + ] + }, + { + "path": "pkg/cmd/project/shared/queries/resolve_fields_test.go", + "created": false, + "hunks": [ + { + "old_start": 60, + "old_len": 14, + "changed_lines": [ + 63, + 70 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "pkg/cmd/project/item-list/item_list.go", + "created": false, + "hunks": [ + { + "old_start": 21, + "old_len": 6, + "changed_lines": [ + 24 + ] + }, + { + "old_start": 55, + "old_len": 6, + "changed_lines": [ + 58 + ] + }, + { + "old_start": 71, + "old_len": 6, + "changed_lines": [ + 74 + ] + }, + { + "old_start": 101, + "old_len": 6, + "changed_lines": [ + 104 + ] + }, + { + "old_start": 142, + "old_len": 15, + "changed_lines": [ + 145, + 148, + 153 + ] + }, + { + "old_start": 162, + "old_len": 8, + "changed_lines": [ + 165, + 170 + ] + } + ] + }, + { + "path": "pkg/cmd/project/item-list/item_list_test.go", + "created": false, + "hunks": [ + { + "old_start": 61, + "old_len": 6, + "changed_lines": [ + 64 + ] + }, + { + "old_start": 95, + "old_len": 6, + "changed_lines": [ + 98 + ] + }, + { + "old_start": 734, + "old_len": 3, + "changed_lines": [ + 737 + ] + } + ] + }, + { + "path": "pkg/cmd/project/shared/queries/queries.go", + "created": false, + "hunks": [ + { + "old_start": 6, + "old_len": 6, + "changed_lines": [ + 9 + ] + }, + { + "old_start": 472, + "old_len": 6, + "changed_lines": [ + 475 + ] + } + ] + }, + { + "path": "pkg/cmd/project/shared/queries/queries_test.go", + "created": false, + "hunks": [ + { + "old_start": 680, + "old_len": 3, + "changed_lines": [ + 683 + ] + } + ] + }, + { + "path": "pkg/cmd/project/shared/queries/resolve_fields.go", + "created": false, + "hunks": [ + { + "old_start": 85, + "old_len": 6, + "changed_lines": [ + 88 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/project/shared/queries/resolve_fields_test.go", + "omission_line": 63 + }, + { + "id": "v-688751de", + "commit": "688751de2ce8d610ce76cd0608930e7912509ed3", + "parent": "a5eea131501c535d0527eb61ade5969bd85d0ff3", + "date": "2026-07-22", + "prompt": "Harden worktree submodule prefixing and cover cmd.Dir stripping", + "complete": { + "files": [ + { + "path": "pkg/cmd/pr/checkout/checkout.go", + "created": false, + "hunks": [ + { + "old_start": 5, + "old_len": 6, + "changed_lines": [ + 8 + ] + }, + { + "old_start": 166, + "old_len": 8, + "changed_lines": [ + 169, + 170 + ] + } + ] + }, + { + "path": "pkg/cmd/pr/checkout/checkout_test.go", + "created": false, + "hunks": [ + { + "old_start": 1170, + "old_len": 3, + "changed_lines": [ + 1173 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "pkg/cmd/pr/checkout/checkout.go", + "created": false, + "hunks": [ + { + "old_start": 5, + "old_len": 6, + "changed_lines": [ + 8 + ] + }, + { + "old_start": 166, + "old_len": 8, + "changed_lines": [ + 169, + 170 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/pr/checkout/checkout_test.go", + "omission_line": 1173 + }, + { + "id": "v-9f14d1ac", + "commit": "9f14d1ac675f25a75d4b940dc88e733f06398e76", + "parent": "9fc654ee0985d08c2d9076785c6993da885435a4", + "date": "2026-07-22", + "prompt": "Simplify submodule worktree prefix to inline conditional", + "complete": { + "files": [ + { + "path": "pkg/cmd/pr/checkout/checkout.go", + "created": false, + "hunks": [ + { + "old_start": 160, + "old_len": 7, + "changed_lines": [ + 163 + ] + }, + { + "old_start": 359, + "old_len": 23, + "changed_lines": [ + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378 + ] + } + ] + }, + { + "path": "pkg/cmd/pr/checkout/checkout_test.go", + "created": false, + "hunks": [ + { + "old_start": 1071, + "old_len": 19, + "changed_lines": [ + 1074, + 1075, + 1076, + 1077, + 1078, + 1079, + 1080, + 1081, + 1082, + 1083, + 1084, + 1085, + 1086, + 1087, + 1088, + 1089 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "pkg/cmd/pr/checkout/checkout.go", + "created": false, + "hunks": [ + { + "old_start": 160, + "old_len": 7, + "changed_lines": [ + 163 + ] + }, + { + "old_start": 359, + "old_len": 23, + "changed_lines": [ + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/pr/checkout/checkout_test.go", + "omission_line": 1074 + }, + { + "id": "v-d5f4bed3", + "commit": "d5f4bed3f49dbbc5d5a2e7bb76ba9f25ba7ba574", + "parent": "2b970995a3c63b7a7600f242d55d15185a82ed8b", + "date": "2026-07-13", + "prompt": "Add Grok skill host support", + "complete": { + "files": [ + { + "path": "internal/skills/registry/registry.go", + "created": false, + "hunks": [ + { + "old_start": 188, + "old_len": 6, + "changed_lines": [ + 191 + ] + } + ] + }, + { + "path": "internal/skills/registry/registry_test.go", + "created": false, + "hunks": [ + { + "old_start": 23, + "old_len": 6, + "changed_lines": [ + 26 + ] + }, + { + "old_start": 159, + "old_len": 6, + "changed_lines": [ + 162 + ] + } + ] + }, + { + "path": "pkg/cmd/skills/install/install.go", + "created": false, + "hunks": [ + { + "old_start": 89, + "old_len": 7, + "changed_lines": [ + 92 + ] + } + ] + }, + { + "path": "pkg/cmd/skills/update/update.go", + "created": false, + "hunks": [ + { + "old_start": 80, + "old_len": 7, + "changed_lines": [ + 83 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "internal/skills/registry/registry.go", + "created": false, + "hunks": [ + { + "old_start": 188, + "old_len": 6, + "changed_lines": [ + 191 + ] + } + ] + }, + { + "path": "internal/skills/registry/registry_test.go", + "created": false, + "hunks": [ + { + "old_start": 23, + "old_len": 6, + "changed_lines": [ + 26 + ] + }, + { + "old_start": 159, + "old_len": 6, + "changed_lines": [ + 162 + ] + } + ] + }, + { + "path": "pkg/cmd/skills/install/install.go", + "created": false, + "hunks": [ + { + "old_start": 89, + "old_len": 7, + "changed_lines": [ + 92 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/skills/update/update.go", + "omission_line": 83 + }, + { + "id": "v-74e77914", + "commit": "74e779140c472bc380ce57978853480805dc16b7", + "parent": "6dae3077b89c9858c5778c0b37a116b1091f8782", + "date": "2026-07-02", + "prompt": "Fix concurrent map writes in codespace port forwarding", + "complete": { + "files": [ + { + "path": "internal/codespaces/connection/connection.go", + "created": false, + "hunks": [ + { + "old_start": 30, + "old_len": 6, + "changed_lines": [ + 33 + ] + } + ] + }, + { + "path": "internal/codespaces/portforwarder/port_forwarder.go", + "created": false, + "hunks": [ + { + "old_start": 36, + "old_len": 7, + "changed_lines": [ + 39 + ] + }, + { + "old_start": 54, + "old_len": 7, + "changed_lines": [ + 57 + ] + }, + { + "old_start": 108, + "old_len": 6, + "changed_lines": [ + 111 + ] + }, + { + "old_start": 166, + "old_len": 18, + "changed_lines": [ + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180 + ] + }, + { + "old_start": 243, + "old_len": 6, + "changed_lines": [ + 246 + ] + }, + { + "old_start": 253, + "old_len": 22, + "changed_lines": [ + 256, + 258, + 263, + 269, + 272 + ] + } + ] + }, + { + "path": "internal/codespaces/portforwarder/port_forwarder_test.go", + "created": false, + "hunks": [ + { + "old_start": 7, + "old_len": 6, + "changed_lines": [ + 10 + ] + }, + { + "old_start": 31, + "old_len": 26, + "changed_lines": [ + 34, + 35, + 36, + 40, + 41, + 42, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53 + ] + }, + { + "old_start": 96, + "old_len": 9, + "changed_lines": [ + 99, + 100, + 101 + ] + }, + { + "old_start": 131, + "old_len": 9, + "changed_lines": [ + 134, + 135, + 136 + ] + }, + { + "old_start": 163, + "old_len": 39, + "changed_lines": [ + 166, + 167, + 168, + 171, + 172, + 173, + 176, + 177, + 178, + 181, + 183, + 184, + 185, + 188, + 189, + 190, + 192, + 193, + 196, + 197, + 199 + ] + }, + { + "old_start": 230, + "old_len": 38, + "changed_lines": [ + 233, + 234, + 235, + 238, + 239, + 240, + 243, + 244, + 245, + 249, + 251, + 252, + 253, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "internal/codespaces/portforwarder/port_forwarder.go", + "created": false, + "hunks": [ + { + "old_start": 36, + "old_len": 7, + "changed_lines": [ + 39 + ] + }, + { + "old_start": 54, + "old_len": 7, + "changed_lines": [ + 57 + ] + }, + { + "old_start": 108, + "old_len": 6, + "changed_lines": [ + 111 + ] + }, + { + "old_start": 166, + "old_len": 18, + "changed_lines": [ + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180 + ] + }, + { + "old_start": 243, + "old_len": 6, + "changed_lines": [ + 246 + ] + }, + { + "old_start": 253, + "old_len": 22, + "changed_lines": [ + 256, + 258, + 263, + 269, + 272 + ] + } + ] + }, + { + "path": "internal/codespaces/portforwarder/port_forwarder_test.go", + "created": false, + "hunks": [ + { + "old_start": 7, + "old_len": 6, + "changed_lines": [ + 10 + ] + }, + { + "old_start": 31, + "old_len": 26, + "changed_lines": [ + 34, + 35, + 36, + 40, + 41, + 42, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53 + ] + }, + { + "old_start": 96, + "old_len": 9, + "changed_lines": [ + 99, + 100, + 101 + ] + }, + { + "old_start": 131, + "old_len": 9, + "changed_lines": [ + 134, + 135, + 136 + ] + }, + { + "old_start": 163, + "old_len": 39, + "changed_lines": [ + 166, + 167, + 168, + 171, + 172, + 173, + 176, + 177, + 178, + 181, + 183, + 184, + 185, + 188, + 189, + 190, + 192, + 193, + 196, + 197, + 199 + ] + }, + { + "old_start": 230, + "old_len": 38, + "changed_lines": [ + 233, + 234, + 235, + 238, + 239, + 240, + 243, + 244, + 245, + 249, + 251, + 252, + 253, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266 + ] + } + ] + } + ] + }, + "omission_file": "internal/codespaces/connection/connection.go", + "omission_line": 33 + }, + { + "id": "v-f1d11210", + "commit": "f1d112104821b055bb0c5656f2989f9213db71f6", + "parent": "326faaac8b5c3b736ddc6bfa573cc97dc6452a24", + "date": "2026-07-02", + "prompt": "honor --dir without agent prompt", + "complete": { + "files": [ + { + "path": "pkg/cmd/skills/install/install.go", + "created": false, + "hunks": [ + { + "old_start": 915, + "old_len": 6, + "changed_lines": [ + 918 + ] + } + ] + }, + { + "path": "pkg/cmd/skills/install/install_test.go", + "created": false, + "hunks": [ + { + "old_start": 473, + "old_len": 6, + "changed_lines": [ + 476 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "pkg/cmd/skills/install/install.go", + "created": false, + "hunks": [ + { + "old_start": 915, + "old_len": 6, + "changed_lines": [ + 918 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/skills/install/install_test.go", + "omission_line": 476 + }, + { + "id": "v-751dc5e0", + "commit": "751dc5e0383f08d1d6a211c97d0479aedb39726b", + "parent": "dd26eb39b04db5bd282509787d2b8fbff11ba694", + "date": "2026-06-24", + "prompt": "don't let a failed draft lookup mask a found release", + "complete": { + "files": [ + { + "path": "pkg/cmd/release/download/download_test.go", + "created": false, + "hunks": [ + { + "old_start": 218, + "old_len": 6, + "changed_lines": [ + 221 + ] + } + ] + }, + { + "path": "pkg/cmd/release/shared/fetch.go", + "created": false, + "hunks": [ + { + "old_start": 201, + "old_len": 15, + "changed_lines": [ + 204, + 205, + 206, + 207, + 208, + 211, + 212 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "pkg/cmd/release/download/download_test.go", + "created": false, + "hunks": [ + { + "old_start": 218, + "old_len": 6, + "changed_lines": [ + 221 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/release/shared/fetch.go", + "omission_line": 204 + }, + { + "id": "v-517dae6a", + "commit": "517dae6a938d4efe73f1219873ebcc74cf4febe1", + "parent": "70bb306bd25eb407f90eabefd98824aed62cf519", + "date": "2026-06-18", + "prompt": "install universal agent to ~/.agents/skills", + "complete": { + "files": [ + { + "path": "internal/skills/registry/registry.go", + "created": false, + "hunks": [ + { + "old_start": 299, + "old_len": 7, + "changed_lines": [ + 302 + ] + } + ] + }, + { + "path": "internal/skills/registry/registry_test.go", + "created": false, + "hunks": [ + { + "old_start": 125, + "old_len": 6, + "changed_lines": [ + 128 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "internal/skills/registry/registry.go", + "created": false, + "hunks": [ + { + "old_start": 299, + "old_len": 7, + "changed_lines": [ + 302 + ] + } + ] + } + ] + }, + "omission_file": "internal/skills/registry/registry_test.go", + "omission_line": 128 + }, + { + "id": "v-8d2b059e", + "commit": "8d2b059e07f71c17068f2286617f23286e05e0c0", + "parent": "869c044391ba73f1adca31f52d4b025430006242", + "date": "2026-06-10", + "prompt": "fix: error when --comments is used with a comment argument", + "complete": { + "files": [ + { + "path": "pkg/cmd/discussion/view/view.go", + "created": false, + "hunks": [ + { + "old_start": 174, + "old_len": 6, + "changed_lines": [ + 177 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/view/view_test.go", + "created": false, + "hunks": [ + { + "old_start": 190, + "old_len": 14, + "changed_lines": [ + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200 + ] + }, + { + "old_start": 881, + "old_len": 7, + "changed_lines": [ + 884 + ] + }, + { + "old_start": 910, + "old_len": 7, + "changed_lines": [ + 913 + ] + }, + { + "old_start": 940, + "old_len": 7, + "changed_lines": [ + 943 + ] + }, + { + "old_start": 1003, + "old_len": 7, + "changed_lines": [ + 1006 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "pkg/cmd/discussion/view/view_test.go", + "created": false, + "hunks": [ + { + "old_start": 190, + "old_len": 14, + "changed_lines": [ + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200 + ] + }, + { + "old_start": 881, + "old_len": 7, + "changed_lines": [ + 884 + ] + }, + { + "old_start": 910, + "old_len": 7, + "changed_lines": [ + 913 + ] + }, + { + "old_start": 940, + "old_len": 7, + "changed_lines": [ + 943 + ] + }, + { + "old_start": 1003, + "old_len": 7, + "changed_lines": [ + 1006 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/discussion/view/view.go", + "omission_line": 177 + }, + { + "id": "v-2618999b", + "commit": "2618999bcb6c85d4554638937dc80c322a76d593", + "parent": "4166ecf2cada209714af79387a5243b334af764b", + "date": "2026-06-06", + "prompt": "feat: add comment manipulation methods", + "complete": { + "files": [ + { + "path": "pkg/cmd/discussion/client/client.go", + "created": false, + "hunks": [ + { + "old_start": 22, + "old_len": 4, + "changed_lines": [ + 25 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/client/client_impl.go", + "created": false, + "hunks": [ + { + "old_start": 1075, + "old_len": 3, + "changed_lines": [ + 1078 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/client/client_impl_test.go", + "created": false, + "hunks": [ + { + "old_start": 3599, + "old_len": 3, + "changed_lines": [ + 3602 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/client/client_mock.go", + "created": false, + "hunks": [ + { + "old_start": 18, + "old_len": 12, + "changed_lines": [ + 21, + 24, + 27 + ] + }, + { + "old_start": 45, + "old_len": 6, + "changed_lines": [ + 48 + ] + }, + { + "old_start": 52, + "old_len": 12, + "changed_lines": [ + 55, + 58, + 61 + ] + }, + { + "old_start": 79, + "old_len": 8, + "changed_lines": [ + 82, + 84 + ] + }, + { + "old_start": 88, + "old_len": 6, + "changed_lines": [ + 91 + ] + }, + { + "old_start": 95, + "old_len": 6, + "changed_lines": [ + 98 + ] + }, + { + "old_start": 162, + "old_len": 9, + "changed_lines": [ + 165, + 167 + ] + }, + { + "old_start": 172, + "old_len": 6, + "changed_lines": [ + 175 + ] + }, + { + "old_start": 210, + "old_len": 6, + "changed_lines": [ + 213 + ] + }, + { + "old_start": 246, + "old_len": 6, + "changed_lines": [ + 249 + ] + }, + { + "old_start": 533, + "old_len": 3, + "changed_lines": [ + 536 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "pkg/cmd/discussion/client/client.go", + "created": false, + "hunks": [ + { + "old_start": 22, + "old_len": 4, + "changed_lines": [ + 25 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/client/client_impl.go", + "created": false, + "hunks": [ + { + "old_start": 1075, + "old_len": 3, + "changed_lines": [ + 1078 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/client/client_mock.go", + "created": false, + "hunks": [ + { + "old_start": 18, + "old_len": 12, + "changed_lines": [ + 21, + 24, + 27 + ] + }, + { + "old_start": 45, + "old_len": 6, + "changed_lines": [ + 48 + ] + }, + { + "old_start": 52, + "old_len": 12, + "changed_lines": [ + 55, + 58, + 61 + ] + }, + { + "old_start": 79, + "old_len": 8, + "changed_lines": [ + 82, + 84 + ] + }, + { + "old_start": 88, + "old_len": 6, + "changed_lines": [ + 91 + ] + }, + { + "old_start": 95, + "old_len": 6, + "changed_lines": [ + 98 + ] + }, + { + "old_start": 162, + "old_len": 9, + "changed_lines": [ + 165, + 167 + ] + }, + { + "old_start": 172, + "old_len": 6, + "changed_lines": [ + 175 + ] + }, + { + "old_start": 210, + "old_len": 6, + "changed_lines": [ + 213 + ] + }, + { + "old_start": 246, + "old_len": 6, + "changed_lines": [ + 249 + ] + }, + { + "old_start": 533, + "old_len": 3, + "changed_lines": [ + 536 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/discussion/client/client_impl_test.go", + "omission_line": 3602 + }, + { + "id": "v-e2d150da", + "commit": "e2d150da420b8bf84f3097f6fad3bbb715ea1cb4", + "parent": "9d413e769a3604194364421db34bcc0128696d09", + "date": "2026-06-09", + "prompt": "remove redundant error wrapping on ListCategories", + "complete": { + "files": [ + { + "path": "pkg/cmd/discussion/create/create.go", + "created": false, + "hunks": [ + { + "old_start": 113, + "old_len": 7, + "changed_lines": [ + 116 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/create/create_test.go", + "created": false, + "hunks": [ + { + "old_start": 242, + "old_len": 7, + "changed_lines": [ + 245 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/edit/edit.go", + "created": false, + "hunks": [ + { + "old_start": 175, + "old_len": 7, + "changed_lines": [ + 178 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "pkg/cmd/discussion/create/create.go", + "created": false, + "hunks": [ + { + "old_start": 113, + "old_len": 7, + "changed_lines": [ + 116 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/create/create_test.go", + "created": false, + "hunks": [ + { + "old_start": 242, + "old_len": 7, + "changed_lines": [ + 245 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/discussion/edit/edit.go", + "omission_line": 178 + }, + { + "id": "v-c1f3c1a1", + "commit": "c1f3c1a164ab67436d525769adbce0fd67dd5e70", + "parent": "e61df0721a01641e15c463a8cbb829fa40242eec", + "date": "2026-06-08", + "prompt": "add missing repo flag override", + "complete": { + "files": [ + { + "path": "pkg/cmd/discussion/list/list.go", + "created": false, + "hunks": [ + { + "old_start": 129, + "old_len": 6, + "changed_lines": [ + 132 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/view/view.go", + "created": false, + "hunks": [ + { + "old_start": 189, + "old_len": 6, + "changed_lines": [ + 192 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "pkg/cmd/discussion/list/list.go", + "created": false, + "hunks": [ + { + "old_start": 129, + "old_len": 6, + "changed_lines": [ + 132 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/discussion/view/view.go", + "omission_line": 192 + }, + { + "id": "v-b1029009", + "commit": "b1029009dbfbcf4240472097e12614dbc3cdcd19", + "parent": "d6a089d5ce30e5b09bac22338f7949ef7ebbd36f", + "date": "2026-06-05", + "prompt": "handle partial failure on create/update label mutations", + "complete": { + "files": [ + { + "path": "pkg/cmd/discussion/client/client.go", + "created": false, + "hunks": [ + { + "old_start": 16, + "old_len": 6, + "changed_lines": [ + 19 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/client/client_impl.go", + "created": false, + "hunks": [ + { + "old_start": 1, + "old_len": 6, + "changed_lines": [ + 4 + ] + }, + { + "old_start": 925, + "old_len": 6, + "changed_lines": [ + 928 + ] + }, + { + "old_start": 957, + "old_len": 12, + "changed_lines": [ + 960, + 963, + 965 + ] + }, + { + "old_start": 974, + "old_len": 9, + "changed_lines": [ + 977, + 980 + ] + }, + { + "old_start": 1021, + "old_len": 12, + "changed_lines": [ + 1024, + 1027, + 1029 + ] + }, + { + "old_start": 1038, + "old_len": 5, + "changed_lines": [ + 1041 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/client/client_impl_test.go", + "created": false, + "hunks": [ + { + "old_start": 2813, + "old_len": 7, + "changed_lines": [ + 2816 + ] + }, + { + "old_start": 2866, + "old_len": 7, + "changed_lines": [ + 2869 + ] + }, + { + "old_start": 2885, + "old_len": 6, + "changed_lines": [ + 2888 + ] + }, + { + "old_start": 3509, + "old_len": 6, + "changed_lines": [ + 3512 + ] + }, + { + "old_start": 3526, + "old_len": 6, + "changed_lines": [ + 3529 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/create/create.go", + "created": false, + "hunks": [ + { + "old_start": 188, + "old_len": 6, + "changed_lines": [ + 191 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/edit/edit.go", + "created": false, + "hunks": [ + { + "old_start": 210, + "old_len": 6, + "changed_lines": [ + 213 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "pkg/cmd/discussion/client/client.go", + "created": false, + "hunks": [ + { + "old_start": 16, + "old_len": 6, + "changed_lines": [ + 19 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/client/client_impl.go", + "created": false, + "hunks": [ + { + "old_start": 1, + "old_len": 6, + "changed_lines": [ + 4 + ] + }, + { + "old_start": 925, + "old_len": 6, + "changed_lines": [ + 928 + ] + }, + { + "old_start": 957, + "old_len": 12, + "changed_lines": [ + 960, + 963, + 965 + ] + }, + { + "old_start": 974, + "old_len": 9, + "changed_lines": [ + 977, + 980 + ] + }, + { + "old_start": 1021, + "old_len": 12, + "changed_lines": [ + 1024, + 1027, + 1029 + ] + }, + { + "old_start": 1038, + "old_len": 5, + "changed_lines": [ + 1041 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/client/client_impl_test.go", + "created": false, + "hunks": [ + { + "old_start": 2813, + "old_len": 7, + "changed_lines": [ + 2816 + ] + }, + { + "old_start": 2866, + "old_len": 7, + "changed_lines": [ + 2869 + ] + }, + { + "old_start": 2885, + "old_len": 6, + "changed_lines": [ + 2888 + ] + }, + { + "old_start": 3509, + "old_len": 6, + "changed_lines": [ + 3512 + ] + }, + { + "old_start": 3526, + "old_len": 6, + "changed_lines": [ + 3529 + ] + } + ] + }, + { + "path": "pkg/cmd/discussion/create/create.go", + "created": false, + "hunks": [ + { + "old_start": 188, + "old_len": 6, + "changed_lines": [ + 191 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/discussion/edit/edit.go", + "omission_line": 213 + }, + { + "id": "v-16a20347", + "commit": "16a20347dd33b8f67abd8990ab0940d35f522233", + "parent": "55808753070c023c45b40592fbfc624d8f03a759", + "date": "2026-05-20", + "prompt": "fix warning message to make it clear", + "complete": { + "files": [ + { + "path": "pkg/cmd/skills/update/update.go", + "created": false, + "hunks": [ + { + "old_start": 338, + "old_len": 7, + "changed_lines": [ + 341 + ] + } + ] + }, + { + "path": "pkg/cmd/skills/update/update_test.go", + "created": false, + "hunks": [ + { + "old_start": 508, + "old_len": 7, + "changed_lines": [ + 511 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "pkg/cmd/skills/update/update.go", + "created": false, + "hunks": [ + { + "old_start": 338, + "old_len": 7, + "changed_lines": [ + 341 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/skills/update/update_test.go", + "omission_line": 511 + }, + { + "id": "v-fb748cb2", + "commit": "fb748cb2bf3a434ff12f5268c9983a65310c6520", + "parent": "00fc8c923ab3f321a412e1c89e425c107122f0bb", + "date": "2026-05-19", + "prompt": "add logic to preview too", + "complete": { + "files": [ + { + "path": "internal/skills/discovery/discovery.go", + "created": false, + "hunks": [ + { + "old_start": 390, + "old_len": 6, + "changed_lines": [ + 393 + ] + } + ] + }, + { + "path": "internal/skills/discovery/discovery_test.go", + "created": false, + "hunks": [ + { + "old_start": 1526, + "old_len": 6, + "changed_lines": [ + 1529 + ] + } + ] + }, + { + "path": "pkg/cmd/skills/install/install.go", + "created": false, + "hunks": [ + { + "old_start": 551, + "old_len": 23, + "changed_lines": [ + 554, + 555, + 556, + 557, + 558, + 559, + 560, + 561, + 562, + 563, + 564, + 565, + 566, + 567, + 568, + 569, + 570 + ] + } + ] + }, + { + "path": "pkg/cmd/skills/preview/preview.go", + "created": false, + "hunks": [ + { + "old_start": 69, + "old_len": 6, + "changed_lines": [ + 72 + ] + }, + { + "old_start": 82, + "old_len": 6, + "changed_lines": [ + 85 + ] + }, + { + "old_start": 153, + "old_len": 25, + "changed_lines": [ + 156, + 157, + 158, + 159, + 160, + 161, + 163, + 164, + 165, + 166, + 168, + 169, + 170, + 172, + 173, + 174 + ] + } + ] + }, + { + "path": "pkg/cmd/skills/preview/preview_test.go", + "created": false, + "hunks": [ + { + "old_start": 261, + "old_len": 6, + "changed_lines": [ + 264 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "internal/skills/discovery/discovery.go", + "created": false, + "hunks": [ + { + "old_start": 390, + "old_len": 6, + "changed_lines": [ + 393 + ] + } + ] + }, + { + "path": "internal/skills/discovery/discovery_test.go", + "created": false, + "hunks": [ + { + "old_start": 1526, + "old_len": 6, + "changed_lines": [ + 1529 + ] + } + ] + }, + { + "path": "pkg/cmd/skills/install/install.go", + "created": false, + "hunks": [ + { + "old_start": 551, + "old_len": 23, + "changed_lines": [ + 554, + 555, + 556, + 557, + 558, + 559, + 560, + 561, + 562, + 563, + 564, + 565, + 566, + 567, + 568, + 569, + 570 + ] + } + ] + }, + { + "path": "pkg/cmd/skills/preview/preview.go", + "created": false, + "hunks": [ + { + "old_start": 69, + "old_len": 6, + "changed_lines": [ + 72 + ] + }, + { + "old_start": 82, + "old_len": 6, + "changed_lines": [ + 85 + ] + }, + { + "old_start": 153, + "old_len": 25, + "changed_lines": [ + 156, + 157, + 158, + 159, + 160, + 161, + 163, + 164, + 165, + 166, + 168, + 169, + 170, + 172, + 173, + 174 + ] + } + ] + } + ] + }, + "omission_file": "pkg/cmd/skills/preview/preview_test.go", + "omission_line": 264 + }, + { + "id": "v-a44721d2", + "commit": "a44721d233be9a2f6f0b5ee5c4f71274acb8d296", + "parent": "9c4184de6f8c208a11e4329b90fa9844efd728e9", + "date": "2026-05-07", + "prompt": "Add explicit build tags to platform-specific echo test files", + "complete": { + "files": [ + { + "path": "internal/prompter/echo_darwin_test.go", + "created": false, + "hunks": [ + { + "old_start": 1, + "old_len": 3, + "changed_lines": [ + 1 + ] + } + ] + }, + { + "path": "internal/prompter/echo_linux_test.go", + "created": false, + "hunks": [ + { + "old_start": 1, + "old_len": 3, + "changed_lines": [ + 1 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "internal/prompter/echo_darwin_test.go", + "created": false, + "hunks": [ + { + "old_start": 1, + "old_len": 3, + "changed_lines": [ + 1 + ] + } + ] + } + ] + }, + "omission_file": "internal/prompter/echo_linux_test.go", + "omission_line": 1 + } + ] +} \ No newline at end of file diff --git a/benchmarks/results/verify-reify/verify-environment.json b/benchmarks/results/verify-reify/verify-environment.json new file mode 100644 index 0000000..ea65ff7 --- /dev/null +++ b/benchmarks/results/verify-reify/verify-environment.json @@ -0,0 +1,25 @@ +{ + "after": null, + "candidates_rejected": 0, + "checker": "symbols changed by this diff, minus symbols present in the diff, where an inbound CALLS edge exists at distance 1, via reify::query::impact", + "count": 60, + "head": "0b0bcf5cf5fc25f4c7325f108a0474e1d2895cda", + "languages": [ + [ + "rust", + 31 + ], + [ + "python", + 12 + ] + ], + "origin": "git@github.com:lambiengcode/reify.git", + "reify_version": "0.2.2", + "repository": ".", + "scan": 4000, + "token_counts": "estimated by reify heuristic-v1", + "trials": 4, + "until": "870905b", + "wall_clock_ms": 867 +} \ No newline at end of file diff --git a/benchmarks/results/verify-reify/verify-outcomes.json b/benchmarks/results/verify-reify/verify-outcomes.json new file mode 100644 index 0000000..9d85313 --- /dev/null +++ b/benchmarks/results/verify-reify/verify-outcomes.json @@ -0,0 +1,270 @@ +[ + { + "task": "v-9af59e47", + "commit": "9af59e472b5bac3dae35323bcac223c922b89882", + "omission_file": "crates/reify/tests/fixtures.rs", + "omission_symbol": null, + "omission_file_reachable": true, + "file_hit": true, + "file_hit_attributable": false, + "symbol_hit": null, + "findings": 19, + "false_alarms": 19, + "changed_symbols": 2, + "verify_tokens": 422, + "verify_latency_ms": 1, + "index_ms": 151, + "cited": [ + "crates/reify-cli/src/main.rs:212", + "crates/reify/benches/queries.rs:28", + "crates/reify/benches/queries.rs:44", + "crates/reify/src/context.rs:1685", + "crates/reify/src/context.rs:2023", + "crates/reify/src/index.rs:1143", + "crates/reify/src/index.rs:1151", + "crates/reify/src/index.rs:1282", + "crates/reify/src/index.rs:1300", + "crates/reify/src/index.rs:1326", + "crates/reify/src/index.rs:1456", + "crates/reify/src/index.rs:1486", + "crates/reify/src/index.rs:1510", + "crates/reify/src/index.rs:1530", + "crates/reify/src/index.rs:1674", + "crates/reify/src/query.rs:1182", + "crates/reify/tests/fixtures.rs:292", + "crates/reify/tests/fixtures.rs:30", + "crates/reify/tests/fixtures.rs:312" + ], + "cited_on_complete": [ + "crates/reify-cli/src/main.rs:212", + "crates/reify/benches/queries.rs:28", + "crates/reify/benches/queries.rs:44", + "crates/reify/src/context.rs:1685", + "crates/reify/src/context.rs:2023", + "crates/reify/src/index.rs:1143", + "crates/reify/src/index.rs:1151", + "crates/reify/src/index.rs:1282", + "crates/reify/src/index.rs:1300", + "crates/reify/src/index.rs:1326", + "crates/reify/src/index.rs:1456", + "crates/reify/src/index.rs:1486", + "crates/reify/src/index.rs:1510", + "crates/reify/src/index.rs:1530", + "crates/reify/src/index.rs:1674", + "crates/reify/src/query.rs:1182", + "crates/reify/tests/fixtures.rs:292", + "crates/reify/tests/fixtures.rs:30", + "crates/reify/tests/fixtures.rs:312" + ] + }, + { + "task": "v-2b7bad4c", + "commit": "2b7bad4ccaca00cc8d751047361ade9a5066c91b", + "omission_file": "assets/make-logo.py", + "omission_symbol": null, + "omission_file_reachable": false, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": null, + "findings": 1, + "false_alarms": 1, + "changed_symbols": 2, + "verify_tokens": 25, + "verify_latency_ms": 0, + "index_ms": 151, + "cited": [ + "assets/make-social-preview.py:136" + ], + "cited_on_complete": [ + "assets/make-social-preview.py:136" + ] + }, + { + "task": "v-deb46ef8", + "commit": "deb46ef835c1694324e161b1055759c391f883a0", + "omission_file": "crates/reify-cli/src/render.rs", + "omission_symbol": null, + "omission_file_reachable": true, + "file_hit": false, + "file_hit_attributable": false, + "symbol_hit": null, + "findings": 4, + "false_alarms": 4, + "changed_symbols": 2, + "verify_tokens": 89, + "verify_latency_ms": 0, + "index_ms": 149, + "cited": [ + "crates/reify-cli/src/main.rs:167", + "crates/reify-cli/src/mcp.rs:231", + "crates/reify-cli/src/mcp.rs:240", + "crates/reify-cli/src/mcp.rs:61" + ], + "cited_on_complete": [ + "crates/reify-cli/src/main.rs:167", + "crates/reify-cli/src/mcp.rs:231", + "crates/reify-cli/src/mcp.rs:240", + "crates/reify-cli/src/mcp.rs:61" + ] + }, + { + "task": "v-7dd36dae", + "commit": "7dd36daec587f5c19afe6cab881886da530d3d15", + "omission_file": "crates/reify-bench/src/conditions.rs", + "omission_symbol": null, + "omission_file_reachable": true, + "file_hit": true, + "file_hit_attributable": false, + "symbol_hit": null, + "findings": 70, + "false_alarms": 70, + "changed_symbols": 8, + "verify_tokens": 1557, + "verify_latency_ms": 2, + "index_ms": 144, + "cited": [ + "crates/reify-bench/src/conditions.rs:146", + "crates/reify-bench/src/conditions.rs:151", + "crates/reify-bench/src/conditions.rs:216", + "crates/reify-bench/src/main.rs:120", + "crates/reify-bench/src/main.rs:372", + "crates/reify-cli/src/main.rs:174", + "crates/reify-cli/src/mcp.rs:122", + "crates/reify/benches/queries.rs:74", + "crates/reify/src/concepts.rs:1030", + "crates/reify/src/concepts.rs:1145", + "crates/reify/src/concepts.rs:1371", + "crates/reify/src/concepts.rs:1426", + "crates/reify/src/concepts.rs:392", + "crates/reify/src/concepts.rs:472", + "crates/reify/src/concepts.rs:505", + "crates/reify/src/concepts.rs:701", + "crates/reify/src/context.rs:1086", + "crates/reify/src/context.rs:1107", + "crates/reify/src/context.rs:1133", + "crates/reify/src/context.rs:1152", + "crates/reify/src/context.rs:1169", + "crates/reify/src/context.rs:1193", + "crates/reify/src/context.rs:1285", + "crates/reify/src/context.rs:1300", + "crates/reify/src/context.rs:1326", + "crates/reify/src/context.rs:1352", + "crates/reify/src/context.rs:1375", + "crates/reify/src/context.rs:1411", + "crates/reify/src/context.rs:1433", + "crates/reify/src/context.rs:1458", + "crates/reify/src/context.rs:1481", + "crates/reify/src/context.rs:1494", + "crates/reify/src/context.rs:172", + "crates/reify/src/discover.rs:121", + "crates/reify/src/extract/code.rs:1214", + "crates/reify/src/extract/code.rs:1249", + "crates/reify/src/extract/code.rs:31", + "crates/reify/src/extract/code.rs:602", + "crates/reify/src/extract/code.rs:834", + "crates/reify/src/extract/code.rs:851", + "crates/reify/src/extract/code.rs:876", + "crates/reify/src/extract/code.rs:895", + "crates/reify/src/extract/code.rs:909", + "crates/reify/src/extract/code.rs:923", + "crates/reify/src/extract/docs.rs:30", + "crates/reify/src/extract/schema.rs:199", + "crates/reify/src/extract/schema.rs:207", + "crates/reify/src/extract/schema.rs:215", + "crates/reify/src/extract/sqlish.rs:146", + "crates/reify/src/extract/sqlish.rs:167", + "crates/reify/src/gitlog.rs:208", + "crates/reify/src/gitlog.rs:527", + "crates/reify/src/gitlog.rs:538", + "crates/reify/src/gitlog.rs:553", + "crates/reify/src/gitlog.rs:571", + "crates/reify/src/index.rs:1484", + "crates/reify/src/index.rs:1518", + "crates/reify/src/index.rs:1565", + "crates/reify/src/index.rs:1580", + "crates/reify/src/index.rs:209", + "crates/reify/src/index.rs:609", + "crates/reify/src/index.rs:892", + "crates/reify/src/rules.rs:619", + "crates/reify/src/store.rs:997", + "crates/reify/tests/fixtures.rs:109", + "crates/reify/tests/fixtures.rs:143", + "crates/reify/tests/fixtures.rs:159", + "crates/reify/tests/fixtures.rs:190", + "crates/reify/tests/fixtures.rs:210", + "crates/reify/tests/fixtures.rs:231" + ], + "cited_on_complete": [ + "crates/reify-bench/src/conditions.rs:146", + "crates/reify-bench/src/conditions.rs:151", + "crates/reify-bench/src/conditions.rs:216", + "crates/reify-bench/src/main.rs:120", + "crates/reify-bench/src/main.rs:372", + "crates/reify-cli/src/main.rs:174", + "crates/reify-cli/src/mcp.rs:122", + "crates/reify/benches/queries.rs:74", + "crates/reify/src/concepts.rs:1030", + "crates/reify/src/concepts.rs:1145", + "crates/reify/src/concepts.rs:1371", + "crates/reify/src/concepts.rs:1426", + "crates/reify/src/concepts.rs:392", + "crates/reify/src/concepts.rs:472", + "crates/reify/src/concepts.rs:505", + "crates/reify/src/concepts.rs:701", + "crates/reify/src/context.rs:1086", + "crates/reify/src/context.rs:1107", + "crates/reify/src/context.rs:1133", + "crates/reify/src/context.rs:1152", + "crates/reify/src/context.rs:1169", + "crates/reify/src/context.rs:1193", + "crates/reify/src/context.rs:1285", + "crates/reify/src/context.rs:1300", + "crates/reify/src/context.rs:1326", + "crates/reify/src/context.rs:1352", + "crates/reify/src/context.rs:1375", + "crates/reify/src/context.rs:1411", + "crates/reify/src/context.rs:1433", + "crates/reify/src/context.rs:1458", + "crates/reify/src/context.rs:1481", + "crates/reify/src/context.rs:1494", + "crates/reify/src/context.rs:172", + "crates/reify/src/discover.rs:121", + "crates/reify/src/extract/code.rs:1214", + "crates/reify/src/extract/code.rs:1249", + "crates/reify/src/extract/code.rs:31", + "crates/reify/src/extract/code.rs:602", + "crates/reify/src/extract/code.rs:834", + "crates/reify/src/extract/code.rs:851", + "crates/reify/src/extract/code.rs:876", + "crates/reify/src/extract/code.rs:895", + "crates/reify/src/extract/code.rs:909", + "crates/reify/src/extract/code.rs:923", + "crates/reify/src/extract/docs.rs:30", + "crates/reify/src/extract/schema.rs:199", + "crates/reify/src/extract/schema.rs:207", + "crates/reify/src/extract/schema.rs:215", + "crates/reify/src/extract/sqlish.rs:146", + "crates/reify/src/extract/sqlish.rs:167", + "crates/reify/src/gitlog.rs:208", + "crates/reify/src/gitlog.rs:527", + "crates/reify/src/gitlog.rs:538", + "crates/reify/src/gitlog.rs:553", + "crates/reify/src/gitlog.rs:571", + "crates/reify/src/index.rs:1484", + "crates/reify/src/index.rs:1518", + "crates/reify/src/index.rs:1565", + "crates/reify/src/index.rs:1580", + "crates/reify/src/index.rs:209", + "crates/reify/src/index.rs:609", + "crates/reify/src/index.rs:892", + "crates/reify/src/rules.rs:619", + "crates/reify/src/store.rs:997", + "crates/reify/tests/fixtures.rs:109", + "crates/reify/tests/fixtures.rs:143", + "crates/reify/tests/fixtures.rs:159", + "crates/reify/tests/fixtures.rs:190", + "crates/reify/tests/fixtures.rs:210", + "crates/reify/tests/fixtures.rs:231" + ] + } +] \ No newline at end of file diff --git a/benchmarks/results/verify-reify/verify-summary.json b/benchmarks/results/verify-reify/verify-summary.json new file mode 100644 index 0000000..3a384b1 --- /dev/null +++ b/benchmarks/results/verify-reify/verify-summary.json @@ -0,0 +1,33 @@ +{ + "tasks": 4, + "omission_recall": 0.5, + "omission_recall_ci": [ + 0.15003571, + 0.84996426 + ], + "omission_recall_attributable": 0.0, + "omission_recall_attributable_ci": [ + 0.0, + 0.48990002 + ], + "reachable_omissions": 3, + "omission_recall_reachable": 0.6666667, + "omission_recall_reachable_ci": [ + 0.20765498, + 0.9385097 + ], + "symbol_scorable": 0, + "omission_recall_symbol": null, + "omission_recall_symbol_ci": null, + "false_alarm_rate": 23.5, + "commits_with_a_false_alarm": 4, + "false_alarm_share_ci": [ + 0.5101, + 1.0 + ], + "median_findings_per_diff": 19, + "median_verify_tokens": 422, + "median_verify_latency_ms": 1, + "median_index_ms": 151, + "diffs_resolving_to_nothing": 0 +} \ No newline at end of file diff --git a/benchmarks/results/verify-reify/verify-tasks.json b/benchmarks/results/verify-reify/verify-tasks.json new file mode 100644 index 0000000..aeb7218 --- /dev/null +++ b/benchmarks/results/verify-reify/verify-tasks.json @@ -0,0 +1,852 @@ +{ + "repository": ".", + "head": "0b0bcf5cf5fc25f4c7325f108a0474e1d2895cda", + "generated_from_commits": 48, + "rejected": [], + "tasks": [ + { + "id": "v-9af59e47", + "commit": "9af59e472b5bac3dae35323bcac223c922b89882", + "parent": "8c9e91d5b8a9dece2dc4edea705b96cb5dde6954", + "date": "2026-08-24", + "prompt": "a repository whose history git cannot read still indexes", + "complete": { + "files": [ + { + "path": "crates/reify/src/index.rs", + "created": false, + "hunks": [ + { + "old_start": 189, + "old_len": 6, + "changed_lines": [ + 192 + ] + }, + { + "old_start": 615, + "old_len": 7, + "changed_lines": [ + 618 + ] + } + ] + }, + { + "path": "crates/reify/tests/fixtures.rs", + "created": false, + "hunks": [ + { + "old_start": 365, + "old_len": 3, + "changed_lines": [ + 368 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "crates/reify/src/index.rs", + "created": false, + "hunks": [ + { + "old_start": 189, + "old_len": 6, + "changed_lines": [ + 192 + ] + }, + { + "old_start": 615, + "old_len": 7, + "changed_lines": [ + 618 + ] + } + ] + } + ] + }, + "omission_file": "crates/reify/tests/fixtures.rs", + "omission_line": 368 + }, + { + "id": "v-2b7bad4c", + "commit": "2b7bad4ccaca00cc8d751047361ade9a5066c91b", + "parent": "5a0d4c70e0962f82125f21e9bbb50c1abede9fa0", + "date": "2026-08-22", + "prompt": "a hand-drawn mascot, and one master it all derives from", + "complete": { + "files": [ + { + "path": "assets/make-logo.py", + "created": false, + "hunks": [ + { + "old_start": 1, + "old_len": 96, + "changed_lines": [ + 2, + 3, + 4, + 5, + 6, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93 + ] + } + ] + }, + { + "path": "assets/make-social-preview.py", + "created": false, + "hunks": [ + { + "old_start": 17, + "old_len": 6, + "changed_lines": [ + 20 + ] + }, + { + "old_start": 59, + "old_len": 23, + "changed_lines": [ + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78 + ] + }, + { + "old_start": 97, + "old_len": 13, + "changed_lines": [ + 100, + 105, + 106 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "assets/make-social-preview.py", + "created": false, + "hunks": [ + { + "old_start": 17, + "old_len": 6, + "changed_lines": [ + 20 + ] + }, + { + "old_start": 59, + "old_len": 23, + "changed_lines": [ + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78 + ] + }, + { + "old_start": 97, + "old_len": 13, + "changed_lines": [ + 100, + 105, + 106 + ] + } + ] + } + ] + }, + "omission_file": "assets/make-logo.py", + "omission_line": 2 + }, + { + "id": "v-deb46ef8", + "commit": "deb46ef835c1694324e161b1055759c391f883a0", + "parent": "cbfcad5a1756473d49576a47242de80392514a43", + "date": "2026-08-21", + "prompt": "TOON output for agents — 57% fewer tokens than the JSON envelope, measured cost in the header", + "complete": { + "files": [ + { + "path": "crates/reify-cli/src/main.rs", + "created": false, + "hunks": [ + { + "old_start": 76, + "old_len": 6, + "changed_lines": [ + 79 + ] + }, + { + "old_start": 212, + "old_len": 6, + "changed_lines": [ + 215 + ] + }, + { + "old_start": 223, + "old_len": 6, + "changed_lines": [ + 226 + ] + }, + { + "old_start": 435, + "old_len": 7, + "changed_lines": [ + 438 + ] + } + ] + }, + { + "path": "crates/reify-cli/src/mcp.rs", + "created": false, + "hunks": [ + { + "old_start": 143, + "old_len": 6, + "changed_lines": [ + 146 + ] + }, + { + "old_start": 167, + "old_len": 7, + "changed_lines": [ + 170 + ] + } + ] + }, + { + "path": "crates/reify-cli/src/render.rs", + "created": false, + "hunks": [ + { + "old_start": 863, + "old_len": 3, + "changed_lines": [ + 866 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "crates/reify-cli/src/main.rs", + "created": false, + "hunks": [ + { + "old_start": 76, + "old_len": 6, + "changed_lines": [ + 79 + ] + }, + { + "old_start": 212, + "old_len": 6, + "changed_lines": [ + 215 + ] + }, + { + "old_start": 223, + "old_len": 6, + "changed_lines": [ + 226 + ] + }, + { + "old_start": 435, + "old_len": 7, + "changed_lines": [ + 438 + ] + } + ] + }, + { + "path": "crates/reify-cli/src/mcp.rs", + "created": false, + "hunks": [ + { + "old_start": 143, + "old_len": 6, + "changed_lines": [ + 146 + ] + }, + { + "old_start": 167, + "old_len": 7, + "changed_lines": [ + 170 + ] + } + ] + } + ] + }, + "omission_file": "crates/reify-cli/src/render.rs", + "omission_line": 866 + }, + { + "id": "v-7dd36dae", + "commit": "7dd36daec587f5c19afe6cab881886da530d3d15", + "parent": "b65dcf2cb0aa1ca550c24549ae19901d7162e1ce", + "date": "2026-08-21", + "prompt": "verbatim identifier lookup, stemmed prefix search, file-aggregate ordering, offer cutoff; bench: rank audit", + "complete": { + "files": [ + { + "path": "crates/reify-bench/src/conditions.rs", + "created": false, + "hunks": [ + { + "old_start": 328, + "old_len": 3, + "changed_lines": [ + 331 + ] + } + ] + }, + { + "path": "crates/reify-bench/src/main.rs", + "created": false, + "hunks": [ + { + "old_start": 80, + "old_len": 6, + "changed_lines": [ + 83 + ] + }, + { + "old_start": 166, + "old_len": 12, + "changed_lines": [ + 169, + 175 + ] + }, + { + "old_start": 212, + "old_len": 99, + "changed_lines": [ + 215, + 216, + 217, + 218, + 219, + 220, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 235, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 292, + 293, + 299, + 303, + 304, + 305, + 306, + 307 + ] + }, + { + "old_start": 318, + "old_len": 6, + "changed_lines": [ + 321 + ] + } + ] + }, + { + "path": "crates/reify/src/context.rs", + "created": false, + "hunks": [ + { + "old_start": 56, + "old_len": 11, + "changed_lines": [ + 59, + 64 + ] + }, + { + "old_start": 141, + "old_len": 6, + "changed_lines": [ + 144 + ] + }, + { + "old_start": 151, + "old_len": 6, + "changed_lines": [ + 154 + ] + }, + { + "old_start": 340, + "old_len": 10, + "changed_lines": [ + 343, + 346 + ] + }, + { + "old_start": 361, + "old_len": 7, + "changed_lines": [ + 364 + ] + }, + { + "old_start": 425, + "old_len": 6, + "changed_lines": [ + 428 + ] + }, + { + "old_start": 846, + "old_len": 32, + "changed_lines": [ + 849, + 852, + 853, + 854, + 856, + 862, + 863, + 864, + 865, + 867, + 868, + 869, + 870, + 871, + 872, + 874 + ] + } + ] + }, + { + "path": "crates/reify/src/store.rs", + "created": false, + "hunks": [ + { + "old_start": 1097, + "old_len": 7, + "changed_lines": [ + 1100 + ] + }, + { + "old_start": 1548, + "old_len": 6, + "changed_lines": [ + 1551 + ] + } + ] + } + ] + }, + "truncated": { + "files": [ + { + "path": "crates/reify-bench/src/main.rs", + "created": false, + "hunks": [ + { + "old_start": 80, + "old_len": 6, + "changed_lines": [ + 83 + ] + }, + { + "old_start": 166, + "old_len": 12, + "changed_lines": [ + 169, + 175 + ] + }, + { + "old_start": 212, + "old_len": 99, + "changed_lines": [ + 215, + 216, + 217, + 218, + 219, + 220, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 235, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 292, + 293, + 299, + 303, + 304, + 305, + 306, + 307 + ] + }, + { + "old_start": 318, + "old_len": 6, + "changed_lines": [ + 321 + ] + } + ] + }, + { + "path": "crates/reify/src/context.rs", + "created": false, + "hunks": [ + { + "old_start": 56, + "old_len": 11, + "changed_lines": [ + 59, + 64 + ] + }, + { + "old_start": 141, + "old_len": 6, + "changed_lines": [ + 144 + ] + }, + { + "old_start": 151, + "old_len": 6, + "changed_lines": [ + 154 + ] + }, + { + "old_start": 340, + "old_len": 10, + "changed_lines": [ + 343, + 346 + ] + }, + { + "old_start": 361, + "old_len": 7, + "changed_lines": [ + 364 + ] + }, + { + "old_start": 425, + "old_len": 6, + "changed_lines": [ + 428 + ] + }, + { + "old_start": 846, + "old_len": 32, + "changed_lines": [ + 849, + 852, + 853, + 854, + 856, + 862, + 863, + 864, + 865, + 867, + 868, + 869, + 870, + 871, + 872, + 874 + ] + } + ] + }, + { + "path": "crates/reify/src/store.rs", + "created": false, + "hunks": [ + { + "old_start": 1097, + "old_len": 7, + "changed_lines": [ + 1100 + ] + }, + { + "old_start": 1548, + "old_len": 6, + "changed_lines": [ + 1551 + ] + } + ] + } + ] + }, + "omission_file": "crates/reify-bench/src/conditions.rs", + "omission_line": 331 + } + ] +} \ No newline at end of file diff --git a/crates/reify-bench/src/conditions.rs b/crates/reify-bench/src/conditions.rs index 80906ad..a9af177 100644 --- a/crates/reify-bench/src/conditions.rs +++ b/crates/reify-bench/src/conditions.rs @@ -14,9 +14,13 @@ use std::collections::{BTreeSet, HashMap}; use reify::concepts::meaningful_words; use reify::context::{self, ContextOptions}; -use reify::store::Store; +use reify::model::{EdgeKind, Node}; +use reify::query; +use reify::store::{Direction, Store}; use reify::tokens; +use crate::tasks; + /// A condition's answer: an ordered list of files, and what it cost to produce. #[derive(Debug, Clone, Serialize)] pub struct Answer { @@ -345,6 +349,170 @@ pub fn rank_audit( }) } +// ---- the checker under test ------------------------------------------------- +// +// `reify verify` does not exist. What exists is the graph it would have to stand on, +// and that is what is measured here: *symbols changed by this diff, minus symbols +// present in the diff, where an inbound `CALLS` edge exists*. The query runs through +// the same `impact` machinery `reify impact` uses, so a number measured here is a +// number about the shipped substrate rather than about a checker written to be +// measured. +// +// Only `CALLS` edges at distance 1 count. `impact` also propagates two hops and +// crosses into the data layer; both are legitimate for "what breaks if I change +// this" and neither is what "the patch forgot to update a call site" means. Widening +// the query would raise recall and raise the false-alarm rate with it, which is the +// trade this benchmark exists to measure rather than to pre-empt. + +/// One thing the change touched that has a dependant the change did not touch. +#[derive(Debug, Clone, Serialize)] +pub struct Finding { + /// `path:line` of the dependant, as `reify impact` cites it. + pub location: String, + pub path: String, + /// The dependant's name. + pub what: String, + /// The changed symbol it depends on, in words an engineer can check. + pub reason: String, +} + +/// What the checker produced for one diff, and what it cost. +#[derive(Debug, Clone, Serialize)] +pub struct Findings { + pub findings: Vec, + /// Symbols the diff changes, `path:line`. The minuend of the query, reported so a + /// zero-finding result can be told apart from a diff that resolved to nothing. + pub changed_symbols: Vec, + /// Tokens the findings output itself would cost the agent that reads it. + pub answer_tokens: u32, + /// Wall clock for the query alone. Indexing is a one-off the real feature would + /// not repeat per check, and is timed separately. + pub elapsed_ms: u128, +} + +impl Findings { + /// The findings as an agent would be shown them; the string `answer_tokens` counts. + pub fn render(&self) -> String { + if self.findings.is_empty() { + return "reify verify: nothing in the graph says this patch is incomplete\n".into(); + } + let mut out = format!( + "reify verify: {} not updated by this patch\n", + self.findings.len() + ); + for finding in &self.findings { + out.push_str(&format!( + " {} {} — {}\n", + finding.location, finding.what, finding.reason + )); + } + out + } +} + +/// Does any symbol in `path` **call** a symbol in another file? +/// +/// The ceiling on this whole construction. Every finding is a caller, so a file whose +/// symbols call nothing outside themselves can never be cited, however good the query +/// gets. The direction matters and is easy to get backwards: what is called *into* +/// the file is irrelevant here. +/// +/// Measured rather than assumed, because "the query needs work" and "there is no edge +/// to find" are different conclusions and only one of them is fixable by writing +/// `reify verify`. +pub fn can_be_cited(store: &Store, path: &str) -> Result { + for symbol in store.symbols_in_file(path)? { + for (callee, _, _) in store.neighbors(symbol.id, Direction::Out, &[EdgeKind::Calls])? { + if callee.path.as_deref() != Some(path) { + return Ok(true); + } + } + } + Ok(false) +} + +/// Run the checker over one patch, against an index built at the patch's parent. +pub fn missing_callers(store: &Store, patch: &tasks::Patch) -> Result { + let started = std::time::Instant::now(); + + // Every symbol whose span overlaps a changed line. This is the exclusion set: + // a symbol the patch already edits is not something the patch forgot. + let mut touched: BTreeSet = BTreeSet::new(); + // The innermost symbol at each changed line. These are the origins — the same + // rule `store.symbol_at` applies, batched so a long hunk costs one query. + let mut origins: Vec = Vec::new(); + let mut seen: BTreeSet = BTreeSet::new(); + + for file in &patch.files { + if file.created { + continue; + } + let symbols = store.symbols_in_file(&file.path)?; + if symbols.is_empty() { + continue; + } + for hunk in &file.hunks { + for &line in &hunk.changed_lines { + let mut innermost: Option<&Node> = None; + for symbol in &symbols { + if symbol.line_start > line || symbol.line_end < line { + continue; + } + touched.insert(symbol.location()); + let narrower = innermost.is_none_or(|best| { + symbol.line_end - symbol.line_start < best.line_end - best.line_start + }); + if narrower { + innermost = Some(symbol); + } + } + if let Some(symbol) = innermost { + if seen.insert(symbol.id) { + origins.push(symbol.clone()); + } + } + } + } + } + + let mut findings: Vec = Vec::new(); + let mut cited: BTreeSet = BTreeSet::new(); + for origin in &origins { + let answer = query::impact(store, &origin.location())?; + for affected in answer.affected { + // Distance 1 and a call: the edge the query is defined on. Data coupling + // and callers-of-callers are `impact`'s job, not this checker's. + if affected.distance != 1 || !affected.reason.starts_with("calls ") { + continue; + } + if touched.contains(&affected.location) || !cited.insert(affected.location.clone()) { + continue; + } + let path = affected + .location + .rsplit_once(':') + .map_or(affected.location.as_str(), |(path, _)| path) + .to_string(); + findings.push(Finding { + location: affected.location, + path, + what: affected.what, + reason: affected.reason, + }); + } + } + findings.sort_by(|a, b| a.location.cmp(&b.location)); + + let mut result = Findings { + findings, + changed_symbols: origins.iter().map(|n| n.location()).collect(), + answer_tokens: 0, + elapsed_ms: started.elapsed().as_millis(), + }; + result.answer_tokens = tokens::estimate(&result.render()); + Ok(result) +} + #[cfg(test)] mod tests { use super::*; @@ -371,6 +539,111 @@ mod tests { } } + /// Three symbols: `caller` and `sibling` both call `target`, all in different + /// files, plus one symbol nothing calls. + fn graph() -> Store { + use reify::model::{uid, EdgeKind, NewEdge, NewNode, NodeKind, Status}; + use reify::store::Batch; + + let symbol = |path: &str, name: &str, start: u32, end: u32| { + let mut node = NewNode::new(uid::symbol(path, name), NodeKind::Symbol, name); + node.path = Some(path.to_string()); + node.line_start = start; + node.line_end = end; + node + }; + let mut batch = Batch::default(); + batch.node(symbol("app/pricing.py", "target", 10, 20)); + batch.node(symbol("app/orders.py", "caller", 5, 15)); + batch.node(symbol("app/report.py", "sibling", 30, 40)); + batch.node(symbol("app/lonely.py", "lonely", 1, 4)); + for from in ["app/orders.py#caller", "app/report.py#sibling"] { + let (path, name) = from.split_once('#').unwrap(); + batch.edge(NewEdge::new( + uid::symbol(path, name), + uid::symbol("app/pricing.py", "target"), + EdgeKind::Calls, + Status::Confirmed, + 1.0, + )); + } + let mut store = Store::in_memory().unwrap(); + store.commit(batch).unwrap(); + store + } + + fn patch(files: &[(&str, u32)]) -> tasks::Patch { + tasks::Patch { + files: files + .iter() + .map(|(path, line)| tasks::FilePatch { + path: path.to_string(), + created: false, + hunks: vec![tasks::Hunk { + old_start: *line, + old_len: 1, + changed_lines: vec![*line], + }], + }) + .collect(), + } + } + + #[test] + fn a_caller_the_patch_did_not_touch_is_a_finding() { + let found = missing_callers(&graph(), &patch(&[("app/pricing.py", 12)])).unwrap(); + let cited: Vec<&str> = found.findings.iter().map(|f| f.path.as_str()).collect(); + assert_eq!(cited, vec!["app/orders.py", "app/report.py"]); + assert_eq!(found.changed_symbols, vec!["app/pricing.py:10"]); + } + + #[test] + fn a_caller_the_patch_did_touch_is_not_a_finding() { + // This is the whole subtrahend: a symbol the patch already edits is not + // something the patch forgot. Without it every complete commit would be + // reported as incomplete. + let found = missing_callers( + &graph(), + &patch(&[("app/pricing.py", 12), ("app/orders.py", 7)]), + ) + .unwrap(); + let cited: Vec<&str> = found.findings.iter().map(|f| f.path.as_str()).collect(); + assert_eq!(cited, vec!["app/report.py"]); + } + + #[test] + fn a_complete_change_leaves_nothing_to_report() { + let found = missing_callers( + &graph(), + &patch(&[ + ("app/pricing.py", 12), + ("app/orders.py", 7), + ("app/report.py", 33), + ]), + ) + .unwrap(); + assert!(found.findings.is_empty(), "{:?}", found.findings); + assert!(found.render().contains("nothing")); + } + + #[test] + fn a_diff_that_resolves_to_no_symbol_reports_nothing_rather_than_guessing() { + let found = missing_callers(&graph(), &patch(&[("app/pricing.py", 900)])).unwrap(); + assert!(found.changed_symbols.is_empty()); + assert!(found.findings.is_empty()); + } + + #[test] + fn only_a_file_that_calls_out_of_itself_can_ever_be_cited() { + // The ceiling on the held-out-hunk construction, and the direction is easy to + // get backwards: `pricing.py` is called *by* two files and calls nothing, so no + // caller-based checker can cite it. + let store = graph(); + assert!(can_be_cited(&store, "app/orders.py").unwrap()); + assert!(!can_be_cited(&store, "app/pricing.py").unwrap()); + assert!(!can_be_cited(&store, "app/lonely.py").unwrap()); + } + #[test] fn content_search_prefers_files_matching_more_distinct_terms() { // pricing.py contains all three task terms; huge.py contains two of them many diff --git a/crates/reify-bench/src/main.rs b/crates/reify-bench/src/main.rs index 626ded3..929f79f 100644 --- a/crates/reify-bench/src/main.rs +++ b/crates/reify-bench/src/main.rs @@ -141,6 +141,40 @@ enum Command { #[arg(long)] out: PathBuf, }, + /// Held-out-hunk evaluation: does the graph notice an incomplete patch? + /// + /// Model-free, deterministic, and free to run. It exists to decide whether + /// `reify verify` is worth building, against the pre-registered condition in + /// `metrics::VERIFY_RECALL_FLOOR`. + VerifyEval { + #[arg(long)] + repo: PathBuf, + #[arg(long)] + out: PathBuf, + #[arg(long, default_value_t = 20)] + count: usize, + #[arg(long, default_value_t = 4_000)] + scan: usize, + /// Only take trials from commits after this revision. + #[arg(long)] + after: Option, + /// Only take trials from commits strictly older than this revision. + #[arg(long)] + until: Option, + /// Where parent trees are extracted. Defaults to a temporary directory, which + /// is removed when the run finishes. + #[arg(long)] + work: Option, + }, + /// Render the held-out-hunk report from one or more `verify-eval` result + /// directories. + VerifyReport { + /// Result directories, as `Label=path`, in the order they should appear. + #[arg(long = "results", value_name = "LABEL=DIR", num_args = 1..)] + results: Vec, + #[arg(long)] + out: PathBuf, + }, /// Render a report from raw results. Report { #[arg(long = "in")] @@ -223,6 +257,24 @@ fn run() -> Result<()> { } => audit(&repo, &tasks, budget), Command::Fit { train, out, budget } => fit(&train, &out, budget), Command::Chart { results, out } => charts(&results, &out), + Command::VerifyEval { + repo, + out, + count, + scan, + after, + until, + work, + } => verify_eval( + &repo, + &out, + count, + scan, + after.as_deref(), + until.as_deref(), + work.as_deref(), + ), + Command::VerifyReport { results, out } => verify_report(&results, &out), Command::Report { input, out } => report(&input, &out), } } @@ -555,8 +607,10 @@ fn agent_experiments( ) -> Result<()> { let wanted = |name: &str| arms.is_empty() || arms.iter().any(|a| a == name); let set: tasks::TaskSet = read_json(task_file)?; + let name = set.repository_name().to_string(); + let name = name.as_str(); let provider = agent::provider_or_explain(repo)?; - eprintln!("provider: {}", provider.label); + eprintln!("provider: {} repository: {name}", provider.label); let store_path = repo .join(reify::index::REIFY_DIR) @@ -573,7 +627,7 @@ fn agent_experiments( // E6: memorisation control. No context at all. if wanted("N-no-context") { - outcomes.push(agent::run(&provider, repo, task, "N-no-context", "")); + outcomes.push(agent::run(&provider, repo, name, task, "N-no-context", "")); } // E1: the budget-matched lexical baseline. @@ -582,6 +636,7 @@ fn agent_experiments( outcomes.push(agent::run( &provider, repo, + name, task, "B-content-grep", &agent::files_block(&grep), @@ -594,6 +649,7 @@ fn agent_experiments( outcomes.push(agent::run( &provider, repo, + name, task, "R-reify", &agent::files_block(&compiled), @@ -612,6 +668,7 @@ fn agent_experiments( outcomes.push(agent::run( &provider, repo, + name, task, "R-shuffled", &agent::files_block(&shuffled), @@ -623,6 +680,7 @@ fn agent_experiments( outcomes.push(agent::run( &provider, repo, + name, task, "O-oracle", &agent::oracle_block(task), @@ -637,6 +695,7 @@ fn agent_experiments( outcomes.push(agent::run( &provider, repo, + name, task, "R-reify-iter3", &agent::files_block(&iterated), @@ -647,6 +706,7 @@ fn agent_experiments( outcomes.push(agent::run( &provider, repo, + name, task, "B-content-grep-x3", &agent::files_block(&grep_wide), @@ -679,6 +739,7 @@ fn agent_experiments( &out.join("agent-environment.json"), &serde_json::json!({ "provider": provider.label, + "repository": name, "tasks": chosen.len(), "budget_tokens": budget, "conditions": names, @@ -818,6 +879,607 @@ fn execute( Ok(()) } +/// Held-out-hunk evaluation: can the graph tell that a patch is incomplete? +/// +/// Model-free and deterministic. For each qualifying merged commit the parent tree is +/// extracted and indexed, the change is fed to the checker twice — once with one file's +/// only hunk withheld, once complete — and the two runs answer two different questions: +/// does a finding cite the withheld hunk, and how many findings does a change that is +/// complete by construction still attract. +fn verify_eval( + repo: &Path, + out: &Path, + count: usize, + scan: usize, + after: Option<&str>, + until: Option<&str>, + work: Option<&Path>, +) -> Result<()> { + let started = std::time::Instant::now(); + let set = tasks::generate_truncated( + repo, + count, + scan, + after, + until, + &std::collections::BTreeSet::new(), + )?; + anyhow::ensure!( + !set.tasks.is_empty(), + "no commit in the scanned history could be truncated; {} candidates were \ + rejected, the commonest reason being `{}`", + set.rejected.len(), + set.rejected + .first() + .map(|(_, why)| why.as_str()) + .unwrap_or("none recorded"), + ); + eprintln!( + "{} trials from {} commits ({} passed every retrieval filter but could not be \ + truncated)", + set.tasks.len(), + set.generated_from_commits, + set.rejected.len() + ); + + let scratch = work.map(Path::to_path_buf).unwrap_or_else(|| { + std::env::temp_dir().join(format!("reify-verify-eval-{}", std::process::id())) + }); + let tree = scratch.join("tree"); + + let mut outcomes: Vec = Vec::new(); + // Taken from the first trial's index and kept: what the repository is written in + // is a property of the repository, not of the label someone passes to the report. + let mut languages: Vec<(String, usize)> = Vec::new(); + for (i, task) in set.tasks.iter().enumerate() { + eprint!("\r trial {}/{} ", i + 1, set.tasks.len()); + let indexing = std::time::Instant::now(); + extract_tree(repo, &task.parent, &tree)?; + let mut store = Store::open( + tree.join(reify::index::REIFY_DIR) + .join(reify::index::STORE_FILE), + )?; + reify::index::index(&mut store, &reify::index::IndexOptions::new(&tree))?; + let index_ms = indexing.elapsed().as_millis(); + if languages.is_empty() { + let mut rows = store.coverage_by_language()?; + rows.sort_by_key(|row| std::cmp::Reverse(row.1)); + languages = rows.into_iter().take(3).map(|(l, n, _)| (l, n)).collect(); + } + + // Resolved at the parent, where the withheld change has not happened yet — the + // same state the checker sees, so a symbol that does not exist there is + // honestly unscorable rather than quietly credited. + let omission_symbol = store + .symbol_at(&task.omission_file, task.omission_line)? + .map(|node| node.location()); + let truncated = conditions::missing_callers(&store, &task.truncated)?; + let complete = conditions::missing_callers(&store, &task.complete)?; + outcomes.push(metrics::score_verify( + task, + omission_symbol, + conditions::can_be_cited(&store, &task.omission_file)?, + &truncated, + &complete, + index_ms, + )); + } + eprintln!(); + let _ = std::fs::remove_dir_all(&scratch); + + let summary = metrics::summarise_verify(&outcomes); + std::fs::create_dir_all(out)?; + write_json(&out.join("verify-outcomes.json"), &outcomes)?; + write_json(&out.join("verify-summary.json"), &summary)?; + write_json(&out.join("verify-tasks.json"), &set)?; + write_json( + &out.join("verify-environment.json"), + &serde_json::json!({ + "reify_version": env!("CARGO_PKG_VERSION"), + "repository": set.repository, + // The local path is wherever the run happened, which is no help to anyone + // reproducing it. The remote is. + "origin": origin(repo), + "head": set.head, + "languages": languages, + // The selection window, so a committed result can be re-run exactly even + // after the branch it was taken from has moved on. + "count": count, + "scan": scan, + "after": after, + "until": until, + "trials": set.tasks.len(), + "candidates_rejected": set.rejected.len(), + "checker": "symbols changed by this diff, minus symbols present in the diff, \ + where an inbound CALLS edge exists at distance 1, via reify::query::impact", + "wall_clock_ms": started.elapsed().as_millis(), + "token_counts": "estimated by reify heuristic-v1", + }), + )?; + + eprintln!("\n{}", render_verify(&summary)); + eprintln!( + "wrote {} ({:.1}s wall clock)", + out.display(), + started.elapsed().as_secs_f32() + ); + Ok(()) +} + +/// The summary as a human reads it, verdict first. +fn render_verify(s: &metrics::VerifySummary) -> String { + let mut text = String::new(); + text.push_str(&format!( + "{:<28} {:.2} (95% CI {:.2}–{:.2}, {}/{} trials)\n", + "omission_recall", + s.omission_recall, + s.omission_recall_ci.0, + s.omission_recall_ci.1, + (s.omission_recall * s.tasks as f32).round() as usize, + s.tasks, + )); + text.push_str(&format!( + "{:<28} {:.2} (95% CI {:.2}–{:.2}) — citations the complete commit does not \ + also produce\n", + " of which attributable", + s.omission_recall_attributable, + s.omission_recall_attributable_ci.0, + s.omission_recall_attributable_ci.1, + )); + match (s.omission_recall_reachable, s.omission_recall_reachable_ci) { + (Some(recall), Some(ci)) => text.push_str(&format!( + "{:<28} {recall:.2} (95% CI {:.2}–{:.2}, {}/{} omitted files call into \ + another file at all — the ceiling on any call-graph checker)\n", + " where citable at all", ci.0, ci.1, s.reachable_omissions, s.tasks + )), + _ => text.push_str(&format!( + "{:<28} — (no omitted file calls into another file; the ceiling on \ + any call-graph checker here is zero)\n", + " where citable at all" + )), + } + match (s.omission_recall_symbol, s.omission_recall_symbol_ci) { + (Some(recall), Some(ci)) => text.push_str(&format!( + "{:<28} {recall:.2} (95% CI {:.2}–{:.2}, {} scorable)\n", + "omission_recall_symbol", ci.0, ci.1, s.symbol_scorable + )), + _ => text.push_str(&format!( + "{:<28} — (no trial's omission fell inside an indexed symbol)\n", + "omission_recall_symbol" + )), + } + text.push_str(&format!( + "{:<28} {:.2} per complete commit ({}/{} commits noisy, 95% CI {:.2}–{:.2})\n", + "false_alarm_rate", + s.false_alarm_rate, + s.commits_with_a_false_alarm, + s.tasks, + s.false_alarm_share_ci.0, + s.false_alarm_share_ci.1, + )); + text.push_str(&format!( + "{:<28} {}\n", + "findings_per_diff (median)", s.median_findings_per_diff + )); + text.push_str(&format!( + "{:<28} {}\n", + "verify_tokens (median)", s.median_verify_tokens + )); + text.push_str(&format!( + "{:<28} {}\n", + "verify_latency_ms (median)", s.median_verify_latency_ms + )); + text.push_str(&format!( + "{:<28} {} (extract + index one parent tree; not part of the query)\n", + "index_ms (median)", s.median_index_ms + )); + if s.diffs_resolving_to_nothing > 0 { + text.push_str(&format!( + "{:<28} {} of {} truncated diffs resolved to no indexed symbol at all\n", + "unresolved", s.diffs_resolving_to_nothing, s.tasks + )); + } + text.push_str(&format!( + "\npre-registered verdict: {}\n {}\n", + match s.verdict() { + metrics::Verdict::Build => "BUILD `reify verify` on this substrate", + metrics::Verdict::DoNotBuild => "DO NOT BUILD `reify verify` on this substrate", + }, + s.why() + )); + text +} + +/// The repository's `origin` remote, so a committed result names something a reader +/// can clone rather than the temporary directory it happened to be run in. +fn origin(repo: &Path) -> Option { + let output = std::process::Command::new("git") + .args(["remote", "get-url", "origin"]) + .current_dir(repo) + .output() + .ok()?; + output + .status + .success() + .then(|| String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +/// Extract a commit's tree into `into`, replacing whatever was there. +/// +/// `git archive` rather than a worktree: a worktree registers itself in the shared +/// git directory, and this harness must not leave anything behind in the repository +/// it is measuring. The cost is a full index per trial, which is reported. +fn extract_tree(repo: &Path, sha: &str, into: &Path) -> Result<()> { + use std::process::{Command, Stdio}; + if into.exists() { + std::fs::remove_dir_all(into).with_context(|| format!("clearing {}", into.display()))?; + } + std::fs::create_dir_all(into)?; + let mut archive = Command::new("git") + .args(["archive", "--format=tar", sha]) + .current_dir(repo) + .stdout(Stdio::piped()) + .spawn() + .context("running git archive")?; + let stdout = archive + .stdout + .take() + .ok_or_else(|| anyhow::anyhow!("git archive produced no output"))?; + let extracted = Command::new("tar") + .arg("-x") + .arg("-C") + .arg(into) + .stdin(Stdio::from(stdout)) + .status() + .context("running tar to extract a parent tree")?; + let archived = archive.wait()?; + anyhow::ensure!( + archived.success() && extracted.success(), + "cannot extract the tree at {sha}" + ); + Ok(()) +} + +/// Render the held-out-hunk report across every repository that was run. +/// +/// Generated from `verify-summary.json` for the same reason the retrieval report is: +/// a table that can drift from its data is a picture, not a measurement. The per-trial +/// appendix is included so the selection rule's effects are visible rather than +/// described — every omitted file is named. +fn verify_report(results: &[String], out: &Path) -> Result<()> { + struct Run { + label: String, + summary: metrics::VerifySummary, + environment: serde_json::Value, + outcomes: Vec, + } + + let mut runs = Vec::new(); + for spec in results { + let (label, dir) = spec + .split_once('=') + .ok_or_else(|| anyhow::anyhow!("expected LABEL=DIR, got `{spec}`"))?; + let dir = Path::new(dir); + runs.push(Run { + label: label.to_string(), + summary: read_json(&dir.join("verify-summary.json"))?, + environment: read_json(&dir.join("verify-environment.json"))?, + outcomes: read_json(&dir.join("verify-outcomes.json"))?, + }); + } + anyhow::ensure!(!runs.is_empty(), "no results given"); + + let mut md = String::from("# Can the graph tell that a patch is incomplete?\n\n"); + md.push_str( + "Generated by `reify-bench verify-report`. Every number is computed from the \ + `verify-summary.json` files named below; nothing is entered by hand.\n\n\ + This benchmark exists to decide one thing: whether `reify verify` — a \ + post-flight check that reads an agent's diff and reports what the patch \ + missed — is worth building on Reify's call graph. It is model-free, \ + deterministic, and costs nothing per run.\n\n", + ); + + md.push_str("## Construction\n\n"); + md.push_str( + "For each merged commit that passes the retrieval benchmark's filters and \ + touches at least two indexable files:\n\n\ + 1. the parent tree is extracted and indexed, so the change is absent from the \ + index by construction;\n\ + 2. one file's **only** hunk is withheld — the *omission*. Removing it removes \ + that file from the patch entirely, so a citation of it cannot be an echo of \ + a hunk still present. Among the files with exactly one hunk, the last by \ + path order is chosen; the choice is arbitrary, fixed, and made before any \ + checker runs;\n\ + 3. the truncated patch goes to the checker;\n\ + 4. **the same commit goes to the checker complete.** A merged commit is \ + complete by definition, so every finding there is a false positive. This \ + control is not optional: without it the metric would reward a checker that \ + simply shouts.\n\n\ + The checker is not `reify verify`, which does not exist. It is the shipped \ + graph query — *symbols changed by this diff, minus symbols present in the \ + diff, where an inbound `CALLS` edge exists at distance 1* — reached through \ + `reify::query::impact`. That deliberately measures the **substrate**, which is \ + the number the decision needs.\n\n", + ); + + md.push_str("## Pre-registered falsification condition\n\n"); + md.push_str(&format!( + "> If `omission_recall` on this substrate is below **{VERIFY_RECALL_FLOOR:.2}**, \ + or `false_alarm_rate` is above **{VERIFY_FALSE_ALARM_CEILING:.1} per commit**, \ + the `reify verify` feature does not get built on this substrate.\n\n\ + Stated in `crates/reify-bench/src/metrics.rs` before the first run and not \ + moved since. A result that kills the feature is a result.\n\n", + VERIFY_RECALL_FLOOR = metrics::VERIFY_RECALL_FLOOR, + VERIFY_FALSE_ALARM_CEILING = metrics::VERIFY_FALSE_ALARM_CEILING, + )); + + md.push_str("## Results\n\n| Metric |"); + for run in &runs { + md.push_str(&format!(" {} |", run.label)); + } + md.push_str("\n|---|"); + for _ in &runs { + md.push_str("---:|"); + } + md.push('\n'); + let row = |label: &str, f: &dyn Fn(&Run) -> String| { + let mut line = format!("| {label} |"); + for run in &runs { + line.push_str(&format!(" {} |", f(run))); + } + line.push('\n'); + line + }; + md.push_str(&row("Most indexed language", &|r| { + r.environment["languages"][0][0] + .as_str() + .unwrap_or("—") + .to_string() + })); + md.push_str(&row("Trials", &|r| r.summary.tasks.to_string())); + md.push_str(&row("`omission_recall`", &|r| { + format!( + "**{:.2}** ({:.2}–{:.2})", + r.summary.omission_recall, + r.summary.omission_recall_ci.0, + r.summary.omission_recall_ci.1 + ) + })); + md.push_str(&row("…attributable to the omission", &|r| { + format!( + "{:.2} ({:.2}–{:.2})", + r.summary.omission_recall_attributable, + r.summary.omission_recall_attributable_ci.0, + r.summary.omission_recall_attributable_ci.1 + ) + })); + md.push_str(&row("`omission_recall_symbol`", &|r| match ( + r.summary.omission_recall_symbol, + r.summary.omission_recall_symbol_ci, + ) { + (Some(v), Some(ci)) => format!( + "{v:.2} ({:.2}–{:.2}) over {}", + ci.0, ci.1, r.summary.symbol_scorable + ), + _ => "— (0 scorable)".into(), + })); + md.push_str(&row("Omitted files a caller query *could* cite", &|r| { + format!("{}/{}", r.summary.reachable_omissions, r.summary.tasks) + })); + md.push_str(&row("`false_alarm_rate` (per complete commit)", &|r| { + format!("**{:.1}**", r.summary.false_alarm_rate) + })); + md.push_str(&row("Complete commits with ≥1 false alarm", &|r| { + format!( + "{}/{} ({:.2}–{:.2})", + r.summary.commits_with_a_false_alarm, + r.summary.tasks, + r.summary.false_alarm_share_ci.0, + r.summary.false_alarm_share_ci.1 + ) + })); + md.push_str(&row("`findings_per_diff` (median)", &|r| { + r.summary.median_findings_per_diff.to_string() + })); + md.push_str(&row("`verify_tokens` (median)", &|r| { + r.summary.median_verify_tokens.to_string() + })); + md.push_str(&row("`verify_latency_ms` (median)", &|r| { + r.summary.median_verify_latency_ms.to_string() + })); + md.push_str(&row("Index per trial, ms (median)", &|r| { + r.summary.median_index_ms.to_string() + })); + md.push_str(&row("Whole run, wall clock", &|r| { + format!( + "{:.0}s", + r.environment["wall_clock_ms"].as_f64().unwrap_or(0.0) / 1000.0 + ) + })); + md.push_str(&row( + "Pre-registered verdict", + &|r| match r.summary.verdict() { + metrics::Verdict::Build => "build".into(), + metrics::Verdict::DoNotBuild => "**do not build**".into(), + }, + )); + + md.push_str("\n## What the numbers say\n\n"); + for run in &runs { + md.push_str(&format!("**{}** — {}\n\n", run.label, run.summary.why())); + } + let all_fail = runs + .iter() + .all(|r| r.summary.verdict() == metrics::Verdict::DoNotBuild); + md.push_str(if all_fail { + "Every repository fails the pre-registered condition, so **`reify verify` does \ + not get built on this substrate**. The condition was written down before the \ + first run precisely so this outcome could not be argued away afterwards.\n\n" + } else { + "At least one repository clears the pre-registered condition. Read the \ + confidence intervals before treating that as settled.\n\n" + }); + + // Which half of the condition actually fails, counted rather than asserted: the + // interesting question is not "did it fail" but "on what". + let failed_recall = runs + .iter() + .filter(|r| r.summary.omission_recall < metrics::VERIFY_RECALL_FLOOR) + .count(); + let failed_noise = runs + .iter() + .filter(|r| r.summary.false_alarm_rate > metrics::VERIFY_FALSE_ALARM_CEILING) + .count(); + md.push_str(&format!( + "**It fails on noise, not on blindness.** {failed_noise} of {} repositories \ + exceed the false-alarm ceiling; {failed_recall} of {} fall below the recall \ + floor (a repository can fail both). The graph does find the omitted file often enough to be interesting; \ + what it cannot do is stay quiet about a patch that is already complete.\n\n", + runs.len(), + runs.len(), + )); + + md.push_str( + "**The negative control takes most of the headline back.** `omission_recall` \ + counts a citation of the omitted file whether or not the complete commit is \ + cited too. The attributable row counts only citations the complete commit does \ + *not* produce, and it is the smaller number in every repository here. The gap \ + is the checker citing a file it would have cited anyway — which is not \ + detection, however it reads next to the label.\n\n", + ); + + // The ceiling either binds or it does not, and which one decides whether a better + // query could help. Asserting the wrong one would be worse than saying nothing. + let tightest = runs + .iter() + .map(|r| r.summary.reachable_omissions as f32 / r.summary.tasks.max(1) as f32) + .fold(f32::INFINITY, f32::min); + md.push_str(&format!( + "**The ceiling is not what binds.** A finding is a caller, so the omitted file \ + can only be cited if something in it calls out of itself. In the least \ + favourable repository here that holds for {:.0}% of omissions, so the edges \ + mostly exist and `omission_recall` is not capped by their absence. The gap \ + between that row and the recall row is a *ranking* gap, not a coverage one.\n\n", + tightest * 100.0 + )); + + md.push_str( + "**The noise is structural, not marginal.** `false_alarm_rate` is findings per \ + commit that is complete by construction. A `CALLS` edge says a caller exists; \ + it does not say the caller needed changing. Nothing in the graph distinguishes \ + a changed signature from an edit inside a body, so every caller of every \ + touched symbol is a candidate. That is a property of the edge, and no \ + rewriting of the query around the same edge removes it.\n\n", + ); + + md.push_str("## Cost and determinism\n\n"); + md.push_str(&format!( + "No model, no network, no provider key: the whole run is a git extract, an \ + index and a graph query. Total wall clock for everything in this report is \ + **{:.0}s**, dominated by re-indexing one parent tree per trial. The query \ + itself is the `verify_latency_ms` row — single-digit milliseconds.\n\n\ + Each run is deterministic given a fixed `HEAD`: task selection, the omission \ + rule and the query contain no randomness and no tunable threshold. A run \ + against a repository whose history is still moving — this one, for instance — \ + should pin the window with `--until `, or the trial set moves with the \ + branch.\n\n\ + ```bash\n\ + reify-bench verify-eval --repo --out results/verify- --until \n\ + reify-bench verify-report --results \"name=results/verify-\" --out benchmarks/REPORT-verify.md\n\ + ```\n\n", + runs + .iter() + .map(|r| r.environment["wall_clock_ms"].as_f64().unwrap_or(0.0)) + .sum::() + / 1000.0 + )); + + md.push_str("## Limitations\n\n"); + md.push_str( + "1. **Small samples.** The intervals are wide and are printed beside every \ + rate. Where two repositories differ by less than their intervals, they have \ + not been shown to differ.\n\ + 2. **The omission-selection rule has a direction.** \"Last by path order, among \ + files with exactly one hunk\" is arbitrary but not neutral: in a repository \ + laid out as `src/` and `tests/`, path order lands on `tests/`. Counted \ + across every run here, TEST_SHARE omissions sit under a path segment \ + named `test` or `tests`. The rule was fixed before any run and has not been \ + changed since; every omitted file is named in the appendix, so the effect is \ + checkable rather than described.\n\ + 3. **`CALLS` at distance 1 only.** `impact` also propagates two hops and crosses \ + into the data layer. Widening the query would raise recall and raise the \ + false-alarm rate with it — the trade this benchmark measures rather than \ + pre-empts.\n\ + 4. **A checker, not the feature.** `reify verify` could use a signature diff, \ + type information, or the model. This measures the substrate those would all \ + stand on.\n\ + 5. **Parent trees are extracted with `git archive`**, so the indexed tree has no \ + git history and no co-change edges. The checker uses neither; a checker that \ + did would need re-measuring.\n\ + 6. **Ground truth is one commit's hunks.** A change that could correctly have \ + been made elsewhere scores as a miss.\n\ + 7. **`impact`'s own bounds are inherited, not bypassed.** It stops at 60 \ + affected nodes and walks depth-first to two hops, so on a widely-called \ + symbol some direct callers can be crowded out by second-hop ones. That is \ + the shipped query's behaviour and measuring around it would measure \ + something that does not exist.\n\n", + ); + + let (mut in_tests, mut trials) = (0usize, 0usize); + for run in &runs { + for outcome in &run.outcomes { + trials += 1; + if outcome + .omission_file + .split('/') + .any(|part| part == "test" || part == "tests") + { + in_tests += 1; + } + } + } + let md = md.replace("TEST_SHARE", &format!("{in_tests} of {trials}")); + + let mut md = md; + md.push_str("## Appendix: every trial\n\n"); + md.push_str( + "`could cite` is whether the omitted file calls out of itself at all — the \ + ceiling for that trial. `cited` is findings on the truncated patch, `noise` is \ + findings on the same commit complete.\n\n", + ); + for run in &runs { + md.push_str(&format!( + "### {} (`{}`, commit `{}`)\n\n", + run.label, + run.environment["origin"] + .as_str() + .or_else(|| run.environment["repository"].as_str()) + .unwrap_or("?"), + run.environment["head"].as_str().unwrap_or("?"), + )); + md.push_str("| Trial | Omitted file | could cite | hit | attributable | cited | noise |\n"); + md.push_str("|---|---|---|---|---|---:|---:|\n"); + let tick = |yes: bool| if yes { "yes" } else { "no" }; + for outcome in &run.outcomes { + md.push_str(&format!( + "| `{}` | `{}` | {} | {} | {} | {} | {} |\n", + outcome.task, + outcome.omission_file, + tick(outcome.omission_file_reachable), + tick(outcome.file_hit), + tick(outcome.file_hit_attributable), + outcome.findings, + outcome.false_alarms, + )); + } + md.push('\n'); + } + + std::fs::write(out, md).with_context(|| format!("writing {}", out.display()))?; + eprintln!("wrote {}", out.display()); + Ok(()) +} + fn report(input: &Path, out: &Path) -> Result<()> { let outcomes: Vec = read_json(&input.join("outcomes.json"))?; let set: tasks::TaskSet = read_json(&input.join("tasks.json"))?; diff --git a/crates/reify-bench/src/metrics.rs b/crates/reify-bench/src/metrics.rs index b97db0e..885d584 100644 --- a/crates/reify-bench/src/metrics.rs +++ b/crates/reify-bench/src/metrics.rs @@ -6,7 +6,8 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; -use crate::conditions::Answer; +use crate::agent::wilson_interval; +use crate::conditions::{Answer, Finding, Findings}; /// How one condition performed on one task. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -200,6 +201,260 @@ pub fn expected_tokens(condition: &str, outcomes: &[Outcome], budget: u32) -> f3 total / mine.len() as f32 } +// ---- the held-out-hunk metrics ---------------------------------------------- + +/// Pre-registered falsification condition for `reify verify`, stated before the first +/// run of this harness and not moved since. +/// +/// > If `omission_recall` on this substrate is below **0.25**, or `false_alarm_rate` +/// > is above **0.1 per commit**, the `reify verify` feature does not get built on +/// > this substrate. +/// +/// Both halves matter. Recall alone would be cleared by a checker that reports every +/// caller of everything; the false-alarm ceiling is what stops that, and it is +/// measured against complete merged commits, where a finding cannot be anything but +/// wrong. A result that fails either half is a result, not a failure of the harness: +/// the response is to publish it and not build the feature, never to widen the query +/// until it passes. +pub const VERIFY_RECALL_FLOOR: f32 = 0.25; +/// Findings per complete merged commit, above which the checker is too noisy to ship. +pub const VERIFY_FALSE_ALARM_CEILING: f32 = 0.1; + +/// One held-out-hunk trial. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VerifyOutcome { + pub task: String, + pub commit: String, + /// The file whose only hunk was withheld. + pub omission_file: String, + /// `path:line` of the symbol the withheld hunk falls inside, when it falls inside + /// one. `None` means the change was outside every indexed symbol — an import, a + /// constant, a top-level statement — and the trial is not scorable at symbol + /// granularity. It is excluded there rather than counted as a miss. + pub omission_symbol: Option, + /// Some symbol in the omission's file calls a symbol in another file, at the + /// parent commit. False means no caller-based checker could ever cite this file, + /// whatever query it runs. + pub omission_file_reachable: bool, + /// A finding cites the omission's file. + pub file_hit: bool, + /// A finding cites the omission's file on the truncated diff and **not** on the + /// complete one. A citation the negative control also produces was not caused by + /// the omission, whatever it looks like next to it. + pub file_hit_attributable: bool, + /// A finding cites the omission's symbol. `None` when not scorable. + pub symbol_hit: Option, + pub findings: usize, + /// Findings against the **complete** commit. Complete by construction, so every + /// one of these is a false positive. + pub false_alarms: usize, + /// Symbols the truncated diff resolved to. Zero means the checker had nothing to + /// work from, which is a different failure from having something and missing. + pub changed_symbols: usize, + pub verify_tokens: u32, + pub verify_latency_ms: u128, + /// Wall clock to extract and index the parent tree. The real feature would run + /// against an index that already exists, so this is the harness's cost, not the + /// checker's, and is kept out of `verify_latency_ms`. + pub index_ms: u128, + /// Every location the truncated run cited, and every location the complete run + /// cited. Written out because a rate nobody can check is a claim, not a + /// measurement: these are what `false_alarms` counts, one line each. + pub cited: Vec, + pub cited_on_complete: Vec, +} + +/// Aggregate figures over a set of held-out-hunk trials. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VerifySummary { + pub tasks: usize, + /// Share of truncated diffs where some finding cites the omitted hunk's file. + pub omission_recall: f32, + pub omission_recall_ci: (f32, f32), + /// The same share, counting only citations the complete commit does **not** also + /// produce. `omission_recall` is the metric as specified; this is the one that + /// says whether the checker responded to the omission or to the file's standing + /// noise. Where the two differ, the gap is the part of the headline that the + /// negative control already explains. + pub omission_recall_attributable: f32, + pub omission_recall_attributable_ci: (f32, f32), + /// Trials whose omitted file calls out of itself at all. This is the ceiling: + /// `omission_recall` cannot exceed `reachable_omissions / tasks` however the query + /// is written, so the two numbers separate a query problem from a substrate + /// problem. + pub reachable_omissions: usize, + /// `omission_recall` restricted to those trials — what the checker managed where + /// there was something to find. + pub omission_recall_reachable: Option, + pub omission_recall_reachable_ci: Option<(f32, f32)>, + /// Trials where the omission falls inside an indexed symbol — the denominator of + /// the symbol-granular figure. + pub symbol_scorable: usize, + /// The same share at symbol granularity, over `symbol_scorable` trials. + pub omission_recall_symbol: Option, + pub omission_recall_symbol_ci: Option<(f32, f32)>, + /// Findings per complete merged commit. A rate over counts, not a proportion, so + /// it carries no Wilson interval; the proportion beside it does. + pub false_alarm_rate: f32, + /// Complete commits producing at least one finding. + pub commits_with_a_false_alarm: usize, + pub false_alarm_share_ci: (f32, f32), + /// Median findings per truncated diff. A checker emitting thirty findings is + /// unusable at any precision, which a mean would hide behind the quiet cases. + pub median_findings_per_diff: usize, + /// Median tokens the findings output would cost the agent that reads it. + pub median_verify_tokens: u32, + /// Median wall clock of the query alone. + pub median_verify_latency_ms: u128, + /// Median wall clock to extract and index one parent tree, reported so the cost of + /// running this in CI is a measured number rather than a promise. + pub median_index_ms: u128, + /// Trials where the truncated diff resolved to no symbol at all. The checker + /// cannot report anything for these; they stay in the denominator, because a + /// substrate that cannot resolve a diff has failed the task. + pub diffs_resolving_to_nothing: usize, +} + +/// Does this substrate clear the pre-registered bar? +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Verdict { + Build, + DoNotBuild, +} + +impl VerifySummary { + pub fn verdict(&self) -> Verdict { + if self.omission_recall < VERIFY_RECALL_FLOOR + || self.false_alarm_rate > VERIFY_FALSE_ALARM_CEILING + { + Verdict::DoNotBuild + } else { + Verdict::Build + } + } + + /// Why the verdict came out the way it did, in one line. + pub fn why(&self) -> String { + let mut failed = Vec::new(); + if self.omission_recall < VERIFY_RECALL_FLOOR { + failed.push(format!( + "omission_recall {:.2} < {VERIFY_RECALL_FLOOR:.2}", + self.omission_recall + )); + } + if self.false_alarm_rate > VERIFY_FALSE_ALARM_CEILING { + failed.push(format!( + "false_alarm_rate {:.2} > {VERIFY_FALSE_ALARM_CEILING:.2}", + self.false_alarm_rate + )); + } + if failed.is_empty() { + format!( + "omission_recall {:.2} >= {VERIFY_RECALL_FLOOR:.2} and false_alarm_rate \ + {:.2} <= {VERIFY_FALSE_ALARM_CEILING:.2}", + self.omission_recall, self.false_alarm_rate + ) + } else { + failed.join("; ") + } + } +} + +/// Score one held-out-hunk trial. +pub fn score_verify( + task: &crate::tasks::TruncatedTask, + omission_symbol: Option, + omission_file_reachable: bool, + truncated: &Findings, + complete: &Findings, + index_ms: u128, +) -> VerifyOutcome { + let cites = |predicate: &dyn Fn(&Finding) -> bool| truncated.findings.iter().any(predicate); + VerifyOutcome { + task: task.id.clone(), + commit: task.commit.clone(), + omission_file: task.omission_file.clone(), + omission_file_reachable, + file_hit: cites(&|f| f.path == task.omission_file), + file_hit_attributable: cites(&|f| f.path == task.omission_file) + && !complete + .findings + .iter() + .any(|f| f.path == task.omission_file), + symbol_hit: omission_symbol + .as_ref() + .map(|symbol| cites(&|f| &f.location == symbol)), + omission_symbol, + findings: truncated.findings.len(), + false_alarms: complete.findings.len(), + changed_symbols: truncated.changed_symbols.len(), + verify_tokens: truncated.answer_tokens, + verify_latency_ms: truncated.elapsed_ms, + index_ms, + cited: truncated + .findings + .iter() + .map(|f| f.location.clone()) + .collect(), + cited_on_complete: complete + .findings + .iter() + .map(|f| f.location.clone()) + .collect(), + } +} + +pub fn summarise_verify(outcomes: &[VerifyOutcome]) -> VerifySummary { + let n = outcomes.len(); + let denominator = n.max(1) as f32; + let file_hits = outcomes.iter().filter(|o| o.file_hit).count(); + let attributable = outcomes.iter().filter(|o| o.file_hit_attributable).count(); + let reachable: Vec<&VerifyOutcome> = outcomes + .iter() + .filter(|o| o.omission_file_reachable) + .collect(); + let reachable_hits = reachable.iter().filter(|o| o.file_hit).count(); + let scorable: Vec<&VerifyOutcome> = outcomes + .iter() + .filter(|o| o.omission_symbol.is_some()) + .collect(); + let symbol_hits = scorable + .iter() + .filter(|o| o.symbol_hit == Some(true)) + .count(); + let noisy = outcomes.iter().filter(|o| o.false_alarms > 0).count(); + + VerifySummary { + tasks: n, + omission_recall: file_hits as f32 / denominator, + omission_recall_ci: wilson_interval(file_hits, n), + omission_recall_attributable: attributable as f32 / denominator, + omission_recall_attributable_ci: wilson_interval(attributable, n), + reachable_omissions: reachable.len(), + omission_recall_reachable: (!reachable.is_empty()) + .then(|| reachable_hits as f32 / reachable.len() as f32), + omission_recall_reachable_ci: (!reachable.is_empty()) + .then(|| wilson_interval(reachable_hits, reachable.len())), + symbol_scorable: scorable.len(), + omission_recall_symbol: (!scorable.is_empty()) + .then(|| symbol_hits as f32 / scorable.len() as f32), + omission_recall_symbol_ci: (!scorable.is_empty()) + .then(|| wilson_interval(symbol_hits, scorable.len())), + false_alarm_rate: outcomes.iter().map(|o| o.false_alarms).sum::() as f32 + / denominator, + commits_with_a_false_alarm: noisy, + false_alarm_share_ci: wilson_interval(noisy, n), + median_findings_per_diff: median(outcomes.iter().map(|o| o.findings).collect()) + .unwrap_or(0), + median_verify_tokens: median(outcomes.iter().map(|o| o.verify_tokens).collect()) + .unwrap_or(0), + median_verify_latency_ms: median(outcomes.iter().map(|o| o.verify_latency_ms).collect()) + .unwrap_or(0), + median_index_ms: median(outcomes.iter().map(|o| o.index_ms).collect()).unwrap_or(0), + diffs_resolving_to_nothing: outcomes.iter().filter(|o| o.changed_symbols == 0).count(), + } +} + fn median(mut values: Vec) -> Option { if values.is_empty() { return None; @@ -221,6 +476,141 @@ mod tests { } } + fn trial(omission_file: &str) -> crate::tasks::TruncatedTask { + crate::tasks::TruncatedTask { + id: "v-1".into(), + commit: "a".repeat(40), + parent: "b".repeat(40), + date: "2026-01-01".into(), + prompt: "fix the credit limit check".into(), + complete: crate::tasks::Patch::default(), + truncated: crate::tasks::Patch::default(), + omission_file: omission_file.into(), + omission_line: 10, + } + } + + fn findings(locations: &[&str]) -> Findings { + Findings { + findings: locations + .iter() + .map(|location| Finding { + location: (*location).into(), + path: location + .rsplit_once(':') + .map_or(*location, |(p, _)| p) + .into(), + what: "f".into(), + reason: "calls g".into(), + }) + .collect(), + changed_symbols: vec!["app/x.py:1".into()], + answer_tokens: 40, + elapsed_ms: 1, + } + } + + #[test] + fn a_citation_the_complete_commit_also_produces_is_a_hit_but_not_attributable() { + // The negative control's whole purpose: the checker cited that file whether or + // not anything was withheld, so the omission did not cause the citation. + let outcome = score_verify( + &trial("app/orders.py"), + None, + true, + &findings(&["app/orders.py:5"]), + &findings(&["app/orders.py:5"]), + 100, + ); + assert!(outcome.file_hit); + assert!(!outcome.file_hit_attributable); + assert_eq!(outcome.false_alarms, 1); + } + + #[test] + fn a_citation_only_the_truncated_diff_produces_is_attributable() { + let outcome = score_verify( + &trial("app/orders.py"), + Some("app/orders.py:5".into()), + true, + &findings(&["app/orders.py:5"]), + &findings(&[]), + 100, + ); + assert!(outcome.file_hit_attributable); + assert_eq!(outcome.symbol_hit, Some(true)); + assert_eq!(outcome.false_alarms, 0); + } + + #[test] + fn an_omission_inside_no_symbol_is_unscorable_rather_than_a_miss() { + let outcome = score_verify( + &trial("app/orders.py"), + None, + false, + &findings(&[]), + &findings(&[]), + 100, + ); + assert_eq!( + outcome.symbol_hit, None, + "counting it as a miss would be a lie" + ); + let summary = summarise_verify(&[outcome]); + assert_eq!(summary.symbol_scorable, 0); + assert_eq!(summary.omission_recall_symbol, None); + assert_eq!(summary.reachable_omissions, 0); + assert_eq!(summary.omission_recall_reachable, None); + } + + #[test] + fn the_pre_registered_condition_fails_on_either_half_alone() { + let quiet_but_blind = VerifySummary { + omission_recall: 0.10, + false_alarm_rate: 0.0, + ..summarise_verify(&[]) + }; + assert_eq!(quiet_but_blind.verdict(), Verdict::DoNotBuild); + assert!(quiet_but_blind.why().contains("omission_recall")); + + let sharp_but_noisy = VerifySummary { + omission_recall: 0.90, + false_alarm_rate: 3.0, + ..summarise_verify(&[]) + }; + assert_eq!(sharp_but_noisy.verdict(), Verdict::DoNotBuild); + assert!(sharp_but_noisy.why().contains("false_alarm_rate")); + + let good = VerifySummary { + omission_recall: 0.40, + false_alarm_rate: 0.05, + ..summarise_verify(&[]) + }; + assert_eq!(good.verdict(), Verdict::Build); + } + + #[test] + fn the_false_alarm_rate_counts_findings_not_commits() { + // A checker that shouts thirty times at one commit and stays silent at nine + // others is not a checker with a 10% false-alarm problem. + let outcomes: Vec = (0..10) + .map(|i| { + score_verify( + &trial("app/orders.py"), + None, + true, + &findings(&[]), + &findings(&if i == 0 { vec!["a.py:1"; 30] } else { vec![] }), + 1, + ) + }) + .collect(); + let summary = summarise_verify(&outcomes); + assert_eq!(summary.commits_with_a_false_alarm, 1); + assert!((summary.false_alarm_rate - 3.0).abs() < 1e-6); + assert_eq!(summary.verdict(), Verdict::DoNotBuild); + } + #[test] fn a_perfect_answer_scores_perfectly() { let truth = vec!["a.py".to_string()]; diff --git a/crates/reify-bench/src/tasks.rs b/crates/reify-bench/src/tasks.rs index f27648f..19d5caa 100644 --- a/crates/reify-bench/src/tasks.rs +++ b/crates/reify-bench/src/tasks.rs @@ -45,6 +45,22 @@ pub struct TaskSet { pub tasks: Vec, } +impl TaskSet { + /// The repository's name, for a prompt that has to say what is being worked on. + /// + /// `repository` is a path — `.bench/medusa` — because that is what the generator + /// was pointed at. The final component is the name a developer would use, and a + /// prompt that names the wrong repository is a validity defect rather than a + /// cosmetic one: see the dated note at the top of `benchmarks/REPORT-medusa.md`. + pub fn repository_name(&self) -> &str { + let trimmed = self.repository.trim_end_matches(['/', '\\']); + trimmed + .rsplit(['/', '\\']) + .find(|part| !part.is_empty()) + .unwrap_or(trimmed) + } +} + /// Upper bound on files a task may touch. /// /// A commit touching twenty files is a refactor or a rename; its "ground truth" would @@ -93,67 +109,21 @@ pub fn generate( // walk *starts below it*: only strictly older commits qualify, which is how a // training corpus is kept disjoint from every evaluation window. let base = after.map(|rev| resolve(root, rev)).transpose()?; - let cutoff = base.as_ref().and_then(|sha| { - history - .commits - .iter() - .position(|c| c.sha.starts_with(sha) || sha.starts_with(&c.sha)) - }); - let start = match until.map(|rev| resolve(root, rev)).transpose()? { - None => 0, - // The scanned list is merge-free, so a merge commit named as the boundary is - // legitimately absent from it. When the boundary is HEAD itself, "strictly - // older than HEAD" excludes nothing the list contains. - Some(sha) if sha == head => 0, - Some(sha) => { - match history - .commits - .iter() - .position(|c| c.sha.starts_with(&sha) || sha.starts_with(&c.sha)) - { - Some(position) => position + 1, - // A merge commit is legitimately absent from the merge-free list, so - // the boundary falls back to its timestamp: strictly-older-than holds - // for every commit authored before it. - None => { - let at = commit_time(root, &sha)?; - history - .commits - .iter() - .position(|c| c.timestamp < at) - .with_context(|| { - format!("--until {sha}: nothing older within the scanned history") - })? - } - } - } - }; + let (start, cutoff) = window(root, &history, &head, after, until)?; - let mut tasks = Vec::new(); - for (i, commit) in history.commits.iter().enumerate().skip(start) { - if tasks.len() >= wanted { - break; - } - if cutoff.is_some_and(|stop| i >= stop) { - break; - } - if exclude.contains(&commit.sha) { - continue; - } - let Some(mut task) = candidate(root, commit) else { - continue; - }; + let tasks = take(&history, start, cutoff, exclude, wanted, |commit| { + let mut task = candidate(root, commit)?; // A file created by the change cannot be retrieved from a base that predates // it. Keeping it as ground truth would score every condition zero and measure // nothing, so the task is narrowed to the files that already existed. if let Some(base) = &base { task.ground_truth.retain(|path| exists_at(root, base, path)); if task.ground_truth.is_empty() { - continue; + return None; } } - tasks.push(task); - } + Some(task) + }); Ok(TaskSet { repository: root.display().to_string(), @@ -164,6 +134,88 @@ pub fn generate( }) } +/// The selection loop both generators share: newest first, stopping at the `--after` +/// cutoff, starting below the `--until` boundary, skipping excluded commits, taking +/// the first `wanted` for which `pick` yields something. +/// +/// Shared rather than copied so a filter can never apply to one generator and not the +/// other — which is the way two task sets silently stop being comparable. +fn take( + history: &gitlog::History, + start: usize, + cutoff: Option, + exclude: &BTreeSet, + wanted: usize, + mut pick: impl FnMut(&gitlog::Commit) -> Option, +) -> Vec { + let mut taken = Vec::new(); + for (i, commit) in history.commits.iter().enumerate().skip(start) { + if taken.len() >= wanted { + break; + } + if cutoff.is_some_and(|stop| i >= stop) { + break; + } + if exclude.contains(&commit.sha) { + continue; + } + if let Some(item) = pick(commit) { + taken.push(item); + } + } + taken +} + +/// The slice of history a generator may draw from: `(start, cutoff)` as indices into +/// the merge-free scan, newest first. +/// +/// `--after` sets the cutoff — the walk stops there, so every task describes a change +/// made after the state an index will be built at. `--until` sets the start — the +/// walk begins strictly *below* it, which is how a training corpus stays disjoint +/// from every evaluation window. +fn window( + root: &Path, + history: &gitlog::History, + head: &str, + after: Option<&str>, + until: Option<&str>, +) -> Result<(usize, Option)> { + let position = |sha: &str| { + history + .commits + .iter() + .position(|c| c.sha.starts_with(sha) || sha.starts_with(&c.sha)) + }; + let cutoff = after + .map(|rev| resolve(root, rev)) + .transpose()? + .and_then(|sha| position(&sha)); + let start = match until.map(|rev| resolve(root, rev)).transpose()? { + None => 0, + // The scanned list is merge-free, so a merge commit named as the boundary is + // legitimately absent from it. When the boundary is HEAD itself, "strictly + // older than HEAD" excludes nothing the list contains. + Some(sha) if sha == head => 0, + Some(sha) => match position(&sha) { + Some(position) => position + 1, + // A merge commit is legitimately absent from the merge-free list, so the + // boundary falls back to its timestamp: strictly-older-than holds for + // every commit authored before it. + None => { + let at = commit_time(root, &sha)?; + history + .commits + .iter() + .position(|c| c.timestamp < at) + .with_context(|| { + format!("--until {sha}: nothing older within the scanned history") + })? + } + }, + }; + Ok((start, cutoff)) +} + /// Author timestamp of a commit, for boundary fallback when the commit itself is a /// merge and therefore missing from the merge-free scan. fn commit_time(root: &Path, sha: &str) -> Result { @@ -284,6 +336,341 @@ fn clean_subject(subject: &str) -> String { out.split_whitespace().collect::>().join(" ") } +// ---- the held-out-hunk task set -------------------------------------------- +// +// A retrieval task asks *which files should I open*. A held-out-hunk task asks the +// opposite question: given a patch that is deliberately incomplete, *what did it +// miss*? The construction is model-free — a merged commit is complete by definition, +// so removing one hunk from it manufactures a known omission and leaves the complete +// commit behind as a negative control. Nothing is hand-labelled and nothing is +// judged; the label is the hunk that was taken out. + +/// One hunk of a unified diff, in pre-image coordinates. +/// +/// Pre-image, because the index this is scored against is built at the parent commit: +/// post-image line numbers name lines that do not exist there. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Hunk { + /// First line of the hunk, context included. + pub old_start: u32, + /// Lines of the pre-image the hunk covers, context included. + pub old_len: u32, + /// Pre-image lines the hunk actually *changes*, with context excluded. + /// + /// Separate from the span because context lines routinely reach into the + /// neighbouring function, and resolving a symbol from them would attribute a + /// change to code the patch never touched. + pub changed_lines: Vec, +} + +impl Hunk { + /// The first line this hunk changes, for resolving the symbol it lands in. + pub fn first_changed(&self) -> u32 { + self.changed_lines + .first() + .copied() + .unwrap_or(self.old_start) + } +} + +/// One file's worth of a change. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FilePatch { + /// Pre-image path, or the post-image path for a file the change creates. + pub path: String, + /// The change creates this file, so it has no pre-image and no indexed symbols. + pub created: bool, + pub hunks: Vec, +} + +/// A change, as the set of files and pre-image lines it touches. +/// +/// Structural rather than textual on purpose: the checker under test reads locations +/// and the graph, never diff text, so carrying the text would invite a checker that +/// greps it. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct Patch { + pub files: Vec, +} + +impl Patch { + /// The same change with one file left out entirely. + fn without(&self, path: &str) -> Patch { + Patch { + files: self + .files + .iter() + .filter(|f| f.path != path) + .cloned() + .collect(), + } + } +} + +/// One held-out-hunk trial: a truncated change, and the complete one it came from. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TruncatedTask { + pub id: String, + pub commit: String, + /// The commit an index must be built at. The change is absent there by + /// construction, which is the same guarantee `--after` gives the retrieval set. + pub parent: String, + pub date: String, + /// The developer's own description, kept for tracing a result back to a change. + pub prompt: String, + /// The change as merged. Complete by construction, so every finding against it is + /// a false positive — this is the negative control, not a second data point. + pub complete: Patch, + /// The change with the omission's file removed. + pub truncated: Patch, + /// The file whose only hunk was withheld. + pub omission_file: String, + /// First pre-image line the withheld hunk changes. + pub omission_line: u32, +} + +/// A frozen set of held-out-hunk trials. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TruncatedSet { + pub repository: String, + pub head: String, + pub generated_from_commits: usize, + /// Commits that passed every retrieval filter but could not be truncated, and why. + /// Reported rather than dropped: a construction that silently discards most of its + /// candidates is measuring the survivors, not the repository. + pub rejected: Vec<(String, String)>, + pub tasks: Vec, +} + +/// Build held-out-hunk trials from a repository's history. +/// +/// Every filter `generate` applies applies here unchanged — the two share `take` and +/// `candidate` — plus two the construction needs: +/// +/// 1. the change must touch **at least two** indexable files that exist at the parent, +/// so removing one leaves a patch behind; +/// 2. one of those files must be touched by **exactly one hunk**, which is the hunk +/// withheld. Removing it removes the file from the patch entirely, so a finding +/// that cites that file cannot be an echo of a hunk still present in it. +/// +/// Among the files with exactly one hunk the **last by path order** is chosen. The +/// choice is arbitrary and fixed; it is made before any checker runs and there is no +/// knob on it. +pub fn generate_truncated( + root: &Path, + wanted: usize, + scan: usize, + after: Option<&str>, + until: Option<&str>, + exclude: &BTreeSet, +) -> Result { + let head = gitlog::head_sha(root).context("reading HEAD")?; + let history = gitlog::history(root, scan)?; + let (start, cutoff) = window(root, &history, &head, after, until)?; + + let mut rejected: Vec<(String, String)> = Vec::new(); + let tasks = take(&history, start, cutoff, exclude, wanted, |commit| { + let task = candidate(root, commit)?; + if task.ground_truth.len() < 2 { + return None; // a one-file change has nothing left after a truncation + } + let parent = match parent_of(root, &commit.sha) { + Some(parent) => parent, + None => { + rejected.push((commit.sha.clone(), "no parent commit".into())); + return None; + } + }; + let patch = match parse_patch(root, &commit.sha) { + Ok(patch) => patch, + Err(_) => { + rejected.push((commit.sha.clone(), "unreadable diff".into())); + return None; + } + }; + + // Only files that exist at the parent and that the indexer treats as code can + // carry a symbol the checker could ever cite. + let indexable: Vec<&FilePatch> = patch + .files + .iter() + .filter(|f| { + !f.created + && !f.hunks.is_empty() + && reify::discover::classify(&f.path).is_code() + && exists_at(root, &parent, &f.path) + }) + .collect(); + if indexable.len() < 2 { + rejected.push((commit.sha.clone(), "fewer than two indexable files".into())); + return None; + } + let Some(omission) = indexable + .iter() + .filter(|f| f.hunks.len() == 1) + .max_by(|a, b| a.path.cmp(&b.path)) + else { + rejected.push(( + commit.sha.clone(), + "no file changed by exactly one hunk".into(), + )); + return None; + }; + let omission_file = omission.path.clone(); + let omission_line = omission.hunks[0].first_changed(); + + let complete = Patch { + files: indexable.iter().map(|f| (*f).clone()).collect(), + }; + Some(TruncatedTask { + id: format!("v-{}", &commit.sha[..8]), + commit: commit.sha.clone(), + parent, + date: commit.date(), + prompt: task.prompt, + truncated: complete.without(&omission_file), + complete, + omission_file, + omission_line, + }) + }); + + Ok(TruncatedSet { + repository: root.display().to_string(), + head, + generated_from_commits: history.commits.len(), + rejected, + tasks, + }) +} + +/// First parent of a commit, or `None` for a root commit. +fn parent_of(root: &Path, sha: &str) -> Option { + let output = Command::new("git") + .args(["rev-parse", &format!("{sha}^")]) + .current_dir(root) + .output() + .ok()?; + if !output.status.success() { + return None; + } + Some(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +/// The change a commit made, against its first parent. +fn parse_patch(root: &Path, sha: &str) -> Result { + let output = Command::new("git") + .args([ + "-c", + "core.quotepath=false", + "show", + "--format=", + "--no-color", + // Renames would report a file as changed with no hunks in it, and a + // similarity threshold is a knob this measurement should not have. + "--no-renames", + "--first-parent", + sha, + ]) + .current_dir(root) + .output() + .context("running git show for a patch")?; + anyhow::ensure!(output.status.success(), "cannot read the diff of {sha}"); + Ok(parse_unified(&String::from_utf8_lossy(&output.stdout))) +} + +/// Parse unified diff text into files and pre-image line ranges. +fn parse_unified(text: &str) -> Patch { + let mut patch = Patch::default(); + let mut cursor = 0u32; + for line in text.lines() { + if let Some(rest) = line.strip_prefix("--- ") { + patch.files.push(FilePatch { + path: strip_prefix_path(rest), + created: rest == "/dev/null", + hunks: Vec::new(), + }); + continue; + } + let Some(file) = patch.files.last_mut() else { + continue; + }; + if let Some(rest) = line.strip_prefix("+++ ") { + // A created file has no pre-image path, so the post-image one names it. + if file.created { + file.path = strip_prefix_path(rest); + } + continue; + } + if let Some(header) = line.strip_prefix("@@ ") { + if let Some((old_start, old_len)) = parse_hunk_header(header) { + file.hunks.push(Hunk { + old_start, + old_len, + changed_lines: Vec::new(), + }); + cursor = old_start; + } + continue; + } + let Some(hunk) = file.hunks.last_mut() else { + continue; + }; + match line.chars().next() { + Some(' ') => cursor += 1, + Some('-') => { + hunk.changed_lines.push(cursor); + cursor += 1; + } + // An inserted line has no pre-image number of its own. + // + // When it replaces lines this hunk has already deleted, it needs no number: + // those lines are recorded and the replacement is the same change. When it + // is a genuine insertion it is attributed to the pre-image line it lands + // *before*, not the one it lands after — code appended past the end of a + // file then resolves to no symbol at all, which is the honest answer, where + // attributing it backwards would credit the patch with changing a function + // it only wrote underneath. + Some('+') => { + let replaces = cursor > 0 && hunk.changed_lines.last() == Some(&(cursor - 1)); + if !replaces { + hunk.changed_lines.push(cursor.max(hunk.old_start)); + } + } + // "\ No newline at end of file", or a blank line git emits as empty. + _ => {} + } + } + for file in &mut patch.files { + for hunk in &mut file.hunks { + hunk.changed_lines.sort_unstable(); + hunk.changed_lines.dedup(); + } + } + patch +} + +/// `a/src/x.rs` -> `src/x.rs`; `/dev/null` -> empty. +fn strip_prefix_path(raw: &str) -> String { + let raw = raw.trim_end(); + if raw == "/dev/null" { + return String::new(); + } + raw.split_once('/') + .map_or(raw, |(_, rest)| rest) + .to_string() +} + +/// `-12,7 +12,8 @@ fn something` -> `(12, 7)`. +fn parse_hunk_header(header: &str) -> Option<(u32, u32)> { + let old = header.split_whitespace().next()?.strip_prefix('-')?; + let (start, len) = match old.split_once(',') { + Some((start, len)) => (start, len.parse().ok()?), + None => (old, 1u32), + }; + Some((start.parse().ok()?, len)) +} + #[cfg(test)] mod tests { use super::*; @@ -335,6 +722,101 @@ mod tests { } } + #[test] + fn the_repository_name_is_the_last_path_component() { + let set = |repository: &str| TaskSet { + repository: repository.into(), + head: String::new(), + generated_from_commits: 0, + base: None, + tasks: Vec::new(), + }; + assert_eq!(set(".bench/medusa").repository_name(), "medusa"); + assert_eq!(set("/a/b/openmrs/").repository_name(), "openmrs"); + assert_eq!(set("reify").repository_name(), "reify"); + } + + const SAMPLE_DIFF: &str = "\ +diff --git a/app/pricing.py b/app/pricing.py +index 111..222 100644 +--- a/app/pricing.py ++++ b/app/pricing.py +@@ -10,6 +10,7 @@ class Pricing: + ctx + ctx +- old_line ++ new_line ++ extra_line + ctx + ctx +@@ -40,3 +41,3 @@ def other(): + ctx +- gone ++ added +diff --git a/app/new.py b/app/new.py +new file mode 100644 +--- /dev/null ++++ b/app/new.py +@@ -0,0 +1,2 @@ ++one ++two +"; + + #[test] + fn a_unified_diff_parses_into_pre_image_line_ranges() { + let patch = parse_unified(SAMPLE_DIFF); + assert_eq!(patch.files.len(), 2); + let pricing = &patch.files[0]; + assert_eq!(pricing.path, "app/pricing.py"); + assert!(!pricing.created); + assert_eq!(pricing.hunks.len(), 2); + assert_eq!( + (pricing.hunks[0].old_start, pricing.hunks[0].old_len), + (10, 6) + ); + // Line 12 is the deletion; the insertions replace it, so it is the only + // pre-image line the hunk changes. + assert_eq!(pricing.hunks[0].changed_lines, vec![12]); + assert_eq!(pricing.hunks[1].changed_lines, vec![41]); + assert_eq!(pricing.hunks[1].first_changed(), 41); + } + + #[test] + fn a_created_file_is_marked_and_named_from_its_post_image() { + let patch = parse_unified(SAMPLE_DIFF); + let created = &patch.files[1]; + assert!( + created.created, + "a file with no pre-image has no indexed symbols" + ); + assert_eq!(created.path, "app/new.py"); + } + + #[test] + fn code_appended_past_the_end_of_a_file_resolves_past_the_end() { + // Attributing an append backwards would credit the patch with changing the + // function it was written underneath. It changed no existing line. + let patch = parse_unified( + "--- a/a.py\n+++ b/a.py\n@@ -8,3 +8,5 @@\n ctx\n ctx\n ctx\n+new\n+new\n", + ); + assert_eq!(patch.files[0].hunks[0].changed_lines, vec![11]); + } + + #[test] + fn truncating_removes_the_file_and_leaves_the_rest() { + let patch = parse_unified(SAMPLE_DIFF); + let truncated = patch.without("app/pricing.py"); + assert_eq!(truncated.files.len(), 1); + assert_eq!(truncated.files[0].path, "app/new.py"); + } + + #[test] + fn a_hunk_header_without_a_length_means_one_line() { + assert_eq!(parse_hunk_header("-12 +12,3 @@ fn x"), Some((12, 1))); + assert_eq!(parse_hunk_header("-12,7 +12,8 @@ fn x"), Some((12, 7))); + assert_eq!(parse_hunk_header("nonsense"), None); + } + #[test] fn a_commit_touching_too_many_files_is_not_a_task() { let files: Vec = (0..30).map(|i| format!("f{i}.py")).collect(); diff --git a/docs/metrics.md b/docs/metrics.md index c5f23a0..ef2a899 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -32,6 +32,28 @@ liability, not marketing. | **Expected tokens** | Mean tokens to reach a changed file, charging a miss the full budget. The comparable single number: a condition cannot improve it by failing more often. | | **Head to head** | Median tokens over only the tasks *both* conditions solved. Removes the difficulty bias in the per-condition median. | +## Held-out-hunk benchmark + +`reify-bench verify-eval`. Each trial takes a merged commit, withholds one file's only +hunk — the *omission* — and asks the graph what the truncated patch missed. The same +commit is then run complete; a merged commit is complete by construction, so every +finding there is a false positive. + +| Metric | Definition | +|---|---| +| **`omission_recall`** | Truncated diffs where some finding cites the omitted hunk's file, over all trials. | +| **…attributable** | The same, counting only citations the *complete* commit does not also produce. A citation the negative control produces too was not caused by the omission. | +| **`omission_recall_symbol`** | The same at symbol granularity, over the trials whose omission falls inside an indexed symbol. Trials where it does not are excluded, never counted as misses. | +| **Omitted files a caller query could cite** | Trials whose omitted file has an outbound cross-file `CALLS` edge at the parent commit. Every finding is a caller, so `omission_recall` cannot exceed this share however the query is written. | +| **`false_alarm_rate`** | Findings per *complete* merged commit. A rate over counts, not a proportion, so it carries no Wilson interval; the share of commits with at least one false alarm is reported beside it and does. | +| **`findings_per_diff`** | Median findings per truncated diff. Median rather than mean because a checker emitting thirty findings once is not a checker with a small problem everywhere. | +| **`verify_tokens`** | Median `heuristic-v1` estimate of the findings output itself — what an agent pays to read the answer. Excludes the diff and the files it would then open. | +| **`verify_latency_ms`** | Median wall clock of the graph query alone. Extracting and indexing the parent tree is reported separately, because the real feature would run against an index that already exists. | + +The checker under test is not `reify verify`, which does not exist. It is the shipped +graph query — *symbols changed by this diff, minus symbols present in the diff, where +an inbound `CALLS` edge exists at distance 1* — reached through `reify::query::impact`. + ## Token estimation `reify` estimates tokens with `heuristic-v1`: Latin script at four bytes per token, CJK From d1c9e17f0fdf91df0113291904345acd290e9e22 Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 24 Aug 2026 11:10:58 +0700 Subject: [PATCH 5/8] docs(readme): remove duplication, fix three self-contradictions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README disagreed with itself in ways that undercut the measurement discipline it is arguing for. - Three different answers for its own index timings. The install section said 4.6s/0.7s where the measured table says 4.2s/0.49s. Unified to the measured numbers. - It said `init` appends "a six-line block", then showed a four-line block and called it "the same four lines". Neither matched what the tool writes. It now shows the actual constant, including the two lines the paraphrase dropped. - Shell completions were documented twice, forty lines apart. - The quickstart and Install repeated the same commands and the same uninstall explanation; Install now carries only what the quickstart does not. - The optimisation war stories are compressed to a paragraph pointing at the changelog, keeping the `git log -L` lazy-fetch story because it is load-bearing for the privacy claim rather than for speed. Adds the verify-benchmark section, and records that `impact` now takes a file. Net 48 lines shorter. Every measurement section — Medusa's no-win row, "where it doesn't work", the p-value section — is untouched. --- README.md | 94 +++++++++++++++++++++++++++---------------------------- 1 file changed, 46 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index c66f4d5..1636e9f 100644 --- a/README.md +++ b/README.md @@ -55,9 +55,8 @@ reify context "the change you are about to make" --toon One static binary — no daemon, no config, no API key, and every release ships a SHA-256 checksum that both the installer above and `reify upgrade` verify before -anything is unpacked. Changed your mind? -`reify uninstall` removes the binary and `reify uninit` cleans one repository, both -showing their plan first. Per-agent wiring, hooks and MCP: Install. +anything is unpacked. Per-agent wiring, hooks, MCP and how to leave cleanly: +Install.

English · Tiếng Việt · 简体中文 @@ -390,13 +389,6 @@ Or build from source: cargo install --path crates/reify-cli ``` -Then, in any repository: - -```bash -reify init # tells you what it will and won't index, and why -reify index # 4.6s for 5,000 files; 0.7s after you edit one -``` - **Stay current, leave cleanly.** `reify upgrade` replaces the binary with the latest release — through `curl` and `tar` as visible subprocesses, never an embedded HTTP client, with the checksum verified before anything is installed; `--check` only asks, @@ -422,16 +414,20 @@ reify completions fish > ~/.config/fish/completions/reify.fish reify init --write-agent-instructions ``` -Appends a six-line block to `AGENTS.md` or `CLAUDE.md`. No protocol, no server, no -per-turn schema tax — this is the level the benchmark measured. For tools that read a -different file (`.cursorrules`, `CONVENTIONS.md`, `.windsurfrules`, `.clinerules/`), -paste the same four lines: +Appends this block to `AGENTS.md` or `CLAUDE.md`. No protocol, no server, no per-turn +schema tax — this is the level the benchmark measured. For tools that read a different +file (`.cursorrules`, `CONVENTIONS.md`, `.windsurfrules`, `.clinerules/`), paste it +there instead: ```markdown -Before changing code here, run `reify context "" --toon`. +## Before changing code in this repository + +Run `reify context "" --toon` and read its output first. Run `reify why :` before modifying unfamiliar logic. Run `reify impact ""` before changing anything shared. -Treat INFERRED claims as leads to verify, not facts. + +Claims marked `INFERRED` are leads to verify against their citation, not facts. +If `conflicts` is non-empty, resolve the disagreement before changing behaviour. ``` **MCP**, if you prefer it: `reify serve --mcp` exposes six tools — `reify_context`, @@ -445,11 +441,7 @@ they cost under 600 tokens, which six still fit inside. [Privacy](#privacy) for why that is a command and not an HTTP client.

-Shell completions, and a pre-edit risk hook - -```bash -reify completions zsh > ~/.zfunc/_reify # also bash, fish -``` +A pre-edit risk hook, and keeping the index fresh Inject a risk header before every edit, under 300 tokens, asserted by a test because it runs on every edit. Non-blocking by default: a hook that blocks edits gets uninstalled, @@ -481,7 +473,7 @@ chmod +x .git/hooks/post-merge && cp .git/hooks/post-merge .git/hooks/post-check |---|---| | `reify context ""` | The minimum knowledge for a change, plus a reading plan. **The one that matters.** `--toon` emits the agent format | | `reify why :` | What this is, what calls it, what data it touches, what changed it | -| `reify impact ""` | What depends on it — including through the database, where no call edge exists | +| `reify impact ""` | What depends on it — callers, importers, and coupling through the database where no call edge exists | | `reify explain ""` | A business concept across every language, table and file it appears in | | `reify flow ""` | The call sequence that carries out a business process | | `reify conflicts` | Documentation that disagrees with the code | @@ -557,32 +549,19 @@ ERPNext, 5,064 files, 8-core M-series laptop. | peak memory, full index | 224 MB | | store size | 47 MB (33% of a 144 MB working tree) | -A full index took **78 seconds** until the full-text index was keyed by node id. `uid` is `UNINDEXED` in FTS5, so `DELETE ... WHERE uid = ?` scanned the whole table once per node — quadratic, and invisible until it was timed per stage. Editing one file took **5.9 seconds** until the repository-wide stages learned to skip when their inputs are provably unchanged. - -Reindexing was **2× slower** until two things stopped being done repository-wide for -a one-line edit. Discovery read and hashed all 5,285 files on every run — 222 ms of -reading to find the handful that moved — and now `stat`s past anything whose size and -modification time are unchanged, hashing the rest across all cores. Reference -resolution reloaded and re-resolved all **144,309** references, 167 ms to resolve and -145 ms to commit, regardless of how little changed; it now re-resolves only references -whose *name* the edit added or removed, plus those inside the edited files, which is -provably the whole affected set. Measured against the previous build on the same -machine: full index 6.75 s → 4.25 s, no-op reindex 256 ms → 101 ms, one file edited -974 ms → 486 ms. - -`reify why` was **1.5 seconds** on a blobless clone, and returned a *worse* answer than -it does now. `git log -L` needs the file's blob at every revision it walks, and on a -partial clone those blobs are not local — so git was silently fetching them from the -remote, one query costing 29.5 s of network and 0.07 s of work. The subprocess now runs -with `GIT_NO_LAZY_FETCH=1`: git answers from local objects or fails, and either way the -command returns in milliseconds. Eleven of twelve sampled symbols used to hit the -timeout; none do. - -That fix is also why the privacy claim below is true of the whole process tree rather -than just this binary. Reify never opened a socket; the git it spawned did. - -`REIFY_TIMING=1 reify index` prints the per-stage breakdown that found every one of -these. +Those numbers are the end of a long optimisation, not a first draft: a full index took +**78 seconds** and a one-file reindex **5.9 seconds** before the stages learned to skip +work whose inputs are provably unchanged. `REIFY_TIMING=1 reify index` prints the +per-stage breakdown that found every one of them, and [CHANGELOG.md](CHANGELOG.md) has +the before-and-after for each. + +One of those fixes is load-bearing for the privacy claim below rather than for speed. +`reify why` runs `git log -L`, which needs the file's blob at every revision it walks — +and on a blobless clone those blobs are not local, so git was silently **fetching them +from the remote**: 29.5 s of network for 0.07 s of work. Every `git` invocation now sets +`GIT_NO_LAZY_FETCH=1`, so it answers from local objects or fails. Reify never opened a +socket; the git it spawned did. That is why the guarantee below is stated over the whole +process tree and not just this binary. ## Reproducing the benchmark @@ -608,6 +587,25 @@ reify-bench chart --results "Mine=results/" --out assets/ The task set is frozen before any condition runs. The report includes a **"Where Reify lost"** section listing every task the baseline won, and it is a required part of the document rather than an optional one. +### The benchmark that killed a feature + +`reify verify` — a post-flight check that reads an agent's diff and reports what the +patch missed — was measured before it was written. The harness withholds one hunk from +a real merged commit, asks the graph what the patch missed, then runs the *complete* +commit through the same query, where every finding is a false positive by construction. +Model-free, deterministic, 116s for three repositories. + +```bash +reify-bench verify-eval --repo --out results/verify- --until +reify-bench verify-report --results "name=results/verify-" --out benchmarks/REPORT-verify.md +``` + +It failed its pre-registered condition on all three: the graph finds the omitted file +often enough — `omission_recall` 0.40 on Go, 0.50 on Rust, 0.10 on Python — but reports +4.4 to 23.5 findings against commits that are already complete. A `CALLS` edge says a +caller exists; it does not say the caller needed changing. **The feature was not +built.** [Full writeup](benchmarks/REPORT-verify.md). + ## Development ```bash From e32f5678ba396cf5910f37fd7947b2d7a91bcf27 Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 24 Aug 2026 11:38:10 +0700 Subject: [PATCH 6/8] feat(doctor): tell people not to use this, when they should not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing told anyone whether Reify would help before they invested in it. The README FAQ answered it informally and `benchmarks/REPORT-medusa.md` already shows the advantage can be zero on a real repository. `reify doctor` makes that measured instead of anecdotal, and runs before there is an index — the whole point is deciding before committing to the tool. The signals were fitted against the four repositories in `benchmarks/`, not invented. Two hypotheses were tested and dropped: size inverts (OFBiz wins biggest at 1,364 code files, Medusa ties at 11,821), and so does the obvious reading of "history and file naming share vocabulary" — pooled corpus-wide, Medusa scores 0.43 against OFBiz's 0.38. Two signals do fit all four outcomes, each explaining a different failure: repo grep margin commit focus subject->path OFBiz +58 0.96 0.80 ERPNext +48 0.98 0.85 OpenMRS +9 0.98 0.48 Medusa 0 0.84 0.79 Commit focus is the share of commits touching few enough files that their subject says anything about them; Medusa is the only measured repository where it falls away and the only one where Reify won nothing. The mechanism is already load-bearing elsewhere: `History::co_changes` discards commits touching more than 20 files for the same reason, so `doctor` uses the same threshold rather than a second definition of a sweeping commit. No suitability score. `docs/metrics.md` forbids printing a number that cannot be defined, and a weighted blend of heuristics tuned on four repositories is exactly that. Named signals, measured values, a plain verdict, and — where the answer is no or marginal — what would change it. The verdict floor is 200 commits, which is where a 95% Wilson interval around Medusa's 0.84 lies wholly below the threshold; at 50 it straddles it, so a shorter history is reported as short rather than condemned. Verified on all four benchmarked repositories, which land where their reports say they should, plus django (yes) and a two-commit scratch repository (too small). No new dependency; the offline guarantee is untouched. --- README.md | 4 +- crates/reify-cli/src/main.rs | 8 + crates/reify-cli/src/render.rs | 203 ++++++++ crates/reify/src/doctor.rs | 884 +++++++++++++++++++++++++++++++++ crates/reify/src/lib.rs | 1 + docs/json-schema/README.md | 47 ++ docs/json-schema/regenerate.sh | 2 + docs/metrics.md | 23 + 8 files changed, 1171 insertions(+), 1 deletion(-) create mode 100644 crates/reify/src/doctor.rs diff --git a/README.md b/README.md index 1636e9f..e7b96a4 100644 --- a/README.md +++ b/README.md @@ -482,6 +482,7 @@ chmod +x .git/hooks/post-merge && cp .git/hooks/post-merge .git/hooks/post-check | `reify preflight ` | A risk header for an editor hook | | `reify report` | System scorecard | | `reify status` | Freshness, coverage, and what was skipped | +| `reify doctor` | Should this repository use Reify at all? Runs before there is an index, and will say no | | `reify llm status \| preview` | Is a model configured, and exactly what would be sent | | `reify upgrade [--check]` | Replace this binary with the latest release. The only networked command; refused under `REIFY_OFFLINE=1` | | `reify uninstall --yes` \| `uninit --yes` | Remove the binary \| one repository's store and instruction block | @@ -645,7 +646,8 @@ which is why every answer comes with a line number instead of a similarity score **My repo is 3,000 lines. Should I use it?** No. Use ripgrep. Under roughly 20k LOC Reify buys you nothing a grep and a scroll wheel -don't. +don't. `reify doctor` applies that floor and three other signals to your repository, before +you index it. **Does it send my proprietary code anywhere?** It cannot. There is no HTTP client in the binary, and a test fails the build if one diff --git a/crates/reify-cli/src/main.rs b/crates/reify-cli/src/main.rs index e4197a7..0852098 100644 --- a/crates/reify-cli/src/main.rs +++ b/crates/reify-cli/src/main.rs @@ -141,6 +141,12 @@ enum Command { path: String, }, + /// Should this repository use Reify at all? Answers before you index. + /// + /// Runs against the working tree and `git log`, never the store, so it works + /// before `reify init`. Willing to say no. + Doctor, + /// Model-assistance status and prompt inspection. Llm { #[command(subcommand)] @@ -308,6 +314,7 @@ fn run() -> Result<()> { let store = open_existing(&root)?; render::preflight(&query::preflight(&store, path)?, cli.json) } + Command::Doctor => render::doctor(&reify::doctor::diagnose(&root)?, cli.json), Command::Llm { action } => match action { LlmAction::Status => render::llm_status(&root, cli.json), LlmAction::Preview { task, budget } => { @@ -550,6 +557,7 @@ mod tests { vec!["reify", "--json", "preflight", "a.py"], vec!["reify", "--json", "llm", "status"], vec!["reify", "--json", "init"], + vec!["reify", "--json", "doctor"], ] { let cli = Cli::try_parse_from(&args).expect("should parse"); assert!(cli.json, "{args:?}"); diff --git a/crates/reify-cli/src/render.rs b/crates/reify-cli/src/render.rs index aff1195..796ab16 100644 --- a/crates/reify-cli/src/render.rs +++ b/crates/reify-cli/src/render.rs @@ -19,6 +19,7 @@ use serde::Serialize; use reify::context::Context; use reify::discover::Discovery; +use reify::doctor::{self, Diagnosis, Verdict}; use reify::index::IndexReport; use reify::llm; use reify::model::{Node, Status}; @@ -759,6 +760,208 @@ pub fn preflight(answer: &Preflight, json: bool) -> Result<()> { Ok(()) } +/// `reify doctor`: should this repository use Reify at all? +/// +/// Named signals with measured values and a plain-language verdict. Deliberately not a +/// score: `docs/metrics.md` forbids printing a number that cannot be defined, and a +/// weighted blend of four heuristics tuned on four repositories is exactly that. +pub fn doctor(answer: &Diagnosis, json: bool) -> Result<()> { + if json { + return emit_json(answer); + } + println!("DOCTOR {}", answer.root); + println!(); + + let floor = doctor::floor_text(); + signal( + "scale", + &doctor::scale_text(&answer.scale), + if answer.verdict == Verdict::TooSmall { + format!("below the {floor} floor") + } else { + format!("above the {floor} floor") + }, + ); + + // Below the floor the other signals were never computed, and saying why is more + // useful than printing three lines of zeroes. + if answer.verdict == Verdict::TooSmall { + verdict_line(answer); + return Ok(()); + } + + match &answer.vocabulary { + Some(v) => signal( + "vocabulary", + &format!( + "{} of focused commits name a file they changed", + doctor::percent(v.locality) + ), + format!("{} of {} commits", v.commits_local, v.commits_considered), + ), + None => signal( + "vocabulary", + "not measurable without git history", + String::new(), + ), + } + match &answer.history { + Some(h) => { + signal( + "history", + &format!( + "{} commits, {} focused enough to attribute", + h.commits_read, + doctor::percent(h.focus) + ), + format!("median commit changes {} file(s)", h.median_files_changed), + ); + // Said only when it is not the case. On all five repositories this was + // calibrated against it sat at 100%, so printing it always would be noise. + if h.usable_share < 0.9 { + signal( + "", + &format!( + "only {} carry a subject worth reading", + doctor::percent(h.usable_share) + ), + String::new(), + ); + } + } + None => signal("history", "could not be read", String::new()), + } + signal( + "documents", + &format!( + "{} document(s) only Reify can read", + answer.documents.unreadable_by_grep + ), + answer.documents.examples.join(", "), + ); + + verdict_line(answer); + Ok(()) +} + +/// One measured signal: name, value, and the note that puts it in context. +fn signal(name: &str, value: &str, note: String) { + let name = if colours_wanted() { + format!("{:<12}", name.bold()) + } else { + format!("{name:<12}") + }; + if note.is_empty() { + println!(" {name}{value}"); + } else if colours_wanted() { + println!(" {name}{value:<48}{}", note.dimmed()); + } else { + println!(" {name}{value:<48}{note}"); + } +} + +fn verdict_line(answer: &Diagnosis) { + let text = answer.verdict.as_str(); + let painted = if colours_wanted() { + match answer.verdict { + Verdict::LikelyWorthIt => text.green().bold().to_string(), + Verdict::TooSmall | Verdict::UnlikelyToHelp => text.red().bold().to_string(), + Verdict::Marginal => text.yellow().bold().to_string(), + } + } else { + text.to_string() + }; + // The verdict word is part of the first wrapped line, so the wrap has to know how + // wide it is — measured on the unpainted text, since colour codes take no columns. + let lead = format!(" {text} — "); + println!( + "\n {painted} — {}", + wrap(&answer.reason, WIDTH, " ", lead.chars().count()) + ); + + if !answer.what_would_change_it.is_empty() { + println!("\n What would change this:"); + for item in &answer.what_would_change_it { + println!(" - {}", wrap(item, WIDTH, " ", 6)); + } + } + if let Some(c) = doctor::comparable(answer.verdict) { + // Named by role rather than by favourability: for a yes the useful comparison + // is the repository where Reify did worst, and for a no it is the one where it + // did best. Calling both "least favourable" would be wrong half the time. + let role = match answer.verdict { + Verdict::UnlikelyToHelp => "did best", + _ => "did worst", + }; + println!( + "\n {}", + wrap( + &format!( + "For comparison, the measured repository where Reify {role}: {}, \ + where {}. See {}.", + c.name, c.outcome, c.report + ), + WIDTH, + " ", + 2, + ) + ); + } + // The verdict is a heuristic over four repositories, and a reader could otherwise + // mistake it for a measurement of theirs. The cost is stated for the same reason + // the answer is: so nobody has to guess what running it will take. + println!( + "\n {}", + wrap( + &format!( + "This is a heuristic fitted to four measured repositories, not a \ + measurement of this one — `reify-bench` measures this one. Read the \ + working tree and {} in {:.1}s; no index needed, and none was used.", + if answer.git_repository { + "the newest 1000 commits" + } else { + "no history" + }, + answer.elapsed_ms as f64 / 1000.0, + ), + WIDTH, + " ", + 2, + ) + ); +} + +/// Terminal width the doctor output is wrapped to. +/// +/// Fixed rather than read from the terminal: the verdict is the one paragraph that must +/// be readable, and it must read the same in a pipe, a CI log and a screenshot. +const WIDTH: usize = 78; + +/// Wrap `text` to `width`, indenting continuation lines by `indent`. +/// +/// `first_column` is how far into the line the caller has already printed, so a verdict +/// word or a bullet marker is counted against the first line's budget rather than +/// pushing it past the right edge. +fn wrap(text: &str, width: usize, indent: &str, first_column: usize) -> String { + let mut out = String::new(); + let mut column = first_column; + let mut fresh = true; + for word in text.split_whitespace() { + if !fresh && column + 1 + word.chars().count() > width { + out.push('\n'); + out.push_str(indent); + column = indent.chars().count(); + } else if !fresh { + out.push(' '); + column += 1; + } + out.push_str(word); + column += word.chars().count(); + fresh = false; + } + out +} + pub fn concepts(overview: &ConceptOverview, json: bool) -> Result<()> { if json { return emit_json(overview); diff --git a/crates/reify/src/doctor.rs b/crates/reify/src/doctor.rs new file mode 100644 index 0000000..9788dc4 --- /dev/null +++ b/crates/reify/src/doctor.rs @@ -0,0 +1,884 @@ +//! Should this repository use Reify at all? +//! +//! A tool that always recommends itself is worthless, and this project has already +//! published a repository — `benchmarks/REPORT-medusa.md` — where Reify ties grep. The +//! most valuable answer this module can give is a confident *no*, because that is what +//! makes the *yes* worth anything. +//! +//! # Where the signals come from +//! +//! Four repositories were measured end to end (`benchmarks/REPORT*.md`), and two +//! hypotheses were tested against them. Both failed: +//! +//! - **Size.** OFBiz has 1,364 code files and shows the largest margin over grep +//! (70% against 12%); Medusa has 11,821 and shows none (18% against 18%). +//! - **Declared vocabulary.** OFBiz declares almost nothing and still wins. +//! +//! Two signals *do* fit all four outcomes, and each explains a different way the tool +//! fails. Measured over the newest [`MAX_COMMITS`] commits of each repository: +//! +//! | | grep margin | commit focus | subject→path | +//! |---|---:|---:|---:| +//! | OFBiz | +58 | 0.96 | 0.80 | +//! | ERPNext | +48 | 0.98 | 0.85 | +//! | OpenMRS | +9 | 0.98 | **0.48** | +//! | Medusa | 0 | **0.84** | 0.79 | +//! +//! **Commit focus** is the share of commits touching few enough files that their +//! subject says something about them. Medusa is the only measured repository where it +//! falls away, and it is the only one where Reify did not win. That is not a +//! coincidence: Reify attaches a commit's vocabulary to every file it touched, so a +//! history of sweeping squashed merges smears each subject across the tree. The same +//! assumption is already load-bearing in [`crate::gitlog::History::co_changes`], which +//! skips commits touching more than [`FOCUSED_COMMIT_FILES`] files because "a sweeping +//! commit couples everything to everything and tells us nothing". +//! +//! **Subject→path locality** is the share of those focused commits whose subject shares +//! a word with a path it changed — the direct test of whether the words a change is +//! described in point at the code it touches. OpenMRS is the one measured repository +//! where it falls away, and it is the one whose margin over grep was small. +//! +//! Between them the two account for every measured outcome, without either one having +//! to explain a case it does not fit. +//! +//! # What was tried and dropped +//! +//! Corpus-wide overlap between commit vocabulary and path vocabulary — the obvious +//! reading of "history and file naming speak the same vocabulary" — was measured first +//! and **inverts**: Medusa scores 0.43 against OFBiz's 0.38. Pooling every subject into +//! one bag throws away the attribution that makes the signal mean anything, so it is +//! not computed here. Neither is a 0-100 suitability score: `docs/metrics.md` forbids +//! printing a number that cannot be defined, and a weighted blend of heuristics tuned on +//! four repositories is exactly that. +//! +//! # What this is not +//! +//! A heuristic fitted to four repositories, not a measurement of yours. `reify-bench` +//! measures a specific repository; this reads one in about a second. + +use anyhow::Result; +use std::collections::BTreeSet; +use std::path::Path; + +use crate::concepts::{meaningful_words, stem}; +use crate::discover::{self, Discovery}; +use crate::gitlog; +use crate::model::Lang; + +pub const SCHEMA: &str = "reify.doctor/1"; + +/// The line count below which the README's FAQ already says not to bother. +/// +/// Deliberately the number the documentation publishes — "Under roughly 20k LOC Reify +/// buys you nothing a grep and a scroll wheel don't" — rather than a second, quieter +/// threshold that contradicts it. +pub const LINES_FLOOR: u64 = 20_000; + +/// How far back history is read. +/// +/// Bounded because a doctor that takes a minute does not get run. One `git log +/// --name-only` of a thousand commits answers in well under a second even on a +/// repository with a hundred thousand of them. +pub const MAX_COMMITS: usize = 1_000; + +/// A commit touching more files than this tells you nothing about any of them. +/// +/// The same threshold [`crate::gitlog::History::co_changes`] already applies, for the +/// same reason, so the two agree about what a meaningful commit is. +pub const FOCUSED_COMMIT_FILES: usize = 20; + +/// Share of commits that must be focused for history to be usable evidence. +/// +/// The three measured repositories where Reify won all sit at 0.96 or above; Medusa, +/// where it tied, sits at 0.84. +const FOCUS_OK: f32 = 0.90; + +/// Share of focused commits whose subject must name something in a path it changed. +/// +/// OFBiz 0.80, ERPNext 0.85 and Medusa 0.79 clear it; OpenMRS, whose margin over grep +/// was 9 points rather than 48, sits at 0.48. +const LOCALITY_STRONG: f32 = 0.70; + +/// Enough commits that the shares above can be told apart from their thresholds. +/// +/// Not a round number picked for feel. Medusa — the measured repository that fails the +/// focus test — sits at 0.84. A 95% Wilson interval around 0.84 lies entirely below +/// [`FOCUS_OK`] at n = 200 (upper bound 0.88) but straddles it at n = 50 (upper bound +/// 0.92). Below this a `no` would be an artefact of the sample size, so a short history +/// is reported as short rather than condemned. +const MIN_COMMITS: usize = 200; + +/// How much of the repository will be indexed at all. +#[derive(Debug, Clone, serde::Serialize)] +pub struct Scale { + /// Files Reify would index. + pub indexable_files: usize, + /// Of those, files in a language Reify parses as code. + pub code_files: usize, + /// Lines across every indexable file. Includes blanks and comments. + pub lines: u64, +} + +/// Do the words a change is described in point at the code it touches? +#[derive(Debug, Clone, serde::Serialize)] +pub struct Vocabulary { + /// Focused commits with a usable subject — the denominator. + pub commits_considered: usize, + /// Of those, commits whose subject shares a word with a path they changed. + pub commits_local: usize, + /// `commits_local / commits_considered`. + pub locality: f32, +} + +/// Is the history attributable, or is every subject smeared across the tree? +#[derive(Debug, Clone, serde::Serialize)] +pub struct HistorySignal { + /// Commits read, bounded by [`MAX_COMMITS`]. + pub commits_read: usize, + /// Whether the walk stopped at that bound rather than at the root commit. + pub truncated: bool, + /// Commits whose subject carries at least two meaningful words. + pub usable_subjects: usize, + /// `usable_subjects / commits_read`. Sat at 1.0 on all five repositories this was + /// calibrated against, so it discriminates nothing there — but a history of `wip` + /// and version bumps is real, and this is what would catch it. + pub usable_share: f32, + /// Commits touching at most [`FOCUSED_COMMIT_FILES`] files. + pub focused_commits: usize, + /// `focused_commits / commits_read`. + pub focus: f32, + /// Files changed by the median commit. + pub median_files_changed: usize, +} + +/// Documents whose text a grep cannot reach. +#[derive(Debug, Clone, serde::Serialize)] +pub struct Documents { + /// Files in a format Reify converts and an agent cannot read. + pub unreadable_by_grep: usize, + /// A few examples, so the claim can be checked. + pub examples: Vec, +} + +/// The answer. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Verdict { + /// Under the line floor. Nothing else was worth measuring. + TooSmall, + /// The signals that separated the measured repositories are present here. + LikelyWorthIt, + /// Mixed. Worth measuring rather than guessing. + Marginal, + /// This repository has the shape of the one where Reify tied grep. + UnlikelyToHelp, +} + +impl Verdict { + pub fn as_str(self) -> &'static str { + match self { + Verdict::TooSmall => "TOO SMALL", + Verdict::LikelyWorthIt => "LIKELY WORTH IT", + Verdict::Marginal => "MARGINAL", + Verdict::UnlikelyToHelp => "UNLIKELY TO HELP", + } + } +} + +/// One measured repository, named so a reader can check the comparison. +pub struct Comparable { + pub name: &'static str, + pub outcome: &'static str, + pub report: &'static str, +} + +/// The measured repository where Reify did worst. +/// +/// Pointed at every favourable verdict on purpose: naming a repository where Reify won +/// proves nothing to someone deciding whether to spend an afternoon on it. +pub const MEDUSA: Comparable = Comparable { + name: "Medusa", + outcome: "Reify tied grep — 18% of tasks each", + report: "benchmarks/REPORT-medusa.md", +}; + +/// The measured repository where Reify did best. +pub const OFBIZ: Comparable = Comparable { + name: "OFBiz", + outcome: "Reify reached a changed file on 70% of tasks against grep's 12%", + report: "benchmarks/REPORT-ofbiz.md", +}; + +#[derive(Debug, Clone, serde::Serialize)] +pub struct Diagnosis { + pub schema: &'static str, + pub root: String, + pub scale: Scale, + pub git_repository: bool, + /// Absent below the floor, and when history cannot be read. + pub vocabulary: Option, + /// Absent below the floor, and when history cannot be read. + pub history: Option, + pub documents: Documents, + pub verdict: Verdict, + /// The one sentence carrying the verdict. + pub reason: String, + /// What would change a no or a maybe. Empty for a clear yes. + pub what_would_change_it: Vec, + /// Wall clock of the measurement itself. + pub elapsed_ms: u64, +} + +/// Formats whose text a grep cannot reach. +/// +/// The one categorical advantage: no agent greps a PDF. RTF is nominally text, but its +/// words are broken up by control words, so a grep on it misleads rather than fails — +/// which is worse. +fn unreadable_by_grep(lang: Lang) -> bool { + matches!( + lang, + Lang::Docx | Lang::Doc | Lang::Odt | Lang::Rtf | Lang::Xlsx | Lang::Pptx | Lang::Pdf + ) +} + +/// Does this subject say anything about the change? +/// +/// Two meaningful words is a low bar that "Merge pull request #123 from acme/topic", +/// "Bump version to 4.2.1" and "wip" all fail and that any sentence describing a change +/// passes. Merges are excluded outright: a merge subject names a branch, not a change. +pub fn subject_is_usable(subject: &str) -> bool { + !subject.starts_with("Merge ") && meaningful_words(subject).len() >= 2 +} + +/// Stem-folded words of a string, so `customer` and `customers` are one word. +fn stems(text: &str) -> BTreeSet { + meaningful_words(text) + .iter() + .map(|w| stem(w).to_string()) + .collect() +} + +/// Read the repository and decide. +/// +/// Reads the working tree and `git log`, never the store: the whole point is deciding +/// before committing to the tool, so the answer must not depend on having run it. It +/// also means the answer does not change once `reify index` has run — there is nothing +/// in the store this would rather use. +pub fn diagnose(root: &Path) -> Result { + let started = std::time::Instant::now(); + let found = discover::discover(root)?; + let scale = measure_scale(&found); + let documents = measure_documents(&found); + let git_repository = gitlog::is_repository(root); + + // Below the floor nothing else is worth measuring, and saying so plainly is the + // whole value of the answer. + if scale.lines < LINES_FLOOR { + return Ok(Diagnosis { + schema: SCHEMA, + root: root.display().to_string(), + reason: format!( + "{}. Under roughly {} lines, ripgrep and a scroll wheel do this job. \ + Nothing else here is worth measuring.", + scale_text(&scale), + floor_text() + ), + scale, + git_repository, + vocabulary: None, + history: None, + documents, + verdict: Verdict::TooSmall, + what_would_change_it: Vec::new(), + elapsed_ms: started.elapsed().as_millis() as u64, + }); + } + + // A repository whose history git will not read still gets an answer, with the + // signals honestly absent rather than silently defaulted. + let log = git_repository + .then(|| gitlog::history(root, MAX_COMMITS).ok()) + .flatten(); + let history = log.as_ref().map(measure_history); + let vocabulary = log.as_ref().map(measure_vocabulary); + + let (verdict, reason, what_would_change_it) = + decide(vocabulary.as_ref(), history.as_ref(), &documents); + + Ok(Diagnosis { + schema: SCHEMA, + root: root.display().to_string(), + scale, + git_repository, + vocabulary, + history, + documents, + verdict, + reason, + what_would_change_it, + elapsed_ms: started.elapsed().as_millis() as u64, + }) +} + +fn measure_scale(found: &Discovery) -> Scale { + Scale { + indexable_files: found.files.len(), + code_files: found.files.iter().filter(|f| f.lang.is_code()).count(), + lines: found.files.iter().map(|f| u64::from(f.lines)).sum(), + } +} + +/// Count document formats across everything walked, indexed or not. +/// +/// Both lists are read on purpose: a `.docx` is binary, so discovery records it as +/// skipped even though indexing converts it. Counting only the indexable list would +/// report zero documents for a repository full of them. +fn measure_documents(found: &Discovery) -> Documents { + let mut examples = Vec::new(); + let mut count = 0; + let paths = found + .files + .iter() + .map(|f| f.path.as_str()) + .chain(found.skipped.iter().map(|(p, _)| p.as_str())); + for path in paths { + if unreadable_by_grep(discover::classify(path)) { + count += 1; + if examples.len() < 3 { + examples.push(path.to_string()); + } + } + } + examples.sort(); + Documents { + unreadable_by_grep: count, + examples, + } +} + +fn measure_history(log: &gitlog::History) -> HistorySignal { + let commits_read = log.commits.len(); + let usable = log + .commits + .iter() + .filter(|c| subject_is_usable(&c.subject)) + .count(); + let mut sizes: Vec = log.commits.iter().map(|c| c.files.len()).collect(); + sizes.sort_unstable(); + let focused = sizes + .iter() + .filter(|&&n| (1..=FOCUSED_COMMIT_FILES).contains(&n)) + .count(); + HistorySignal { + commits_read, + truncated: log.truncated, + usable_subjects: usable, + usable_share: share(usable, commits_read), + focused_commits: focused, + focus: share(focused, commits_read), + median_files_changed: sizes.get(sizes.len() / 2).copied().unwrap_or(0), + } +} + +/// How often a commit subject names something in a path that commit changed. +/// +/// Per commit rather than pooled across the repository. The pooled version — every +/// subject word against every path word — was measured on the same four repositories +/// and inverts, because pooling throws away the attribution that makes the question +/// mean anything: it asks whether the words appear *somewhere*, not whether they point +/// at the code that actually changed. +fn measure_vocabulary(log: &gitlog::History) -> Vocabulary { + let mut considered = 0; + let mut local = 0; + for commit in &log.commits { + if !(1..=FOCUSED_COMMIT_FILES).contains(&commit.files.len()) + || !subject_is_usable(&commit.subject) + { + continue; + } + considered += 1; + let subject = stems(&commit.subject); + if commit + .files + .iter() + .any(|path| stems(path).iter().any(|w| subject.contains(w))) + { + local += 1; + } + } + Vocabulary { + commits_considered: considered, + commits_local: local, + locality: share(local, considered), + } +} + +fn share(part: usize, whole: usize) -> f32 { + if whole == 0 { + 0.0 + } else { + part as f32 / whole as f32 + } +} + +/// Below this, a line count is printed exactly rather than abbreviated. +/// +/// Rounding to the nearest thousand is at most a 5% misstatement here and grows worse +/// the smaller the number gets: at 2 lines it is not an abbreviation, it is a wrong +/// answer. In a command whose whole job is honest measurement, that is the one thing it +/// must not do. +const ABBREVIATE_LINES_ABOVE: u64 = 10_000; + +/// A line count, abbreviated only where abbreviating is not misleading. +/// +/// Carries its own `~` when it is approximate, so no caller can mark an exact figure as +/// an estimate or an estimate as exact. +/// +/// Discovery counts lines, not statements: blanks and comments are in there. Above the +/// threshold a figure printed to the unit invites it to be read as a measurement of code +/// size, which it is not. +pub fn lines_text(lines: u64) -> String { + if lines < ABBREVIATE_LINES_ABOVE { + return lines.to_string(); + } + format!("~{}k", (lines as f64 / 1000.0).round() as u64) +} + +/// The line floor, for prose that names it. A threshold, so never marked approximate. +pub fn floor_text() -> String { + format!("{}k", LINES_FLOOR / 1000) +} + +/// `n file` or `n files`. +/// +/// Trivial, and shared rather than inlined so the signal line and the verdict sentence +/// cannot disagree about the same count. +fn count(n: usize, noun: &str) -> String { + if n == 1 { + format!("{n} {noun}") + } else { + format!("{n} {noun}s") + } +} + +/// The scale signal as one phrase, used by both the signal line and the verdict. +/// +/// One function rather than two format strings: they state the same measurement, and the +/// only reason they were ever two was that nobody had noticed the duplication yet. +pub fn scale_text(scale: &Scale) -> String { + format!( + "{}, {} lines", + count(scale.indexable_files, "indexable file"), + lines_text(scale.lines) + ) +} + +/// Pick the verdict, and say what it rests on. +/// +/// Four rules, each traceable to a measured repository: +/// +/// - a history of sweeping commits is the Medusa shape, the one case measured where +/// Reify won nothing; +/// - subjects that do name the code they change is the OFBiz and ERPNext shape, the two +/// large margins; +/// - subjects that do not, over an otherwise focused history, is the OpenMRS shape, +/// where Reify won by 9 points rather than 48; +/// - documents no grep can read are a categorical advantage rather than a comparative +/// one, so they are stated wherever they exist. +fn decide( + vocabulary: Option<&Vocabulary>, + history: Option<&HistorySignal>, + documents: &Documents, +) -> (Verdict, String, Vec) { + let documents_note = format!( + "{} document(s) here hold text no grep can reach, and Reify converts and \ + indexes them", + documents.unreadable_by_grep + ); + + let (Some(vocabulary), Some(history)) = (vocabulary, history) else { + let mut changes = vec![ + "`reify-bench` measures this repository directly, rather than comparing its \ + shape to four others." + .to_string(), + ]; + if documents.unreadable_by_grep == 0 { + changes.push( + "A readable git history. Reify reads commit subjects to connect a change \ + request to code, and without one the strongest signal is missing." + .to_string(), + ); + } + return ( + Verdict::Marginal, + format!( + "No readable git history, so neither measured signal can be computed here.{}", + if documents.unreadable_by_grep > 0 { + format!(" What is clear is that {documents_note}.") + } else { + String::new() + } + ), + changes, + ); + }; + + if history.commits_read < MIN_COMMITS { + return ( + Verdict::Marginal, + format!( + "Only {} commits to read. Both measured signals are shares over commits, \ + and below {MIN_COMMITS} their confidence intervals straddle the \ + thresholds — so a verdict either way would be an artefact of the sample \ + size rather than a reading of this repository.", + history.commits_read + ), + vec![ + format!("More history: at least {MIN_COMMITS} commits."), + "`reify-bench` measures this repository directly, and does not need a \ + long history to do it." + .to_string(), + ], + ); + } + + // The Medusa shape. Reify attaches a commit's vocabulary to every file it touched, + // so a history of sweeping merges spreads each subject across the tree. + if history.focus < FOCUS_OK { + let reason = format!( + "Commits here are sweeping: only {} touch few enough files for their subject \ + to say anything about them, and the median commit changes {} files. Reify \ + attaches a commit's words to every file it touched, so that history is \ + spread too thin to retrieve on. This is the shape of the one measured \ + repository where Reify tied grep.", + percent(history.focus), + history.median_files_changed + ); + if documents.unreadable_by_grep > 0 { + return ( + Verdict::Marginal, + format!("{reason} Against that, {documents_note} — which is an advantage no search tool recovers however well it is used."), + vec![ + "Nothing, for retrieval. The documents are the reason to run it here, \ + not the ranking." + .to_string(), + ], + ); + } + return ( + Verdict::UnlikelyToHelp, + reason, + vec![ + "Smaller commits, whose subject names what they changed. Squashed merges \ + of a hundred files carry no vocabulary any one of them can be found by." + .to_string(), + "Business documents in `.docx`, `.pdf`, `.xlsx` or `.pptx` committed to \ + the tree. Reify reads those and an agent cannot, whatever the history \ + looks like." + .to_string(), + ], + ); + } + + // The OFBiz and ERPNext shape: the words changes are described in name the code. + if vocabulary.locality >= LOCALITY_STRONG { + let mut reason = format!( + "{} of this repository's focused commits have a subject naming something in \ + a path they changed. That agreement between how changes are described and \ + how code is named is what separated the repositories where Reify helped \ + from the one where it did not.", + percent(vocabulary.locality) + ); + if documents.unreadable_by_grep > 0 { + reason.push_str(&format!(" On top of that, {documents_note}.")); + } + return (Verdict::LikelyWorthIt, reason, Vec::new()); + } + + // The OpenMRS shape: an attributable history whose subjects nonetheless describe + // changes in words the code does not use. Measured a modest win, not a large one. + let mut reason = format!( + "History here is attributable, but only {} of its commits have a subject naming \ + something in a path they changed. Of the four measured repositories the one \ + that looked like this beat grep by 9 points rather than 48.", + percent(vocabulary.locality) + ); + if documents.unreadable_by_grep > 0 { + reason.push_str(&format!( + " That said, {documents_note}, which is an advantage no search tool recovers." + )); + return (Verdict::LikelyWorthIt, reason, Vec::new()); + } + ( + Verdict::Marginal, + reason, + vec![ + "Commit subjects that name the thing being changed, in the words the code \ + uses for it. That is the signal Reify's retrieval is built on." + .to_string(), + "A declared glossary — `.reify/glossary.toml` — which bridges the words your \ + team uses to the identifiers the code uses." + .to_string(), + "`reify-bench` measures this repository, rather than comparing its shape to \ + four others." + .to_string(), + ], + ) +} + +pub fn percent(fraction: f32) -> String { + format!("{}%", (fraction * 100.0).round() as i64) +} + +/// The measured repository a verdict should be read against. +pub fn comparable(verdict: Verdict) -> Option { + match verdict { + Verdict::LikelyWorthIt | Verdict::Marginal => Some(MEDUSA), + Verdict::UnlikelyToHelp => Some(OFBIZ), + Verdict::TooSmall => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn tmp(name: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("reify-doctor-{}-{name}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(&d).unwrap(); + d + } + + fn vocab(locality: f32) -> Vocabulary { + Vocabulary { + commits_considered: 400, + commits_local: (400.0 * locality) as usize, + locality, + } + } + + fn history(focus: f32) -> HistorySignal { + HistorySignal { + commits_read: 500, + truncated: true, + usable_subjects: 500, + usable_share: 1.0, + focused_commits: (500.0 * focus) as usize, + focus, + median_files_changed: if focus < FOCUS_OK { 4 } else { 1 }, + } + } + + fn no_documents() -> Documents { + Documents { + unreadable_by_grep: 0, + examples: Vec::new(), + } + } + + #[test] + fn a_tiny_repository_is_told_not_to_bother_and_nothing_else_is_measured() { + let d = tmp("tiny"); + fs::write(d.join("main.py"), "def f():\n return 1\n").unwrap(); + let answer = diagnose(&d).unwrap(); + assert_eq!(answer.verdict, Verdict::TooSmall); + assert!( + answer.vocabulary.is_none() && answer.history.is_none(), + "below the floor nothing else is worth measuring" + ); + assert!(answer.reason.contains("ripgrep")); + assert!( + answer.reason.starts_with("1 indexable file, 2 lines."), + "a two-line repository is reported as two lines, not as ~1k: {}", + answer.reason + ); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn the_floor_is_the_one_the_documentation_publishes() { + // A second, quieter threshold that contradicts the README would be worse than + // no threshold at all. + assert_eq!(LINES_FLOOR, 20_000); + } + + #[test] + fn the_focus_threshold_agrees_with_what_co_change_already_calls_a_sweeping_commit() { + // Two different numbers for "a commit too broad to learn from" would be two + // different definitions of the same thing. + assert_eq!(FOCUSED_COMMIT_FILES, 20); + } + + // The four measured repositories, as they were measured over their newest 500 + // commits. These are the fit; a threshold change that reclassifies one of them is + // a change of claim, not a tweak. + #[test] + fn each_measured_repository_lands_where_its_benchmark_says_it_should() { + let cases = [ + // repo, focus, locality, expected + ("ofbiz +58", 0.956, 0.803, Verdict::LikelyWorthIt), + ("erpnext +48", 0.982, 0.845, Verdict::LikelyWorthIt), + ("openmrs +9", 0.976, 0.480, Verdict::Marginal), + ("medusa +0", 0.836, 0.792, Verdict::UnlikelyToHelp), + ]; + for (name, focus, locality, expected) in cases { + let (verdict, _, _) = decide( + Some(&vocab(locality)), + Some(&history(focus)), + &no_documents(), + ); + assert_eq!(verdict, expected, "{name}"); + } + } + + #[test] + fn a_sweeping_history_is_a_no_and_says_what_would_change_it() { + let (verdict, reason, changes) = + decide(Some(&vocab(0.79)), Some(&history(0.84)), &no_documents()); + assert_eq!( + verdict, + Verdict::UnlikelyToHelp, + "strong subject vocabulary must not rescue a history it cannot be attributed to" + ); + assert!(reason.contains("tied grep")); + assert!(!changes.is_empty(), "a no must say what would change it"); + } + + #[test] + fn documents_no_grep_can_read_are_stated_wherever_they_exist() { + let documents = Documents { + unreadable_by_grep: 12, + examples: vec!["docs/spec.pdf".into()], + }; + // They lift the OpenMRS shape to a yes... + let (verdict, reason, _) = decide(Some(&vocab(0.48)), Some(&history(0.97)), &documents); + assert_eq!(verdict, Verdict::LikelyWorthIt); + assert!(reason.contains("12 document")); + + // ...and they are the reason to bother even where retrieval looks unpromising, + // but they do not turn a sweeping history into a good one. + let (verdict, reason, _) = decide(Some(&vocab(0.79)), Some(&history(0.84)), &documents); + assert_eq!(verdict, Verdict::Marginal); + assert!(reason.contains("tied grep") && reason.contains("12 document")); + } + + #[test] + fn an_unreadable_history_is_marginal_rather_than_a_guess() { + let (verdict, reason, changes) = decide(None, None, &no_documents()); + assert_eq!(verdict, Verdict::Marginal); + assert!(reason.contains("No readable git history")); + assert!(changes.iter().any(|c| c.contains("reify-bench"))); + } + + #[test] + fn too_little_history_is_admitted_rather_than_measured() { + let mut thin = history(0.5); + thin.commits_read = MIN_COMMITS - 1; + let (verdict, reason, changes) = decide(Some(&vocab(0.9)), Some(&thin), &no_documents()); + assert_eq!( + verdict, + Verdict::Marginal, + "a sweeping-looking history too short to measure is reported as short, not \ + condemned: at this sample size the interval straddles the threshold" + ); + assert!(reason.contains(&format!("{} commits", MIN_COMMITS - 1))); + assert!(changes.iter().any(|c| c.contains("More history"))); + } + + #[test] + fn a_history_of_merges_and_version_bumps_does_not_read_as_usable() { + assert!(!subject_is_usable( + "Merge pull request #123 from acme/topic" + )); + assert!(!subject_is_usable("wip")); + assert!(!subject_is_usable("v1.2.3")); + assert!(subject_is_usable( + "fix: sales order approval ignores the credit limit" + )); + } + + #[test] + fn locality_counts_a_subject_that_names_a_path_it_changed() { + let commit = |subject: &str, files: &[&str]| gitlog::Commit { + sha: "0".repeat(40), + timestamp: 0, + author: "a".into(), + subject: subject.into(), + class: gitlog::classify(subject), + files: files.iter().map(|f| f.to_string()).collect(), + }; + let log = gitlog::History { + commits: vec![ + commit("fix invoice rounding", &["app/invoice.py"]), + commit("tighten the release checklist", &["app/invoice.py"]), + // Excluded: too sweeping to attribute either way. + commit("reformat everything", &vec!["f.py"; 40]), + ], + truncated: false, + }; + let measured = measure_vocabulary(&log); + assert_eq!(measured.commits_considered, 2, "the sweep is not counted"); + assert_eq!(measured.commits_local, 1); + assert_eq!(measured.locality, 0.5); + } + + #[test] + fn a_yes_is_pointed_at_the_least_favourable_measured_repository() { + // Naming a repository where Reify won proves nothing to someone deciding + // whether to spend an afternoon on it. + assert_eq!(comparable(Verdict::LikelyWorthIt).unwrap().name, "Medusa"); + assert_eq!(comparable(Verdict::UnlikelyToHelp).unwrap().name, "OFBiz"); + assert!(comparable(Verdict::TooSmall).is_none()); + } + + #[test] + fn documents_are_counted_even_though_discovery_skips_them_as_binary() { + let d = tmp("docs"); + // Real `.docx` bytes are a zip; what matters here is the NUL that makes + // discovery classify it as binary and skip it. + fs::write(d.join("spec.docx"), [0x50, 0x4b, 0x03, 0x04, 0x00, 0x01]).unwrap(); + fs::write(d.join("a.py"), "x = 1\n").unwrap(); + let found = discover::discover(&d).unwrap(); + assert!( + found.files.iter().all(|f| f.path != "spec.docx"), + "the premise of this test: discovery skips it as binary" + ); + assert_eq!(measure_documents(&found).unreadable_by_grep, 1); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn a_line_count_is_abbreviated_only_where_abbreviating_is_not_misleading() { + // Large counts are estimates and say so. + assert_eq!(lines_text(612_345), "~612k"); + assert_eq!(lines_text(10_000), "~10k"); + // Small ones are exact. Rounding 2 lines up to "1k" in a command whose job is + // honest measurement is the one thing it must not do. + assert_eq!(lines_text(9_999), "9999"); + assert_eq!(lines_text(4_200), "4200"); + assert_eq!(lines_text(2), "2"); + assert_eq!(lines_text(0), "0"); + // A threshold is never marked approximate. + assert_eq!(floor_text(), "20k"); + } + + #[test] + fn a_count_of_one_reads_as_one() { + let one = Scale { + indexable_files: 1, + code_files: 1, + lines: 2, + }; + assert_eq!(scale_text(&one), "1 indexable file, 2 lines"); + assert_eq!( + scale_text(&Scale { + indexable_files: 4_178, + code_files: 2_974, + lines: 713_087, + }), + "4178 indexable files, ~713k lines" + ); + } +} diff --git a/crates/reify/src/lib.rs b/crates/reify/src/lib.rs index 87c7b56..5b2d90e 100644 --- a/crates/reify/src/lib.rs +++ b/crates/reify/src/lib.rs @@ -1,6 +1,7 @@ pub mod concepts; pub mod context; pub mod discover; +pub mod doctor; pub mod extract; pub mod gitlog; pub mod index; diff --git a/docs/json-schema/README.md b/docs/json-schema/README.md index 3aa2714..2816cff 100644 --- a/docs/json-schema/README.md +++ b/docs/json-schema/README.md @@ -197,3 +197,50 @@ version. "suggested_command": "string" } ``` + +## `reify doctor --json` + +Answers before there is an index, so it reads the working tree and `git log` rather than +the store. `verdict` is one of `too_small`, `likely_worth_it`, `marginal`, +`unlikely_to_help`. `vocabulary` and `history` are `null` below the line floor and when +git history cannot be read — absent rather than defaulted, so a consumer cannot mistake +"not measured" for "measured zero". Metric definitions: [`../metrics.md`](../metrics.md). + +```json +{ + "schema": "string", + "root": "string", + "scale": { + "indexable_files": "integer", + "code_files": "integer", + "lines": "integer" + }, + "git_repository": "boolean", + "vocabulary": { + "commits_considered": "integer", + "commits_local": "integer", + "locality": "number" + }, + "history": { + "commits_read": "integer", + "truncated": "boolean", + "usable_subjects": "integer", + "usable_share": "number", + "focused_commits": "integer", + "focus": "number", + "median_files_changed": "integer" + }, + "documents": { + "unreadable_by_grep": "integer", + "examples": [ + "string" + ] + }, + "verdict": "string", + "reason": "string", + "what_would_change_it": [ + "string" + ], + "elapsed_ms": "integer" +} +``` diff --git a/docs/json-schema/regenerate.sh b/docs/json-schema/regenerate.sh index a10635e..1d7c259 100755 --- a/docs/json-schema/regenerate.sh +++ b/docs/json-schema/regenerate.sh @@ -9,4 +9,6 @@ reify -C "$REPO" --json context "approval for corporate orders" > /tmp/reify-ctx reify -C "$REPO" --json why "SalesOrder.requires_approval" > /tmp/reify-why.json reify -C "$REPO" --json impact "requires_approval" > /tmp/reify-impact.json reify -C "$REPO" --json preflight "app/order.py" > /tmp/reify-pre.json +# doctor needs no index; point it at a repository with real history instead. +reify -C "$REPO" --json doctor > /tmp/reify-doctor.json echo "Now run the shape extractor in docs/json-schema/ to rebuild README.md" diff --git a/docs/metrics.md b/docs/metrics.md index ef2a899..ff1a5e0 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -20,6 +20,29 @@ liability, not marketing. | **Documented symbols** | Symbols with a docstring or leading comment, over all symbols. | | **Knowledge coverage** | Symbols reachable from at least one concept or document section, over all symbols. Measures how much of the code the knowledge layer can say anything about. | +## `reify doctor` + +Every figure is measured over the working tree and the newest 1000 commits. No index is +read, so these do not change once `reify index` has run. + +| Metric | Definition | +|---|---| +| **Indexable files** | Files `reify index` would parse. The same walk `reify init` reports, so the two agree by construction. | +| **Lines** | Lines across those files, printed rounded to the nearest thousand. Includes blanks and comments — it is a size estimate, not a measure of code. | +| **Commit focus** | Commits touching between 1 and 20 files, over all commits read. 20 is the threshold `gitlog::History::co_changes` already uses to discard a commit as too sweeping to learn from. | +| **Subject→path locality** | Focused commits whose subject shares a stem-folded meaningful word with a path that commit changed, over all focused commits with a usable subject. Per commit, never pooled: the pooled version was measured on the same four repositories and inverts. | +| **Usable subject** | A subject that is not a merge and carries at least two meaningful words. Excludes "Merge pull request #123 from…", "Bump version to 4.2.1" and "wip". | +| **Documents only Reify can read** | Files classified `.docx`, `.doc`, `.odt`, `.rtf`, `.xlsx`, `.pptx` or `.pdf`. Counted across everything walked, indexable or not, because these are binary and discovery records them as skipped. | + +The **verdict** is not a metric and is deliberately not a score. It is a rule over the +figures above, fitted to the four repositories in `benchmarks/REPORT*.md`, and the +command says so in its own output. A 0-100 "suitability" number blended from four +heuristics tuned on four repositories is exactly the false precision this page exists to +forbid. + +Thresholds and their evidence are in the module documentation of +`crates/reify/src/doctor.rs`, next to the code that applies them. + ## Benchmark | Metric | Definition | From 75eb0ffd902c04c9666abe03aa50617d96082699 Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 24 Aug 2026 11:47:09 +0700 Subject: [PATCH 7/8] feat(install): wire the integration each agent here should have MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `reify init --write-agent-instructions` handled one file, chosen from a list four filenames long. `reify install` detects which agents this repository is actually configured for and wires each one in its own format and location, shows its plan before writing, and is fully reversed by `reify uninit`. It installs the shell-command instruction block, not MCP. That is the existing reasoned position in docs/integration/claude-code.md — an MCP server's tool schemas are re-sent on every turn of every session, and for a tool whose purpose is reducing context, a per-turn tax to deliver it is self-defeating — and nothing here overturns it. Every agent this can detect runs shell commands, so level 0 is right for all of them. `--mcp` is the deliberate opt-in and says what it costs before it writes anything. Two rules do most of the work: Detection requires evidence *in the repository*. `~/.cursor` says the user has Cursor installed, not that this repository is worked on with it; creating a `.cursorrules` on that basis is exactly the guess the command must not make. Home directories are read, reported as corroboration, and never acted on alone — an agent seen only there is listed so its absence from the plan is explained. Every detection prints what it rests on. Everything written stays inside the repository. A machine-wide MCP registration cannot be undone by a per-repository `uninit` without breaking every other repository relying on it, so the MCP entries written are the repository-scoped ones and a client with no such config gets the block instead, with the reason stated. Config merging is textual, not a serde round-trip: re-serialising sorts a user's keys and drops their formatting, and the file is theirs. Adding the entry to a hand-written `.mcp.json` changes exactly one line, an unrelated server's env block survives byte for byte, removal restores the original exactly, and the result is parsed before it is written so a bad splice fails loudly. A config that exists but does not parse is reported and skipped — never overwritten. `uninit` derives its removal targets from the same table `install` plans from, so a new agent cannot be added to one without appearing in the other, and a test asserts it. The MCP surface stays at six tools: these are operator commands and belong to the CLI. --- README.md | 7 +- crates/reify-cli/src/install.rs | 947 +++++++++++++++++++++++++++++ crates/reify-cli/src/main.rs | 25 + crates/reify-cli/src/render.rs | 88 +++ crates/reify-cli/src/selfmanage.rs | 91 ++- docs/integration/claude-code.md | 17 +- docs/integration/generic-cli.md | 20 +- docs/json-schema/README.md | 36 ++ docs/json-schema/regenerate.sh | 2 + 9 files changed, 1208 insertions(+), 25 deletions(-) create mode 100644 crates/reify-cli/src/install.rs diff --git a/README.md b/README.md index e7b96a4..2b34b29 100644 --- a/README.md +++ b/README.md @@ -394,8 +394,8 @@ release — through `curl` and `tar` as visible subprocesses, never an embedded client, with the checksum verified before anything is installed; `--check` only asks, and `REIFY_OFFLINE=1` refuses the command outright. `reify uninstall --yes` removes the binary and nothing else; `reify uninit --yes` removes one repository's `.reify/` store -and the instruction block `init` wrote. Both show their plan first when run without -`--yes`. +and every agent integration `init` or `install` wrote. Both show their plan first when +run without `--yes`.
Shell completions @@ -483,9 +483,10 @@ chmod +x .git/hooks/post-merge && cp .git/hooks/post-merge .git/hooks/post-check | `reify report` | System scorecard | | `reify status` | Freshness, coverage, and what was skipped | | `reify doctor` | Should this repository use Reify at all? Runs before there is an index, and will say no | +| `reify install [--yes]` | Detect the agents configured here and wire each one. Shows its plan first; `--mcp` opts into MCP instead | | `reify llm status \| preview` | Is a model configured, and exactly what would be sent | | `reify upgrade [--check]` | Replace this binary with the latest release. The only networked command; refused under `REIFY_OFFLINE=1` | -| `reify uninstall --yes` \| `uninit --yes` | Remove the binary \| one repository's store and instruction block | +| `reify uninstall --yes` \| `uninit --yes` | Remove the binary \| one repository's store and everything `install` wrote | | `reify serve --mcp` | Model Context Protocol over stdio | | `reify completions ` | Completion script | diff --git a/crates/reify-cli/src/install.rs b/crates/reify-cli/src/install.rs new file mode 100644 index 0000000..7dc44e8 --- /dev/null +++ b/crates/reify-cli/src/install.rs @@ -0,0 +1,947 @@ +//! `reify install`: detect the agents that are here, wire each the integration +//! `docs/integration/` recommends for it. +//! +//! # Why this installs a shell command and not an MCP server +//! +//! `docs/integration/claude-code.md` ranks the integrations cheapest first and says to +//! start at level 0, the instruction block: "an MCP server's tool schemas are re-sent on +//! every turn of every session. A CLI costs nothing until it is called. For a tool whose +//! entire purpose is reducing context, paying a per-turn tax to deliver it would be +//! self-defeating." +//! +//! That argument holds, and every agent this command can detect can run a shell command, +//! so level 0 is what gets installed by default. `--mcp` is the deliberate opt-in for +//! the client that cannot, and it says what it costs before it writes anything. +//! +//! # What it will not do +//! +//! **Nothing outside the repository.** The home directory is read as *evidence* that an +//! agent exists — `~/.claude` is Claude Code's real config location — but nothing is +//! written there. A machine-wide MCP registration cannot be undone by a per-repository +//! `reify uninit` without breaking every other repository that relies on it, and an +//! integration that cannot be reversed is one nobody should install. So the MCP entries +//! written here are the repository-scoped ones (`.mcp.json`, `.cursor/mcp.json`); for a +//! client whose only MCP config is machine-wide, the plan says so and writes the +//! instruction block instead. +//! +//! **Nothing it cannot parse.** A config that exists but does not parse is reported and +//! skipped. Overwriting it would be the one failure mode that actually costs somebody +//! their afternoon. +//! +//! **Nothing twice.** Every step checks for its own output first, so a second run is a +//! no-op and says so. + +use anyhow::{Context, Result}; +use std::path::{Path, PathBuf}; + +pub const SCHEMA: &str = "reify.install/1"; + +/// The MCP server entry, written as one line so it disturbs a hand-formatted config as +/// little as possible. +const MCP_ENTRY: &str = r#""reify": { "command": "reify", "args": ["serve", "--mcp"] }"#; + +/// The key under which MCP clients list their servers. +const MCP_SERVERS: &str = "mcpServers"; + +/// Our own key inside it. +const MCP_NAME: &str = "reify"; + +/// A marker that identifies our instruction block wherever it was written. +/// +/// The block itself is [`crate::AGENT_INSTRUCTIONS`]; this is the substring used to +/// recognise it, and it matches what `reify init --write-agent-instructions` already +/// looks for so the two commands never double up on the same file. +const INSTRUCTION_MARKER: &str = "reify context"; + +/// An agent Reify knows how to wire, and the evidence it is here. +struct Known { + name: &'static str, + /// Paths in the repository whose presence is evidence this agent is configured here. + repo_markers: &'static [&'static str], + /// Paths under the user's home that are this agent's real config location. + /// + /// Evidence only. Nothing is ever written to any of them. + home_markers: &'static [&'static str], + /// A rules directory. When it exists, a dedicated file goes in it rather than + /// appending to a shared one — a file of our own is cleanly removable. + rules_dir: Option<(&'static str, &'static str)>, + /// Otherwise, the instruction file the block is appended to. + instruction_file: &'static str, + /// This client's *repository-scoped* MCP config, if it has one. + mcp_config: Option<&'static str>, +} + +/// The agents, and what each one reads. +/// +/// `AGENTS.md` is deliberately its own row rather than evidence for Codex or OpenCode: +/// it is a shared convention that a dozen tools read, and claiming a specific agent is +/// installed because a generic file exists is exactly the guess this command must not +/// make. +const KNOWN: &[Known] = &[ + Known { + name: "Claude Code", + repo_markers: &["CLAUDE.md", ".claude"], + home_markers: &[".claude"], + rules_dir: None, + instruction_file: "CLAUDE.md", + mcp_config: Some(".mcp.json"), + }, + Known { + name: "Cursor", + repo_markers: &[".cursor", ".cursorrules"], + home_markers: &[".cursor"], + rules_dir: Some((".cursor/rules", "reify.mdc")), + instruction_file: ".cursorrules", + mcp_config: Some(".cursor/mcp.json"), + }, + Known { + name: "Windsurf", + repo_markers: &[".windsurf", ".windsurfrules"], + home_markers: &[".codeium/windsurf"], + rules_dir: Some((".windsurf/rules", "reify.md")), + instruction_file: ".windsurfrules", + // Windsurf's MCP config is machine-wide only, so there is nothing repository + // scoped to write. The instruction block is the integration here. + mcp_config: None, + }, + Known { + name: "Cline", + repo_markers: &[".clinerules"], + home_markers: &[], + rules_dir: Some((".clinerules", "reify.md")), + instruction_file: ".clinerules", + mcp_config: None, + }, + Known { + name: "GitHub Copilot", + repo_markers: &[".github/copilot-instructions.md"], + home_markers: &[], + rules_dir: None, + instruction_file: ".github/copilot-instructions.md", + mcp_config: None, + }, + Known { + name: "Codex", + repo_markers: &[".codex"], + home_markers: &[".codex"], + rules_dir: None, + instruction_file: "AGENTS.md", + mcp_config: None, + }, + Known { + name: "OpenCode", + repo_markers: &[".opencode"], + home_markers: &[".config/opencode"], + rules_dir: None, + instruction_file: "AGENTS.md", + mcp_config: None, + }, + Known { + name: "Aider", + repo_markers: &["CONVENTIONS.md", ".aider.conf.yml"], + home_markers: &[".aider.conf.yml"], + rules_dir: None, + instruction_file: "CONVENTIONS.md", + mcp_config: None, + }, + Known { + name: "any agent reading AGENTS.md", + repo_markers: &["AGENTS.md"], + home_markers: &[], + rules_dir: None, + instruction_file: "AGENTS.md", + mcp_config: None, + }, +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Kind { + /// Append the instruction block to a file the agent already reads. + Instructions, + /// Write a dedicated rule file into the agent's rules directory. + RuleFile, + /// Merge a server entry into this client's repository-scoped MCP config. + Mcp, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum State { + /// Not there yet; `--yes` will write it. + Planned, + /// Already there. A second run changes nothing. + AlreadyPresent, + /// The file exists and could not be parsed, so it was left alone. + Skipped, +} + +/// One thing `install` would do, and to what. +#[derive(Debug, Clone, serde::Serialize)] +pub struct Step { + /// Repository-relative, always with `/` separators. + pub path: String, + pub kind: Kind, + /// Every agent this one write serves. More than one when two agents read the + /// same file. + pub agents: Vec, + /// What made Reify think those agents are here. + pub evidence: Vec, + pub state: State, + /// Present only when `state` is `skipped`. + pub problem: Option, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct Plan { + pub schema: &'static str, + pub root: String, + /// Whether MCP was requested, and therefore whether the per-turn cost was accepted. + pub mcp: bool, + /// Whether the plan was applied, or only shown. + pub applied: bool, + pub steps: Vec, + /// The block to paste by hand, present when no agent was recognised. + pub instruction_block: Option, + /// Agents installed on this machine that nothing in this repository configures. + /// + /// Reported rather than acted on: it explains why an agent the user knows they have + /// is not in the plan, without pretending a home directory says anything about this + /// repository. + pub detected_elsewhere: Vec, +} + +impl Plan { + pub fn has_work(&self) -> bool { + self.steps.iter().any(|s| s.state == State::Planned) + } +} + +/// Build the plan without writing anything. +pub fn plan(root: &Path, mcp: bool) -> Result { + plan_with_home(root, mcp, home_dir().as_deref()) +} + +/// The same, with the home directory supplied. +/// +/// Injected rather than read from the environment so the rule that home evidence never +/// triggers a write can be tested without mutating a process-wide variable that every +/// other test in this binary shares. +pub fn plan_with_home(root: &Path, mcp: bool, home: Option<&Path>) -> Result { + let mut steps: Vec = Vec::new(); + + let mut elsewhere: Vec = Vec::new(); + + for agent in KNOWN { + let mut evidence: Vec = agent + .repo_markers + .iter() + .filter(|m| root.join(m).exists()) + .map(|m| format!("{m} is here")) + .collect(); + let at_home: Vec = home + .into_iter() + .flat_map(|home| { + agent + .home_markers + .iter() + .filter(move |m| home.join(m).exists()) + .map(|m| format!("~/{m} exists")) + }) + .collect(); + + // Repository evidence is required before anything is written. `~/.cursor` + // means this user has Cursor installed, not that this repository is worked on + // with it — creating a `.cursorrules` on that basis is the guess this command + // exists to avoid. Home evidence corroborates; it never triggers. + if evidence.is_empty() { + if !at_home.is_empty() { + elsewhere.push(format!("{} ({})", agent.name, at_home.join(", "))); + } + continue; + } + evidence.extend(at_home); + + // MCP is registered *instead of* the instruction block where the client has a + // repository-scoped config: an agent given both pays for the schemas every turn + // and reads instructions telling it to use the CLI anyway. + let target = match (mcp, agent.mcp_config) { + (true, Some(config)) => (config.to_string(), Kind::Mcp), + _ => match agent.rules_dir { + Some((dir, file)) if root.join(dir).is_dir() => { + (format!("{dir}/{file}"), Kind::RuleFile) + } + _ => (agent.instruction_file.to_string(), Kind::Instructions), + }, + }; + + // Two agents reading one file is one write, credited to both. + match steps.iter_mut().find(|s| s.path == target.0) { + Some(existing) => { + existing.agents.push(agent.name.to_string()); + existing.evidence.extend(evidence); + } + None => { + let (state, problem) = inspect(root, &target.0, target.1); + steps.push(Step { + path: target.0, + kind: target.1, + agents: vec![agent.name.to_string()], + evidence, + state, + problem, + }); + } + } + } + steps.sort_by(|a, b| a.path.cmp(&b.path)); + + Ok(Plan { + schema: SCHEMA, + root: root.display().to_string(), + mcp, + applied: false, + instruction_block: steps + .is_empty() + .then(|| crate::AGENT_INSTRUCTIONS.trim().to_string()), + steps, + detected_elsewhere: elsewhere, + }) +} + +/// Is this step already done, or is its target unusable? +fn inspect(root: &Path, rel: &str, kind: Kind) -> (State, Option) { + let path = root.join(rel); + let Ok(text) = std::fs::read_to_string(&path) else { + // Absent is the normal case: it will be created. + return (State::Planned, None); + }; + match kind { + Kind::Instructions | Kind::RuleFile => { + if text.contains(INSTRUCTION_MARKER) { + (State::AlreadyPresent, None) + } else { + (State::Planned, None) + } + } + Kind::Mcp => match serde_json::from_str::(&text) { + Ok(value) => { + if value + .get(MCP_SERVERS) + .and_then(|s| s.get(MCP_NAME)) + .is_some() + { + (State::AlreadyPresent, None) + } else { + (State::Planned, None) + } + } + // Never overwritten. A config that exists but does not parse is somebody's + // work in progress, and clobbering it is the one outcome worth avoiding + // above all others. + Err(e) => ( + State::Skipped, + Some(format!("{rel} is not valid JSON ({e}); left untouched")), + ), + }, + } +} + +/// Apply every planned step. +pub fn apply(root: &Path, plan: &mut Plan) -> Result<()> { + for step in &mut plan.steps { + if step.state != State::Planned { + continue; + } + let path = root.join(&step.path); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating {}", parent.display()))?; + } + match step.kind { + Kind::Instructions => { + let mut text = std::fs::read_to_string(&path).unwrap_or_default(); + if !text.is_empty() && !text.ends_with('\n') { + text.push('\n'); + } + text.push_str(crate::AGENT_INSTRUCTIONS); + std::fs::write(&path, text) + .with_context(|| format!("writing {}", path.display()))?; + } + Kind::RuleFile => { + std::fs::write(&path, rule_file_body(&step.path)) + .with_context(|| format!("writing {}", path.display()))?; + } + Kind::Mcp => { + let before = std::fs::read_to_string(&path).unwrap_or_default(); + let after = with_mcp_entry(&before)?; + std::fs::write(&path, after) + .with_context(|| format!("writing {}", path.display()))?; + } + } + step.state = State::AlreadyPresent; + } + plan.applied = true; + Ok(()) +} + +/// The contents of a dedicated rule file. +/// +/// Cursor's `.mdc` rules need frontmatter to apply to every request; without it the +/// file is written and silently never read, which is worse than not writing it. +fn rule_file_body(rel: &str) -> String { + let block = crate::AGENT_INSTRUCTIONS.trim_start(); + if rel.ends_with(".mdc") { + format!("---\ndescription: Reify\nalwaysApply: true\n---\n\n{block}") + } else { + block.to_string() + } +} + +/// Splice our server entry into an MCP config, preserving everything else byte for byte. +/// +/// Textual rather than a `serde_json` round-trip on purpose. Re-serialising sorts the +/// user's keys, collapses their indentation and drops the shape of a file they wrote by +/// hand — this config is theirs, and the only part of it that should change is the part +/// being added. The result is parsed before it is returned, so a splice that would +/// produce broken JSON fails loudly instead of being written. +pub fn with_mcp_entry(text: &str) -> Result { + if text.trim().is_empty() { + return Ok(format!( + "{{\n \"{MCP_SERVERS}\": {{\n {MCP_ENTRY}\n }}\n}}\n" + )); + } + let parsed: serde_json::Value = + serde_json::from_str(text).context("the existing MCP config is not valid JSON")?; + if parsed + .get(MCP_SERVERS) + .and_then(|s| s.get(MCP_NAME)) + .is_some() + { + return Ok(text.to_string()); + } + + let out = match body_start(text, Some(MCP_SERVERS)) { + // `mcpServers` is there: add one member to it. + Some(at) => splice(text, at, MCP_ENTRY, 4), + // It is not: add the whole key to the root object. + None => { + let at = body_start(text, None) + .context("the existing MCP config has no top-level object")?; + splice( + text, + at, + &format!("\"{MCP_SERVERS}\": {{ {MCP_ENTRY} }}"), + 2, + ) + } + }; + serde_json::from_str::(&out) + .context("adding the server entry would have produced invalid JSON; nothing written")?; + Ok(out) +} + +/// Remove our server entry, leaving everything else byte for byte. +/// +/// Returns `None` when there was nothing to remove. +pub fn without_mcp_entry(text: &str) -> Result> { + let parsed: serde_json::Value = + serde_json::from_str(text).context("the MCP config is not valid JSON")?; + if parsed + .get(MCP_SERVERS) + .and_then(|s| s.get(MCP_NAME)) + .is_none() + { + return Ok(None); + } + let Some(span) = member_span(text, MCP_SERVERS, MCP_NAME) else { + return Ok(None); + }; + let mut out = String::with_capacity(text.len()); + out.push_str(&text[..span.0]); + out.push_str(&text[span.1..]); + serde_json::from_str::(&out) + .context("removing the server entry would have produced invalid JSON; nothing written")?; + Ok(Some(out)) +} + +/// Insert `member` just inside an object whose body starts at `at`. +fn splice(text: &str, at: usize, member: &str, indent: usize) -> String { + let rest = &text[at..]; + let empty = rest.trim_start().starts_with('}'); + let pad = " ".repeat(indent); + let mut out = String::with_capacity(text.len() + member.len() + 8); + out.push_str(&text[..at]); + out.push('\n'); + out.push_str(&pad); + out.push_str(member); + if !empty { + out.push(','); + } + // An object that was `{}` gets its closing brace put on its own line; one that + // already had members keeps whatever the author wrote after the brace. + if empty { + out.push('\n'); + out.push_str(&" ".repeat(indent.saturating_sub(2))); + out.push_str(rest.trim_start()); + } else { + out.push_str(rest); + } + out +} + +/// Byte offset just past the `{` opening the root object, or the object at top-level +/// `key`. +/// +/// A small scanner rather than a JSON library: the caller has already parsed the text +/// for validity, and what is needed here is a *position in the original bytes*, which no +/// parse tree carries. +fn body_start(text: &str, key: Option<&str>) -> Option { + let bytes = text.as_bytes(); + let mut depth = 0usize; + let mut in_string = false; + let mut escaped = false; + let mut opened_at = 0usize; + let mut awaiting = false; + + for (i, &c) in bytes.iter().enumerate() { + if in_string { + if escaped { + escaped = false; + } else if c == b'\\' { + escaped = true; + } else if c == b'"' { + in_string = false; + // A key sits at depth 1 — inside the root object — and is followed by + // a colon. Anything else and this was a value that happened to match. + if depth == 1 && key.is_some_and(|k| &text[opened_at + 1..i] == k) { + awaiting = true; + } + } + continue; + } + if awaiting && !c.is_ascii_whitespace() && c != b':' && c != b'{' { + awaiting = false; + } + match c { + b'"' => { + in_string = true; + opened_at = i; + } + b'{' => { + depth += 1; + if awaiting { + return Some(i + 1); + } + if key.is_none() && depth == 1 { + return Some(i + 1); + } + } + b'}' | b']' => depth = depth.saturating_sub(1), + b'[' => depth += 1, + _ => {} + } + } + None +} + +/// The byte range covering `parent.member` and the comma that separates it from its +/// neighbours, so cutting it leaves valid JSON. +fn member_span(text: &str, parent: &str, member: &str) -> Option<(usize, usize)> { + let body = body_start(text, Some(parent))?; + let bytes = text.as_bytes(); + let mut i = body; + let mut depth = 0usize; + let mut in_string = false; + let mut escaped = false; + let mut key_at: Option<(usize, usize)> = None; + let mut quote_at = 0usize; + + while i < bytes.len() { + let c = bytes[i]; + if in_string { + if escaped { + escaped = false; + } else if c == b'\\' { + escaped = true; + } else if c == b'"' { + in_string = false; + if depth == 0 && &text[quote_at + 1..i] == member { + key_at = Some((quote_at, i)); + } + } + i += 1; + continue; + } + match c { + b'"' => { + in_string = true; + quote_at = i; + } + b'{' | b'[' => depth += 1, + b'}' | b']' => { + if depth == 0 { + return None; // end of the parent object, member not found + } + depth -= 1; + // A value that has just closed at the parent's own level ends the + // member we were tracking. + if depth == 0 { + if let Some((start, _)) = key_at { + return Some(widen(text, start, i + 1)); + } + } + } + b',' if depth == 0 => { + if let Some((start, _)) = key_at { + return Some(widen(text, start, i + 1)); + } + } + _ => {} + } + i += 1; + } + None +} + +/// Widen a member's range to swallow one separating comma and the whitespace around it. +/// +/// Without this, removing the last member of an object leaves a trailing comma, which is +/// not valid JSON. +fn widen(text: &str, start: usize, end: usize) -> (usize, usize) { + let bytes = text.as_bytes(); + let mut end = end; + // A comma after the member: take it, plus the newline it sat on. + let mut probe = end; + while probe < bytes.len() && bytes[probe].is_ascii_whitespace() && bytes[probe] != b'\n' { + probe += 1; + } + if probe < bytes.len() && bytes[probe] == b',' { + end = probe + 1; + } else { + // No comma after, so this was the last member: take the one before it instead. + let mut back = start; + while back > 0 && bytes[back - 1].is_ascii_whitespace() { + back -= 1; + } + if back > 0 && bytes[back - 1] == b',' { + return (back - 1, end); + } + } + // Leading whitespace on the member's own line goes with it. + let mut begin = start; + while begin > 0 && (bytes[begin - 1] == b' ' || bytes[begin - 1] == b'\t') { + begin -= 1; + } + if begin > 0 && bytes[begin - 1] == b'\n' { + begin -= 1; + } + (begin, end) +} + +/// Every repository path `install` could ever write to, for `uninit` to undo. +/// +/// Derived from the same table `plan` walks, so a new agent cannot be added to one +/// without appearing in the other. +pub fn removable_targets() -> Vec<(String, Kind)> { + let mut out: Vec<(String, Kind)> = Vec::new(); + for agent in KNOWN { + out.push((agent.instruction_file.to_string(), Kind::Instructions)); + if let Some((dir, file)) = agent.rules_dir { + out.push((format!("{dir}/{file}"), Kind::RuleFile)); + } + if let Some(config) = agent.mcp_config { + out.push((config.to_string(), Kind::Mcp)); + } + } + out.sort_by(|a, b| a.0.cmp(&b.0)); + out.dedup_by(|a, b| a.0 == b.0); + out +} + +fn home_dir() -> Option { + std::env::var_os("HOME") + .or_else(|| std::env::var_os("USERPROFILE")) + .filter(|h| !h.is_empty()) + .map(PathBuf::from) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn tmp(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("reify-install-{}-{name}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(&d).unwrap(); + d + } + + /// A repository with a `CLAUDE.md` and nothing else Reify recognises. + fn claude_repo(name: &str) -> PathBuf { + let d = tmp(name); + fs::write(d.join("CLAUDE.md"), "# My project\n\nSome house rules.\n").unwrap(); + d + } + + #[test] + fn a_claude_repository_gets_the_shell_integration_not_mcp() { + // The documented position: level 0 first, because MCP schemas cost tokens on + // every turn of every session. + let d = claude_repo("level0"); + let plan = plan_with_home(&d, false, None).unwrap(); + assert_eq!(plan.steps.len(), 1); + assert_eq!(plan.steps[0].path, "CLAUDE.md"); + assert_eq!(plan.steps[0].kind, Kind::Instructions); + assert_eq!(plan.steps[0].agents, vec!["Claude Code"]); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn mcp_is_the_deliberate_opt_in_and_replaces_the_block_rather_than_joining_it() { + let d = claude_repo("mcpoptin"); + let plan = plan_with_home(&d, true, None).unwrap(); + assert_eq!(plan.steps.len(), 1, "one integration per agent, not two"); + assert_eq!(plan.steps[0].path, ".mcp.json"); + assert_eq!(plan.steps[0].kind, Kind::Mcp); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn planning_writes_nothing_and_applying_is_idempotent() { + let d = claude_repo("idempotent"); + let before = fs::read_to_string(d.join("CLAUDE.md")).unwrap(); + + let mut p = plan_with_home(&d, false, None).unwrap(); + assert!(p.has_work()); + assert_eq!( + fs::read_to_string(d.join("CLAUDE.md")).unwrap(), + before, + "a plan must not write" + ); + + apply(&d, &mut p).unwrap(); + let after = fs::read_to_string(d.join("CLAUDE.md")).unwrap(); + assert!(after.starts_with(&before), "the user's content is kept"); + assert!(after.contains("reify context")); + + let again = plan_with_home(&d, false, None).unwrap(); + assert!(!again.has_work(), "a second run has nothing to do"); + let mut again = again; + apply(&d, &mut again).unwrap(); + assert_eq!( + fs::read_to_string(d.join("CLAUDE.md")).unwrap(), + after, + "applying a no-op plan changes nothing" + ); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn a_rules_directory_gets_its_own_file_rather_than_a_shared_one() { + let d = tmp("rulesdir"); + fs::create_dir_all(d.join(".cursor/rules")).unwrap(); + let p = plan_with_home(&d, false, None).unwrap(); + let step = p.steps.iter().find(|s| s.kind == Kind::RuleFile).unwrap(); + assert_eq!(step.path, ".cursor/rules/reify.mdc"); + let mut p = p; + apply(&d, &mut p).unwrap(); + let body = fs::read_to_string(d.join(".cursor/rules/reify.mdc")).unwrap(); + assert!( + body.starts_with("---\n") && body.contains("alwaysApply: true"), + "a Cursor rule without frontmatter is written and never read" + ); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn two_agents_reading_one_file_are_one_write_credited_to_both() { + let d = tmp("shared"); + fs::write(d.join("AGENTS.md"), "# rules\n").unwrap(); + fs::create_dir_all(d.join(".codex")).unwrap(); + let p = plan_with_home(&d, false, None).unwrap(); + let step = p.steps.iter().find(|s| s.path == "AGENTS.md").unwrap(); + assert!(step.agents.len() >= 2, "{:?}", step.agents); + assert_eq!( + p.steps.iter().filter(|s| s.path == "AGENTS.md").count(), + 1, + "one file, one write" + ); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn an_agent_installed_on_this_machine_but_not_in_this_repository_is_never_written_for() { + // `~/.cursor` means the user has Cursor, not that this repository is worked on + // with it. Creating a `.cursorrules` on that evidence is the guess this command + // exists to avoid — so it is reported instead, which also explains the silence. + let d = tmp("homeonly"); + fs::write(d.join("main.rs"), "fn main() {}").unwrap(); + let home = tmp("fakehome"); + fs::create_dir_all(home.join(".cursor")).unwrap(); + + let p = plan_with_home(&d, false, Some(&home)).unwrap(); + assert!(p.steps.is_empty(), "{:?}", p.steps); + assert!(p.detected_elsewhere.iter().any(|a| a.starts_with("Cursor"))); + assert!(p.instruction_block.is_some(), "hand over the block instead"); + assert!(!d.join(".cursorrules").exists()); + let _ = fs::remove_dir_all(&d); + let _ = fs::remove_dir_all(&home); + } + + #[test] + fn home_evidence_corroborates_repository_evidence_rather_than_replacing_it() { + let d = claude_repo("corroborate"); + let home = tmp("fakehome2"); + fs::create_dir_all(home.join(".claude")).unwrap(); + let p = plan_with_home(&d, false, Some(&home)).unwrap(); + assert_eq!(p.steps.len(), 1); + assert_eq!( + p.steps[0].evidence, + vec!["CLAUDE.md is here", "~/.claude exists"], + "both are stated, so the detection can be checked" + ); + let _ = fs::remove_dir_all(&d); + let _ = fs::remove_dir_all(&home); + } + + #[test] + fn nothing_recognised_offers_the_block_to_paste_rather_than_guessing() { + let d = tmp("unknown"); + fs::write(d.join("main.rs"), "fn main() {}").unwrap(); + let p = plan_with_home(&d, false, None).unwrap(); + assert!(p.steps.is_empty()); + assert!(p.instruction_block.unwrap().contains("reify context")); + let _ = fs::remove_dir_all(&d); + } + + // The requirement most likely to break somebody's setup. Everything the user wrote + // must survive, byte for byte, apart from the entry being added. + #[test] + fn an_existing_mcp_config_survives_byte_identical_apart_from_the_added_entry() { + let original = r#"{ + "mcpServers": { + "postgres": { + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-postgres", "postgres://localhost/db"], + "env": { "PGPASSWORD": "hunter2" } + } + }, + "somethingElse": [1, 2, 3] +} +"#; + let updated = with_mcp_entry(original).unwrap(); + + // The added line, and nothing else. + let removed: Vec<&str> = original + .lines() + .filter(|l| !updated.lines().any(|u| u == *l)) + .collect(); + assert!(removed.is_empty(), "lines disappeared: {removed:?}"); + let added: Vec<&str> = updated + .lines() + .filter(|l| !original.lines().any(|o| o == *l)) + .collect(); + assert_eq!(added.len(), 1, "expected exactly one added line: {added:?}"); + assert!(added[0].contains("\"reify\"")); + + // And the user's own content is intact when read back. + let after: serde_json::Value = serde_json::from_str(&updated).unwrap(); + let before: serde_json::Value = serde_json::from_str(original).unwrap(); + assert_eq!(after["somethingElse"], before["somethingElse"]); + assert_eq!( + after[MCP_SERVERS]["postgres"], before[MCP_SERVERS]["postgres"], + "an unrelated server must survive exactly" + ); + assert!(after[MCP_SERVERS][MCP_NAME]["command"] == "reify"); + + // Removing it puts the file back the way it was. + let restored = without_mcp_entry(&updated).unwrap().unwrap(); + assert_eq!(restored, original, "uninstalling must be a clean reversal"); + } + + #[test] + fn an_empty_or_absent_config_is_created_whole() { + let fresh = with_mcp_entry("").unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&fresh).unwrap(); + assert_eq!(parsed[MCP_SERVERS][MCP_NAME]["command"], "reify"); + + let empty_object = with_mcp_entry("{}\n").unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&empty_object).unwrap(); + assert_eq!(parsed[MCP_SERVERS][MCP_NAME]["args"][0], "serve"); + + let no_servers_key = with_mcp_entry("{\n \"other\": true\n}\n").unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&no_servers_key).unwrap(); + assert_eq!(parsed["other"], true); + assert_eq!(parsed[MCP_SERVERS][MCP_NAME]["command"], "reify"); + + let empty_servers = with_mcp_entry("{\n \"mcpServers\": {}\n}\n").unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&empty_servers).unwrap(); + assert_eq!(parsed[MCP_SERVERS][MCP_NAME]["command"], "reify"); + } + + #[test] + fn adding_an_entry_twice_changes_nothing() { + let once = with_mcp_entry("{\n \"mcpServers\": {}\n}\n").unwrap(); + assert_eq!(with_mcp_entry(&once).unwrap(), once); + } + + #[test] + fn a_config_that_does_not_parse_is_reported_and_left_alone() { + let d = tmp("broken"); + fs::write(d.join("CLAUDE.md"), "# x\n").unwrap(); + let broken = "{ \"mcpServers\": { oops }"; + fs::write(d.join(".mcp.json"), broken).unwrap(); + + let mut p = plan_with_home(&d, true, None).unwrap(); + let step = &p.steps[0]; + assert_eq!(step.state, State::Skipped); + assert!(step.problem.as_ref().unwrap().contains("not valid JSON")); + + apply(&d, &mut p).unwrap(); + assert_eq!( + fs::read_to_string(d.join(".mcp.json")).unwrap(), + broken, + "an unparsable config is never overwritten" + ); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn a_string_that_merely_looks_like_the_servers_key_is_not_mistaken_for_it() { + // `body_start` scans bytes, so it has to tell a key from a value that happens + // to spell the same thing. + let text = "{\n \"note\": \"mcpServers\",\n \"mcpServers\": {\n \"a\": {}\n }\n}\n"; + let updated = with_mcp_entry(text).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&updated).unwrap(); + assert_eq!(parsed["note"], "mcpServers"); + assert!(parsed[MCP_SERVERS]["a"].is_object()); + assert_eq!(parsed[MCP_SERVERS][MCP_NAME]["command"], "reify"); + } + + #[test] + fn removing_the_only_entry_leaves_valid_json() { + let text = with_mcp_entry("{}").unwrap(); + let stripped = without_mcp_entry(&text).unwrap().unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&stripped).unwrap(); + assert!(parsed[MCP_SERVERS].get(MCP_NAME).is_none()); + assert!(without_mcp_entry(&stripped).unwrap().is_none()); + } + + #[test] + fn every_agent_in_the_table_is_reachable_by_uninit() { + // A new agent added to KNOWN without a removal path would leave orphans. + let targets = removable_targets(); + for agent in KNOWN { + assert!( + targets.iter().any(|(p, _)| p == agent.instruction_file + || agent + .rules_dir + .is_some_and(|(d, f)| *p == format!("{d}/{f}"))), + "{} has no removal path", + agent.name + ); + } + } +} diff --git a/crates/reify-cli/src/main.rs b/crates/reify-cli/src/main.rs index 0852098..a40e04a 100644 --- a/crates/reify-cli/src/main.rs +++ b/crates/reify-cli/src/main.rs @@ -5,6 +5,7 @@ //! no network call at all in this build, which is asserted by a test rather than //! promised in a README. +mod install; mod mcp; mod render; mod selfmanage; @@ -141,6 +142,22 @@ enum Command { path: String, }, + /// Detect the agents present here and wire each the integration it should have. + /// + /// Shows the plan and stops, unless `--yes`. Reversible with `reify uninit`. + Install { + /// Actually write it; without this flag, only the plan is shown. + #[arg(long)] + yes: bool, + /// Register the MCP server instead of the shell-command instruction block. + /// + /// `docs/integration/` recommends the instruction block: an MCP server's tool + /// schemas are re-sent every turn of every session, and a CLI costs nothing + /// until it is called. Use this for a client that cannot run a shell command. + #[arg(long)] + mcp: bool, + }, + /// Should this repository use Reify at all? Answers before you index. /// /// Runs against the working tree and `git log`, never the store, so it works @@ -314,6 +331,13 @@ fn run() -> Result<()> { let store = open_existing(&root)?; render::preflight(&query::preflight(&store, path)?, cli.json) } + Command::Install { yes, mcp } => { + let mut plan = install::plan(&root, *mcp)?; + if *yes && plan.has_work() { + install::apply(&root, &mut plan)?; + } + render::install(&plan, *yes, cli.json) + } Command::Doctor => render::doctor(&reify::doctor::diagnose(&root)?, cli.json), Command::Llm { action } => match action { LlmAction::Status => render::llm_status(&root, cli.json), @@ -558,6 +582,7 @@ mod tests { vec!["reify", "--json", "llm", "status"], vec!["reify", "--json", "init"], vec!["reify", "--json", "doctor"], + vec!["reify", "--json", "install"], ] { let cli = Cli::try_parse_from(&args).expect("should parse"); assert!(cli.json, "{args:?}"); diff --git a/crates/reify-cli/src/render.rs b/crates/reify-cli/src/render.rs index 796ab16..c5f5607 100644 --- a/crates/reify-cli/src/render.rs +++ b/crates/reify-cli/src/render.rs @@ -20,6 +20,8 @@ use serde::Serialize; use reify::context::Context; use reify::discover::Discovery; use reify::doctor::{self, Diagnosis, Verdict}; + +use crate::install::{Kind as InstallKind, Plan, State as InstallState, Step as InstallStep}; use reify::index::IndexReport; use reify::llm; use reify::model::{Node, Status}; @@ -760,6 +762,92 @@ pub fn preflight(answer: &Preflight, json: bool) -> Result<()> { Ok(()) } +/// `reify install`: what was found, and what will be done about it. +/// +/// Shows the plan and stops unless `--yes`, matching `uninstall`, `uninit` and +/// `upgrade`. A command that changes a repository's agent configuration without showing +/// its work first is one people learn not to run. +pub fn install(plan: &Plan, yes: bool, json: bool) -> Result<()> { + if json { + return emit_json(plan); + } + println!("INSTALL {}", plan.root); + + if plan.steps.is_empty() { + // Guessing which agent is present from a directory name that might mean + // anything is worse than handing over the block and letting a human place it. + println!("\nFound no agent I recognise here."); + println!("Nothing was written. Add this to whatever instruction file your tool reads:\n"); + for line in plan.instruction_block.iter().flat_map(|b| b.lines()) { + println!(" {line}"); + } + return Ok(()); + } + + println!(); + for step in &plan.steps { + println!(" {}", step.agents.join(", ")); + // Detection has to be checkable, so what the claim rests on is printed with it. + println!(" detected because {}", step.evidence.join(", ")); + println!(" {}", install_action(step, plan.applied)); + } + + if !plan.detected_elsewhere.is_empty() { + println!("\n Installed on this machine but not configured in this repository,"); + println!(" so nothing was planned for them:"); + for agent in &plan.detected_elsewhere { + println!(" {agent}"); + } + } + + if plan.mcp { + println!( + "\n {}", + wrap( + "You asked for MCP. Its tool schemas are re-sent on every turn of every \ + session, where the shell-command block costs nothing until it is \ + called — see docs/integration/claude-code.md. Drop --mcp for the \ + cheaper integration.", + WIDTH, + " ", + 2, + ) + ); + } + + if !plan.has_work() && !plan.applied { + println!("\nNothing to do; everything above is already wired."); + return Ok(()); + } + if !yes { + println!("\nNothing was written. Re-run with --yes to apply."); + return Ok(()); + } + println!("\nDone. `reify uninit` removes everything written here."); + Ok(()) +} + +/// What one step will do, has done, or is not doing — as one readable phrase. +fn install_action(step: &InstallStep, applied: bool) -> String { + if step.state == InstallState::Skipped { + return match &step.problem { + Some(problem) => format!("skipping {}: {problem}", step.path), + None => format!("skipping {}", step.path), + }; + } + let what = match step.kind { + InstallKind::Mcp => format!("the MCP server entry in {}", step.path), + InstallKind::RuleFile => format!("the rule file {}", step.path), + InstallKind::Instructions => format!("the instruction block in {}", step.path), + }; + match (step.state, applied) { + (InstallState::Planned, _) => format!("will write {what}"), + (InstallState::AlreadyPresent, true) => format!("wrote {what}"), + (InstallState::AlreadyPresent, false) => format!("already has {what}"), + (InstallState::Skipped, _) => unreachable!("handled above"), + } +} + /// `reify doctor`: should this repository use Reify at all? /// /// Named signals with measured values and a plain-language verdict. Deliberately not a diff --git a/crates/reify-cli/src/selfmanage.rs b/crates/reify-cli/src/selfmanage.rs index c096579..81b8088 100644 --- a/crates/reify-cli/src/selfmanage.rs +++ b/crates/reify-cli/src/selfmanage.rs @@ -225,25 +225,58 @@ pub fn uninstall(yes: bool) -> Result<()> { Ok(()) } -/// `reify uninit [--yes]`: remove this repository's store and instruction block. +/// `reify uninit [--yes]`: remove this repository's store and everything +/// `reify install` wrote. +/// +/// The removal targets are derived from the same table `install` plans from, so an agent +/// cannot be added to one without appearing in the other — otherwise `install` quietly +/// leaves orphans that only turn up when somebody wonders why their agent still mentions +/// a tool they removed. pub fn uninit(root: &Path, yes: bool) -> Result<()> { let store = root.join(reify::index::REIFY_DIR); let mut planned: Vec = Vec::new(); - if store.is_dir() { - planned.push(format!("remove {}", store.display())); - } - let mut instruction_files: Vec = Vec::new(); - for name in ["AGENTS.md", "CLAUDE.md"] { - let path = root.join(name); - if let Ok(text) = std::fs::read_to_string(&path) { - if text.contains(crate::AGENT_INSTRUCTIONS) { - planned.push(format!("strip the Reify instruction block from {name}")); - instruction_files.push(path); + let mut edits: Vec = Vec::new(); + + for (rel, kind) in crate::install::removable_targets() { + let path = root.join(&rel); + let Ok(text) = std::fs::read_to_string(&path) else { + continue; + }; + match kind { + crate::install::Kind::Mcp => match crate::install::without_mcp_entry(&text) { + Ok(Some(stripped)) => { + planned.push(format!("remove the reify server entry from {rel}")); + edits.push(Removal::Rewrite(path, stripped)); + } + Ok(None) => {} + // An unparsable config is left exactly as it is, on the way out as much + // as on the way in. + Err(e) => planned.push(format!("leave {rel} alone ({e})")), + }, + _ if !text.contains(crate::AGENT_INSTRUCTIONS.trim()) => {} + crate::install::Kind::RuleFile => { + planned.push(format!("remove {rel}")); + edits.push(Removal::Delete(path)); + } + crate::install::Kind::Instructions => { + let stripped = strip_block(&text); + if stripped.trim().is_empty() { + // Nothing but our own block was ever in it. + planned.push(format!("remove {rel}")); + edits.push(Removal::Delete(path)); + } else { + planned.push(format!("strip the Reify instruction block from {rel}")); + edits.push(Removal::Rewrite(path, stripped)); + } } } } + + if store.is_dir() { + planned.push(format!("remove {}", store.display())); + } if planned.is_empty() { - println!("nothing to remove: no `.reify/` store or instruction block here"); + println!("nothing to remove: no `.reify/` store or Reify integration here"); return Ok(()); } for step in &planned { @@ -254,10 +287,13 @@ pub fn uninit(root: &Path, yes: bool) -> Result<()> { println!("Nothing was removed. Re-run with --yes to apply."); return Ok(()); } - for path in instruction_files { - let text = std::fs::read_to_string(&path)?; - std::fs::write(&path, text.replace(crate::AGENT_INSTRUCTIONS, "")) - .with_context(|| format!("rewriting {}", path.display()))?; + for edit in edits { + match edit { + Removal::Rewrite(path, text) => std::fs::write(&path, text) + .with_context(|| format!("rewriting {}", path.display()))?, + Removal::Delete(path) => std::fs::remove_file(&path) + .with_context(|| format!("removing {}", path.display()))?, + } } if store.is_dir() { std::fs::remove_dir_all(&store).with_context(|| format!("removing {}", store.display()))?; @@ -266,6 +302,29 @@ pub fn uninit(root: &Path, yes: bool) -> Result<()> { Ok(()) } +enum Removal { + Rewrite(PathBuf, String), + Delete(PathBuf), +} + +/// Take our block out of a file somebody else also writes to. +/// +/// The full constant is tried first, and it carries the newline that `install` and +/// `init` push in front of it — so removing it restores the file byte for byte rather +/// than leaving the blank lines the block was separated by. The trimmed form is the +/// fallback, for a file where somebody pasted the block by hand. +fn strip_block(text: &str) -> String { + for block in [crate::AGENT_INSTRUCTIONS, crate::AGENT_INSTRUCTIONS.trim()] { + if let Some(at) = text.find(block) { + let mut out = String::with_capacity(text.len()); + out.push_str(&text[..at]); + out.push_str(&text[at + block.len()..]); + return out; + } + } + text.to_string() +} + #[cfg(test)] mod tests { use super::*; diff --git a/docs/integration/claude-code.md b/docs/integration/claude-code.md index 0db37c2..72210fb 100644 --- a/docs/integration/claude-code.md +++ b/docs/integration/claude-code.md @@ -3,10 +3,15 @@ Four levels, cheapest first. **Start at level 0** — it works today, costs nothing until used, and is what the benchmark measured. +`reify install` picks the right level for whichever agents this repository is actually +configured for and shows its plan before writing anything; `reify install --yes` applies +it. It installs level 0 by default, for the reason below. `reify uninit` removes +everything it wrote. + ## Level 0 — a shell command (recommended) -`reify init` finds your `AGENTS.md` or `CLAUDE.md` and tells you what to add. -`reify init --write-agent-instructions` appends it for you: +`reify install` writes it into whatever files the agents here already read. +`reify init --write-agent-instructions` appends it to one file: ```markdown ## Before changing code in this repo @@ -25,9 +30,15 @@ is reducing context, paying a per-turn tax to deliver it would be self-defeating ## Level 1 — MCP, if your client cannot run a shell command ```bash -reify serve --mcp +reify install --mcp --yes # merges the server entry into .mcp.json +reify serve --mcp # or register it by hand ``` +`--mcp` is an opt-in to the per-turn cost above, and `install` says so before writing. +It merges into an existing `.mcp.json` rather than replacing it: unrelated servers, their +environment blocks and the file's formatting survive untouched, and a config that does +not parse is reported and skipped rather than overwritten. + Six tools — `reify_context`, `reify_why`, `reify_impact`, `reify_explain`, `reify_flow`, `reify_conflicts` — and that is the whole surface, deliberately. `mcp::tests::the_tool_schemas_stay_small_enough_to_be_worth_sending` asserts the diff --git a/docs/integration/generic-cli.md b/docs/integration/generic-cli.md index a75c6a4..ce4c780 100644 --- a/docs/integration/generic-cli.md +++ b/docs/integration/generic-cli.md @@ -38,7 +38,21 @@ Claims marked INFERRED are leads to verify, not facts. If `conflicts` is non-empty, resolve the disagreement before changing behaviour. ``` -## Codex, Cursor, OpenCode, Aider, Pi +## Codex, Cursor, Windsurf, Cline, Copilot, OpenCode, Aider, Pi -No adapter needed. Put the block above in whatever instruction file the tool reads -(`AGENTS.md`, `.cursorrules`, `CONVENTIONS.md`). The CLI is the interface. +No adapter needed. `reify install` finds which of these this repository is configured for +and writes the block into each one's own file — a dedicated rule file where the tool has +a rules directory (`.cursor/rules/`, `.windsurf/rules/`, `.clinerules/`), an append where +it reads a single file (`AGENTS.md`, `.cursorrules`, `.github/copilot-instructions.md`, +`CONVENTIONS.md`). It shows the plan and stops unless `--yes`. + +Detection needs evidence **in the repository**. A tool installed on your machine but not +configured here is listed and left alone: `~/.cursor` says you have Cursor, not that this +repository is worked on with it, and creating a `.cursorrules` on that basis would be a +guess. Where nothing is recognised, `install` prints the block for you to place. + +Everything it writes is inside the repository, so `reify uninit` reverses all of it. That +is also why no machine-wide MCP config is touched: a per-repository uninstall cannot +safely undo a machine-wide registration. + +The CLI is the interface; put the block above anywhere yourself if you prefer. diff --git a/docs/json-schema/README.md b/docs/json-schema/README.md index 2816cff..c2fd8cc 100644 --- a/docs/json-schema/README.md +++ b/docs/json-schema/README.md @@ -244,3 +244,39 @@ git history cannot be read — absent rather than defaulted, so a consumer canno "elapsed_ms": "integer" } ``` + +## `reify install --json` + +The plan, whether or not it was applied. `applied` is false unless `--yes` was passed. +`kind` is one of `instructions`, `rule_file`, `mcp`; `state` is one of `planned`, +`already_present`, `skipped`. `problem` is non-null only when `state` is `skipped`, and +says why the file was left alone. `evidence` is what each detection rests on, so a +consumer can check the claim rather than trust it. `instruction_block` is non-null only +when no agent was recognised — it is the text to paste by hand. + +```json +{ + "schema": "string", + "root": "string", + "mcp": "boolean", + "applied": "boolean", + "steps": [ + { + "path": "string", + "kind": "string", + "agents": [ + "string" + ], + "evidence": [ + "string" + ], + "state": "string", + "problem": "null" + } + ], + "instruction_block": "null", + "detected_elsewhere": [ + "string" + ] +} +``` diff --git a/docs/json-schema/regenerate.sh b/docs/json-schema/regenerate.sh index 1d7c259..76e8914 100755 --- a/docs/json-schema/regenerate.sh +++ b/docs/json-schema/regenerate.sh @@ -11,4 +11,6 @@ reify -C "$REPO" --json impact "requires_approval" > /tmp/reify-im reify -C "$REPO" --json preflight "app/order.py" > /tmp/reify-pre.json # doctor needs no index; point it at a repository with real history instead. reify -C "$REPO" --json doctor > /tmp/reify-doctor.json +# install without --yes writes nothing, so this is safe to run anywhere. +reify -C "$REPO" --json install > /tmp/reify-install.json echo "Now run the shape extractor in docs/json-schema/ to rebuild README.md" From 6ff262d91e80b5a92666c2f39c6efa0d498f7f6d Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 24 Aug 2026 12:38:08 +0700 Subject: [PATCH 8/8] docs(readme): lead with doctor and install The quickstart still taught `init --write-agent-instructions`, the hand-wiring path, and skipped the question `doctor` exists to answer. It now runs the two commands in the order someone actually needs them: should you use this, then wire it. The Install section gains the same pair, with what `install` detects, that an agent present on the machine but not configured in the repository is reported rather than written to, and that nothing outside the repository is touched. Both blocks were run as written against a scratch repository before being documented. --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2b34b29..a7f1afd 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,8 @@ ```bash curl -fsSL https://raw.githubusercontent.com/lambiengcode/reify/main/install.sh | sh cd your-repository -reify init --write-agent-instructions # wires your agent through AGENTS.md / CLAUDE.md +reify doctor # should you even use this? it will say no +reify install --yes # detects your agents and wires each one reify index # 4.2 s for 5,000 files; 0.5 s after one edit reify context "the change you are about to make" --toon ``` @@ -389,6 +390,19 @@ Or build from source: cargo install --path crates/reify-cli ``` +Then, in the repository you want it for: + +```bash +reify doctor # is this repository one Reify helps? it is willing to say no +reify install # shows what it found and what it would wire; --yes applies it +``` + +`install` detects the agents actually configured in the repository — `AGENTS.md`, +`CLAUDE.md`, `.cursor/`, `.clinerules/` and the rest — and wires each one the +[cheapest way that works](#wire-it-into-your-agent). An agent installed on the machine +but not configured here is reported, not written to, and nothing outside the repository +is touched. `--mcp` opts into MCP instead, and says what that costs before it writes. + **Stay current, leave cleanly.** `reify upgrade` replaces the binary with the latest release — through `curl` and `tar` as visible subprocesses, never an embedded HTTP client, with the checksum verified before anything is installed; `--check` only asks,