From f3fff1a57eb3cd1bef727e543522145f299f5dfa Mon Sep 17 00:00:00 2001 From: CLAUDE <355856132+HoTT-stuff@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:56:35 +0000 Subject: [PATCH 1/3] fix(adjudicate): never report pass when no report was processed or one failed to load `adjudicate` printed "verdict: pass (processed 0, failed 1)" and exited 0 for a missing or unrecognised report. Add an `error` verdict (fail-closed) for zero processed or any failed report, record failures in notes and stderr, and exit 2 (same as assail --fail-on-incomplete). A real `fail` still dominates `error`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0136P7ZkERqsQWMogHRpM2z8 --- CHANGELOG.adoc | 10 ++++ man/panic-attack.1 | 1 + src/adjudicate/mod.rs | 62 ++++++++++++++++++++++- src/main.rs | 10 ++++ tests/adjudicate_cli.rs | 106 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 tests/adjudicate_cli.rs diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 36725dc3..f08072ff 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -2,6 +2,16 @@ === [Unreleased] +==== Fixed — adjudicate no longer passes when it processed nothing + +* `+panic-attack adjudicate+` printed `+verdict: pass (processed 0, failed 1)+` and +exited 0 when a report was missing or unrecognised. A run that processed no +report, or failed to load any report, now gets the new verdict `+error+` (JSON +`+verdict+`; each failure is in `+notes+` and echoed to stderr) and exits with +status 2, the same code as `+assail --fail-on-incomplete+`. A real `+fail+` from a +loaded report still reports `+fail+`, but exits 2 too when any other report did +not load. Complete input keeps its exit code 0, including for `+warn+` and `+fail+`. + ==== Fixed — the shell `/tmp/` rule ignores comments and reports a line * `+analyze_shell+` raised "Hardcoded /tmp/ path without mktemp" on a diff --git a/man/panic-attack.1 b/man/panic-attack.1 index a3e140d0..90693a4f 100644 --- a/man/panic-attack.1 +++ b/man/panic-attack.1 @@ -33,6 +33,7 @@ Copies a target into an isolated workspace, optionally includes related files, a .TP .B adjudicate REPORTS... Aggregates assault/amuck/abduct report artifacts into a campaign-level verdict using rule-based inference. +Verdicts are pass, warn, fail, or error. error means no report was processed or at least one failed to load; it is never a pass. Exits 2 in that case, 0 otherwise. .TP .B axial TARGET Observes reaction signals by executing an external tool against a target and/or by inspecting existing report artifacts. diff --git a/src/adjudicate/mod.rs b/src/adjudicate/mod.rs index 6e741842..09c7246d 100644 --- a/src/adjudicate/mod.rs +++ b/src/adjudicate/mod.rs @@ -75,6 +75,31 @@ pub struct PriorityFinding { pub message: String, } +/// Verdict for a run that could not adjudicate its input: no report processed, +/// or at least one report failed to load/parse. Distinct from `pass`/`warn`/`fail`. +pub const VERDICT_ERROR: &str = "error"; + +/// Process exit status for an incomplete adjudication (`error` verdict). Matches +/// `assail --fail-on-incomplete`: 2 means "ran, but the result is not trustworthy". +pub const EXIT_INCOMPLETE: i32 = 2; + +impl AdjudicateReport { + /// True when no report was processed or any report failed to load. + pub fn is_incomplete(&self) -> bool { + self.processed_reports == 0 || self.failed_reports > 0 + } + + /// Exit status: `EXIT_INCOMPLETE` when incomplete, otherwise 0 (a `fail` + /// verdict on fully loaded input keeps its existing exit 0). + pub fn exit_code(&self) -> i32 { + if self.is_incomplete() { + EXIT_INCOMPLETE + } else { + 0 + } + } +} + pub fn run(config: AdjudicateConfig) -> Result { if config.reports.is_empty() { return Err(anyhow!("provide at least one report path")); @@ -198,15 +223,36 @@ pub fn run(config: AdjudicateConfig) -> Result { let has_fail = !db.get_facts("campaign_fail").is_empty(); let has_warn = !db.get_facts("campaign_warn").is_empty(); + // Fail-closed: a run that processed nothing, or could not load every report, + // has not adjudicated the campaign and must never read as `pass`/`warn`. + // A genuine `fail` signal from a loaded report still dominates `error`. + let incomplete = processed == 0 || failed > 0; + if processed == 0 { + notes.push("no report was processed; nothing was adjudicated".to_string()); + } let verdict = if has_fail { "fail" + } else if incomplete { + VERDICT_ERROR } else if has_warn { "warn" } else { "pass" }; - let priorities = build_priorities(&totals, verdict); + let mut priorities = build_priorities(&totals, verdict); + if incomplete { + priorities.insert( + 0, + PriorityFinding { + level: "high".to_string(), + message: format!( + "adjudication incomplete: {} report(s) processed, {} failed to load", + processed, failed + ), + }, + ); + } Ok(AdjudicateReport { schema_version: adjudicate_schema_version(), @@ -397,4 +443,18 @@ mod tests { assert_eq!(out.totals.amuck_reports, 1); assert_eq!(out.verdict, "warn"); } + + #[test] + fn adjudicate_zero_processed_is_error_not_pass() { + let dir = TempDir::new().expect("tempdir should create"); + let out = run(AdjudicateConfig { + reports: vec![dir.path().join("missing.json")], + }) + .expect("adjudicate should run"); + assert_eq!(out.processed_reports, 0); + assert_eq!(out.failed_reports, 1); + assert_eq!(out.verdict, VERDICT_ERROR); + assert!(out.is_incomplete()); + assert_eq!(out.exit_code(), EXIT_INCOMPLETE); + } } diff --git a/src/main.rs b/src/main.rs index df68732b..127e3ea8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2049,6 +2049,16 @@ fn run_main() -> Result<()> { "adjudicate report saved to: {}", report_path.display() ); + for note in &report.notes { + eprintln!("adjudicate: {}", note); + } + if report.is_incomplete() { + eprintln!( + "adjudicate: incomplete ({} processed, {} failed); verdict is not a pass", + report.processed_reports, report.failed_reports + ); + std::process::exit(report.exit_code()); + } } Commands::Axial { diff --git a/tests/adjudicate_cli.rs b/tests/adjudicate_cli.rs new file mode 100644 index 00000000..21b281fb --- /dev/null +++ b/tests/adjudicate_cli.rs @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: MPL-2.0 +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell + +//! CLI contract for `panic-attack adjudicate`: an adjudication that processed +//! no report, or failed to load any report, must never read as `pass`. + +use std::fs; +use std::path::Path; +use std::process::{Command, Output}; +use tempfile::TempDir; + +fn adjudicate(dir: &Path, reports: &[&Path]) -> (Output, serde_json::Value) { + let out = dir.join("out.json"); + let output = Command::new(env!("CARGO_BIN_EXE_panic-attack")) + .current_dir(dir) + .arg("adjudicate") + .args(reports) + .arg("--output") + .arg(&out) + .output() + .expect("binary should run"); + let json = fs::read_to_string(&out) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or(serde_json::Value::Null); + (output, json) +} + +fn good_abduct(dir: &Path) -> std::path::PathBuf { + let p = dir.join("good-abduct.json"); + let report = serde_json::json!({ + "schema_version": "2.5", + "created_at": "2026-01-01T00:00:00Z", + "target": "src/main.rs", + "source_root": ".", + "workspace_dir": "runtime/abduct", + "dependency_scope": "none", + "selected_files": 0, + "locked_files": 0, + "mtime_shifted_files": 0, + "mtime_offset_days": 0, + "time_mode": "normal", + "time_scale": null, + "virtual_now": null, + "files": [], + "execution": null, + }); + fs::write(&p, report.to_string()).unwrap(); + p +} + +#[test] +fn missing_file_is_not_pass() { + let dir = TempDir::new().unwrap(); + let missing = dir.path().join("nope.json"); + let (o, j) = adjudicate(dir.path(), &[&missing]); + assert_eq!(o.status.code(), Some(2), "{:?}", o); + assert_eq!(j["verdict"], "error"); + assert_eq!(j["processed_reports"], 0); + assert_eq!(j["failed_reports"], 1); + assert!(j["notes"].to_string().contains("nope.json")); + assert!(!String::from_utf8_lossy(&o.stdout).contains("verdict: pass")); +} + +#[test] +fn unrecognised_file_is_not_pass() { + let dir = TempDir::new().unwrap(); + let junk = dir.path().join("junk.json"); + fs::write(&junk, r#"{"hello": "world"}"#).unwrap(); + let (o, j) = adjudicate(dir.path(), &[&junk]); + assert_eq!(o.status.code(), Some(2), "{:?}", o); + assert_eq!(j["verdict"], "error"); + assert_eq!(j["failed_reports"], 1); + assert!(String::from_utf8_lossy(&o.stderr).contains("junk.json")); +} + +#[test] +fn good_plus_bad_is_not_pass() { + let dir = TempDir::new().unwrap(); + let good = good_abduct(dir.path()); + let missing = dir.path().join("gone.json"); + + // Control: the good report alone passes and exits 0. + let (o, j) = adjudicate(dir.path(), &[&good]); + assert_eq!(j["processed_reports"], 1, "fixture must parse: {:?}", o); + assert_eq!(j["verdict"], "pass"); + assert_eq!(o.status.code(), Some(0)); + + let (o, j) = adjudicate(dir.path(), &[&good, &missing]); + assert_eq!(j["processed_reports"], 1); + assert_eq!(j["failed_reports"], 1); + assert_eq!(j["verdict"], "error"); + assert_eq!(o.status.code(), Some(2)); +} + +#[test] +fn zero_arguments_is_rejected() { + let dir = TempDir::new().unwrap(); + let o = Command::new(env!("CARGO_BIN_EXE_panic-attack")) + .current_dir(dir.path()) + .arg("adjudicate") + .output() + .unwrap(); + assert_eq!(o.status.code(), Some(2)); + assert!(!String::from_utf8_lossy(&o.stdout).contains("verdict: pass")); +} From 4b7d268ef8198cdebd53486147730c5e482d4124 Mon Sep 17 00:00:00 2001 From: CLAUDE <355856132+HoTT-stuff@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:24:20 +0000 Subject: [PATCH 2/3] fix(adjudicate): exit 1 on a complete fail verdict; exit 2 on incomplete input Exit contract: 0 pass/warn, 1 complete fail, 2 incomplete or error (zero processed or any report failed to load; wins over everything). Verdict precedence stays fail > error > warn > pass. CLI tests for each code, man page and CHANGELOG updated, and the Chapel MassPanic adjudicate pass reads exit 1 as a valid fail verdict instead of dropping it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0136P7ZkERqsQWMogHRpM2z8 --- CHANGELOG.adoc | 10 ++-- chapel/src/MassPanic.chpl | 6 ++- man/panic-attack.1 | 6 ++- src/adjudicate/mod.rs | 14 +++-- src/main.rs | 5 +- tests/adjudicate_cli.rs | 104 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 135 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index f08072ff..60157595 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -8,9 +8,13 @@ exited 0 when a report was missing or unrecognised. A run that processed no report, or failed to load any report, now gets the new verdict `+error+` (JSON `+verdict+`; each failure is in `+notes+` and echoed to stderr) and exits with -status 2, the same code as `+assail --fail-on-incomplete+`. A real `+fail+` from a -loaded report still reports `+fail+`, but exits 2 too when any other report did -not load. Complete input keeps its exit code 0, including for `+warn+` and `+fail+`. +status 2, the same code as `+assail --fail-on-incomplete+`. Verdict precedence is +`+fail+` > `+error+` > `+warn+` > `+pass+`: a real `+fail+` from a loaded report is +still reported as `+fail+`. Exit contract: `+2+` for incomplete input (zero +processed or any report failed to load), whatever the verdict, winning over +everything; `+1+` for a complete `+fail+` (new: previously exit 0); `+0+` for a +complete `+pass+` or `+warn+`. The Chapel `+MassPanic+` adjudicate pass now reads +exit 1 as a valid `+fail+` verdict and exit 2 as `+error+`. ==== Fixed — the shell `/tmp/` rule ignores comments and reports a line diff --git a/chapel/src/MassPanic.chpl b/chapel/src/MassPanic.chpl index 320a14bd..94b7bf2b 100644 --- a/chapel/src/MassPanic.chpl +++ b/chapel/src/MassPanic.chpl @@ -843,7 +843,11 @@ module MassPanic { } sub.wait(); - if sub.exitCode == 0 { + // Exit contract: 0 pass/warn, 1 complete fail (a valid verdict), + // 2 incomplete/error (no verdict is trustworthy). + if sub.exitCode == 2 { + adjResult.verdict = "error"; + } else if sub.exitCode == 0 || sub.exitCode == 1 { result = parseRepoResult(jsonOutput, repoPath); result.fingerprint = computeFingerprint(repoPath); } else { diff --git a/man/panic-attack.1 b/man/panic-attack.1 index 90693a4f..e6f94e60 100644 --- a/man/panic-attack.1 +++ b/man/panic-attack.1 @@ -33,7 +33,11 @@ Copies a target into an isolated workspace, optionally includes related files, a .TP .B adjudicate REPORTS... Aggregates assault/amuck/abduct report artifacts into a campaign-level verdict using rule-based inference. -Verdicts are pass, warn, fail, or error. error means no report was processed or at least one failed to load; it is never a pass. Exits 2 in that case, 0 otherwise. +Verdict precedence is fail > error > warn > pass. The verdict error means no report was processed or at least one failed to load; it is never a pass. +.RS +.PP +Exit status: 2 if the input is incomplete (zero reports processed or any report failed to load), whatever the verdict; this wins over every other status. 1 if all reports loaded and the verdict is fail. 0 if all reports loaded and the verdict is pass or warn. +.RE .TP .B axial TARGET Observes reaction signals by executing an external tool against a target and/or by inspecting existing report artifacts. diff --git a/src/adjudicate/mod.rs b/src/adjudicate/mod.rs index 09c7246d..ee33b700 100644 --- a/src/adjudicate/mod.rs +++ b/src/adjudicate/mod.rs @@ -79,8 +79,12 @@ pub struct PriorityFinding { /// or at least one report failed to load/parse. Distinct from `pass`/`warn`/`fail`. pub const VERDICT_ERROR: &str = "error"; -/// Process exit status for an incomplete adjudication (`error` verdict). Matches -/// `assail --fail-on-incomplete`: 2 means "ran, but the result is not trustworthy". +/// Exit status for a complete run whose verdict is `fail`. +pub const EXIT_FAIL: i32 = 1; + +/// Exit status for incomplete input (zero processed or any report failed to load), +/// whatever the verdict. Matches `assail --fail-on-incomplete`: 2 means "ran, but +/// the result is not trustworthy". Wins over every other status. pub const EXIT_INCOMPLETE: i32 = 2; impl AdjudicateReport { @@ -89,11 +93,13 @@ impl AdjudicateReport { self.processed_reports == 0 || self.failed_reports > 0 } - /// Exit status: `EXIT_INCOMPLETE` when incomplete, otherwise 0 (a `fail` - /// verdict on fully loaded input keeps its existing exit 0). + /// Exit status. Precedence: incomplete input (2) > complete `fail` (1) > + /// complete `pass`/`warn` (0). pub fn exit_code(&self) -> i32 { if self.is_incomplete() { EXIT_INCOMPLETE + } else if self.verdict == "fail" { + EXIT_FAIL } else { 0 } diff --git a/src/main.rs b/src/main.rs index 127e3ea8..0383e59a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2057,7 +2057,10 @@ fn run_main() -> Result<()> { "adjudicate: incomplete ({} processed, {} failed); verdict is not a pass", report.processed_reports, report.failed_reports ); - std::process::exit(report.exit_code()); + } + let code = report.exit_code(); + if code != 0 { + std::process::exit(code); } } diff --git a/tests/adjudicate_cli.rs b/tests/adjudicate_cli.rs index 21b281fb..49fd6bd3 100644 --- a/tests/adjudicate_cli.rs +++ b/tests/adjudicate_cli.rs @@ -104,3 +104,107 @@ fn zero_arguments_is_rejected() { assert_eq!(o.status.code(), Some(2)); assert!(!String::from_utf8_lossy(&o.stdout).contains("verdict: pass")); } + +fn assault_json(crashes: usize) -> String { + assault_json_with(crashes, false) +} + +fn assault_json_with(crashes: usize, failed_attack: bool) -> String { + use panic_attack::types::*; + let report = AssaultReport { + schema_version: "2.5".to_string(), + assail_report: AssailReport { + coverage: Default::default(), + schema_version: "2.5".to_string(), + program_path: std::path::PathBuf::from("/tmp/t"), + language: Language::Rust, + frameworks: vec![], + weak_points: vec![], + statistics: ProgramStatistics { + total_lines: 1, + unsafe_blocks: 0, + panic_sites: 0, + unwrap_calls: 0, + safe_unwrap_calls: 0, + allocation_sites: 0, + io_operations: 0, + threading_constructs: 0, + }, + file_statistics: vec![], + dependency_graph: DependencyGraph { edges: vec![] }, + taint_matrix: TaintMatrix { rows: vec![] }, + recommended_attacks: vec![], + migration_metrics: None, + suppressed_count: 0, + }, + attack_results: if failed_attack { + vec![AttackResult { + program: std::path::PathBuf::from("/tmp/t"), + axis: AttackAxis::Cpu, + success: false, + skipped: false, + skip_reason: None, + exit_code: Some(1), + duration: std::time::Duration::from_millis(1), + peak_memory: 0, + crashes: vec![], + signatures_detected: vec![], + }] + } else { + vec![] + }, + total_crashes: crashes, + total_signatures: 0, + overall_assessment: OverallAssessment { + robustness_score: 50.0, + critical_issues: vec![], + recommendations: vec![], + }, + timeline: None, + }; + serde_json::to_string(&report).unwrap() +} + +#[test] +fn exit_0_on_complete_pass() { + let dir = TempDir::new().unwrap(); + let p = dir.path().join("clean.json"); + fs::write(&p, assault_json(0)).unwrap(); + let (o, j) = adjudicate(dir.path(), &[&p]); + assert_eq!(j["verdict"], "pass", "{:?}", o); + assert_eq!(o.status.code(), Some(0)); +} + +#[test] +fn exit_0_on_complete_warn() { + let dir = TempDir::new().unwrap(); + let p = dir.path().join("flaky.json"); + fs::write(&p, assault_json_with(0, true)).unwrap(); + let (o, j) = adjudicate(dir.path(), &[&p]); + assert_eq!(j["verdict"], "warn", "{:?}", o); + assert_eq!(o.status.code(), Some(0)); +} + +#[test] +fn exit_1_on_complete_fail() { + let dir = TempDir::new().unwrap(); + let p = dir.path().join("crashy.json"); + fs::write(&p, assault_json(3)).unwrap(); + let (o, j) = adjudicate(dir.path(), &[&p]); + assert_eq!(j["verdict"], "fail", "{:?}", o); + assert_eq!(j["failed_reports"], 0); + assert_eq!(o.status.code(), Some(1)); +} + +#[test] +fn exit_2_wins_over_fail_when_a_report_is_missing() { + let dir = TempDir::new().unwrap(); + let p = dir.path().join("crashy.json"); + fs::write(&p, assault_json(3)).unwrap(); + let missing = dir.path().join("gone.json"); + let (o, j) = adjudicate(dir.path(), &[&p, &missing]); + // Verdict precedence keeps the real failure visible... + assert_eq!(j["verdict"], "fail"); + // ...but incomplete input wins the exit code. + assert_eq!(o.status.code(), Some(2)); +} From 9c326ca0fab0625da80c6c331f7f986eeb0037c1 Mon Sep 17 00:00:00 2001 From: CLAUDE <355856132+HoTT-stuff@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:25:56 +0000 Subject: [PATCH 3/3] fix(chapel): apply the adjudicate exit contract in runAdjudicatePass, not scanRepo 4b7d268 put the exit-code branch in scanRepo, which runs the main scan and has no adjResult, so chapel-multilocale failed to compile (MassPanic.chpl:849 'adjResult' undeclared). scanRepo is restored to main; runAdjudicatePass now reads a verdict on exit 0 or 1 and records 'error' on exit 2. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0136P7ZkERqsQWMogHRpM2z8 --- chapel/src/MassPanic.chpl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/chapel/src/MassPanic.chpl b/chapel/src/MassPanic.chpl index 94b7bf2b..1fa2b19c 100644 --- a/chapel/src/MassPanic.chpl +++ b/chapel/src/MassPanic.chpl @@ -843,11 +843,7 @@ module MassPanic { } sub.wait(); - // Exit contract: 0 pass/warn, 1 complete fail (a valid verdict), - // 2 incomplete/error (no verdict is trustworthy). - if sub.exitCode == 2 { - adjResult.verdict = "error"; - } else if sub.exitCode == 0 || sub.exitCode == 1 { + if sub.exitCode == 0 { result = parseRepoResult(jsonOutput, repoPath); result.fingerprint = computeFingerprint(repoPath); } else { @@ -1052,7 +1048,11 @@ module MassPanic { } sub.wait(); - if sub.exitCode == 0 { + // adjudicate exit contract: 0 pass/warn, 1 complete fail (a valid + // verdict), 2 incomplete input or error (no verdict is trustworthy). + if sub.exitCode == 2 { + adjResult.verdict = "error"; + } else if sub.exitCode == 0 || sub.exitCode == 1 { // Extract crash count from adjudicate JSON adjResult.crashes = extractInt(jsonOutput, "\"crashes\":"); if adjResult.crashes == 0 then