diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 36725dc3..60157595 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -2,6 +2,20 @@ === [Unreleased] +==== Fixed — adjudicate no longer passes when it processed nothing + +* `+panic-attack adjudicate+` printed `+verdict: pass (processed 0, failed 1)+` and +exited 0 when a report was missing or unrecognised. A run that processed no +report, or failed to load any report, now gets the new verdict `+error+` (JSON +`+verdict+`; each failure is in `+notes+` and echoed to stderr) and exits with +status 2, the same code as `+assail --fail-on-incomplete+`. Verdict precedence is +`+fail+` > `+error+` > `+warn+` > `+pass+`: a real `+fail+` from a loaded report is +still reported as `+fail+`. Exit contract: `+2+` for incomplete input (zero +processed or any report failed to load), whatever the verdict, winning over +everything; `+1+` for a complete `+fail+` (new: previously exit 0); `+0+` for a +complete `+pass+` or `+warn+`. The Chapel `+MassPanic+` adjudicate pass now reads +exit 1 as a valid `+fail+` verdict and exit 2 as `+error+`. + ==== Fixed — the shell `/tmp/` rule ignores comments and reports a line * `+analyze_shell+` raised "Hardcoded /tmp/ path without mktemp" on a diff --git a/chapel/src/MassPanic.chpl b/chapel/src/MassPanic.chpl index 320a14bd..1fa2b19c 100644 --- a/chapel/src/MassPanic.chpl +++ b/chapel/src/MassPanic.chpl @@ -1048,7 +1048,11 @@ module MassPanic { } sub.wait(); - if sub.exitCode == 0 { + // adjudicate exit contract: 0 pass/warn, 1 complete fail (a valid + // verdict), 2 incomplete input or error (no verdict is trustworthy). + if sub.exitCode == 2 { + adjResult.verdict = "error"; + } else if sub.exitCode == 0 || sub.exitCode == 1 { // Extract crash count from adjudicate JSON adjResult.crashes = extractInt(jsonOutput, "\"crashes\":"); if adjResult.crashes == 0 then diff --git a/man/panic-attack.1 b/man/panic-attack.1 index a3e140d0..e6f94e60 100644 --- a/man/panic-attack.1 +++ b/man/panic-attack.1 @@ -33,6 +33,11 @@ Copies a target into an isolated workspace, optionally includes related files, a .TP .B adjudicate REPORTS... Aggregates assault/amuck/abduct report artifacts into a campaign-level verdict using rule-based inference. +Verdict precedence is fail > error > warn > pass. The verdict error means no report was processed or at least one failed to load; it is never a pass. +.RS +.PP +Exit status: 2 if the input is incomplete (zero reports processed or any report failed to load), whatever the verdict; this wins over every other status. 1 if all reports loaded and the verdict is fail. 0 if all reports loaded and the verdict is pass or warn. +.RE .TP .B axial TARGET Observes reaction signals by executing an external tool against a target and/or by inspecting existing report artifacts. diff --git a/src/adjudicate/mod.rs b/src/adjudicate/mod.rs index 6e741842..ee33b700 100644 --- a/src/adjudicate/mod.rs +++ b/src/adjudicate/mod.rs @@ -75,6 +75,37 @@ pub struct PriorityFinding { pub message: String, } +/// Verdict for a run that could not adjudicate its input: no report processed, +/// or at least one report failed to load/parse. Distinct from `pass`/`warn`/`fail`. +pub const VERDICT_ERROR: &str = "error"; + +/// Exit status for a complete run whose verdict is `fail`. +pub const EXIT_FAIL: i32 = 1; + +/// Exit status for incomplete input (zero processed or any report failed to load), +/// whatever the verdict. Matches `assail --fail-on-incomplete`: 2 means "ran, but +/// the result is not trustworthy". Wins over every other status. +pub const EXIT_INCOMPLETE: i32 = 2; + +impl AdjudicateReport { + /// True when no report was processed or any report failed to load. + pub fn is_incomplete(&self) -> bool { + self.processed_reports == 0 || self.failed_reports > 0 + } + + /// Exit status. Precedence: incomplete input (2) > complete `fail` (1) > + /// complete `pass`/`warn` (0). + pub fn exit_code(&self) -> i32 { + if self.is_incomplete() { + EXIT_INCOMPLETE + } else if self.verdict == "fail" { + EXIT_FAIL + } else { + 0 + } + } +} + pub fn run(config: AdjudicateConfig) -> Result { if config.reports.is_empty() { return Err(anyhow!("provide at least one report path")); @@ -198,15 +229,36 @@ pub fn run(config: AdjudicateConfig) -> Result { let has_fail = !db.get_facts("campaign_fail").is_empty(); let has_warn = !db.get_facts("campaign_warn").is_empty(); + // Fail-closed: a run that processed nothing, or could not load every report, + // has not adjudicated the campaign and must never read as `pass`/`warn`. + // A genuine `fail` signal from a loaded report still dominates `error`. + let incomplete = processed == 0 || failed > 0; + if processed == 0 { + notes.push("no report was processed; nothing was adjudicated".to_string()); + } let verdict = if has_fail { "fail" + } else if incomplete { + VERDICT_ERROR } else if has_warn { "warn" } else { "pass" }; - let priorities = build_priorities(&totals, verdict); + let mut priorities = build_priorities(&totals, verdict); + if incomplete { + priorities.insert( + 0, + PriorityFinding { + level: "high".to_string(), + message: format!( + "adjudication incomplete: {} report(s) processed, {} failed to load", + processed, failed + ), + }, + ); + } Ok(AdjudicateReport { schema_version: adjudicate_schema_version(), @@ -397,4 +449,18 @@ mod tests { assert_eq!(out.totals.amuck_reports, 1); assert_eq!(out.verdict, "warn"); } + + #[test] + fn adjudicate_zero_processed_is_error_not_pass() { + let dir = TempDir::new().expect("tempdir should create"); + let out = run(AdjudicateConfig { + reports: vec![dir.path().join("missing.json")], + }) + .expect("adjudicate should run"); + assert_eq!(out.processed_reports, 0); + assert_eq!(out.failed_reports, 1); + assert_eq!(out.verdict, VERDICT_ERROR); + assert!(out.is_incomplete()); + assert_eq!(out.exit_code(), EXIT_INCOMPLETE); + } } diff --git a/src/main.rs b/src/main.rs index df68732b..0383e59a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2049,6 +2049,19 @@ fn run_main() -> Result<()> { "adjudicate report saved to: {}", report_path.display() ); + for note in &report.notes { + eprintln!("adjudicate: {}", note); + } + if report.is_incomplete() { + eprintln!( + "adjudicate: incomplete ({} processed, {} failed); verdict is not a pass", + report.processed_reports, report.failed_reports + ); + } + let code = report.exit_code(); + if code != 0 { + std::process::exit(code); + } } Commands::Axial { diff --git a/tests/adjudicate_cli.rs b/tests/adjudicate_cli.rs new file mode 100644 index 00000000..49fd6bd3 --- /dev/null +++ b/tests/adjudicate_cli.rs @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: MPL-2.0 +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell + +//! CLI contract for `panic-attack adjudicate`: an adjudication that processed +//! no report, or failed to load any report, must never read as `pass`. + +use std::fs; +use std::path::Path; +use std::process::{Command, Output}; +use tempfile::TempDir; + +fn adjudicate(dir: &Path, reports: &[&Path]) -> (Output, serde_json::Value) { + let out = dir.join("out.json"); + let output = Command::new(env!("CARGO_BIN_EXE_panic-attack")) + .current_dir(dir) + .arg("adjudicate") + .args(reports) + .arg("--output") + .arg(&out) + .output() + .expect("binary should run"); + let json = fs::read_to_string(&out) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or(serde_json::Value::Null); + (output, json) +} + +fn good_abduct(dir: &Path) -> std::path::PathBuf { + let p = dir.join("good-abduct.json"); + let report = serde_json::json!({ + "schema_version": "2.5", + "created_at": "2026-01-01T00:00:00Z", + "target": "src/main.rs", + "source_root": ".", + "workspace_dir": "runtime/abduct", + "dependency_scope": "none", + "selected_files": 0, + "locked_files": 0, + "mtime_shifted_files": 0, + "mtime_offset_days": 0, + "time_mode": "normal", + "time_scale": null, + "virtual_now": null, + "files": [], + "execution": null, + }); + fs::write(&p, report.to_string()).unwrap(); + p +} + +#[test] +fn missing_file_is_not_pass() { + let dir = TempDir::new().unwrap(); + let missing = dir.path().join("nope.json"); + let (o, j) = adjudicate(dir.path(), &[&missing]); + assert_eq!(o.status.code(), Some(2), "{:?}", o); + assert_eq!(j["verdict"], "error"); + assert_eq!(j["processed_reports"], 0); + assert_eq!(j["failed_reports"], 1); + assert!(j["notes"].to_string().contains("nope.json")); + assert!(!String::from_utf8_lossy(&o.stdout).contains("verdict: pass")); +} + +#[test] +fn unrecognised_file_is_not_pass() { + let dir = TempDir::new().unwrap(); + let junk = dir.path().join("junk.json"); + fs::write(&junk, r#"{"hello": "world"}"#).unwrap(); + let (o, j) = adjudicate(dir.path(), &[&junk]); + assert_eq!(o.status.code(), Some(2), "{:?}", o); + assert_eq!(j["verdict"], "error"); + assert_eq!(j["failed_reports"], 1); + assert!(String::from_utf8_lossy(&o.stderr).contains("junk.json")); +} + +#[test] +fn good_plus_bad_is_not_pass() { + let dir = TempDir::new().unwrap(); + let good = good_abduct(dir.path()); + let missing = dir.path().join("gone.json"); + + // Control: the good report alone passes and exits 0. + let (o, j) = adjudicate(dir.path(), &[&good]); + assert_eq!(j["processed_reports"], 1, "fixture must parse: {:?}", o); + assert_eq!(j["verdict"], "pass"); + assert_eq!(o.status.code(), Some(0)); + + let (o, j) = adjudicate(dir.path(), &[&good, &missing]); + assert_eq!(j["processed_reports"], 1); + assert_eq!(j["failed_reports"], 1); + assert_eq!(j["verdict"], "error"); + assert_eq!(o.status.code(), Some(2)); +} + +#[test] +fn zero_arguments_is_rejected() { + let dir = TempDir::new().unwrap(); + let o = Command::new(env!("CARGO_BIN_EXE_panic-attack")) + .current_dir(dir.path()) + .arg("adjudicate") + .output() + .unwrap(); + assert_eq!(o.status.code(), Some(2)); + assert!(!String::from_utf8_lossy(&o.stdout).contains("verdict: pass")); +} + +fn assault_json(crashes: usize) -> String { + assault_json_with(crashes, false) +} + +fn assault_json_with(crashes: usize, failed_attack: bool) -> String { + use panic_attack::types::*; + let report = AssaultReport { + schema_version: "2.5".to_string(), + assail_report: AssailReport { + coverage: Default::default(), + schema_version: "2.5".to_string(), + program_path: std::path::PathBuf::from("/tmp/t"), + language: Language::Rust, + frameworks: vec![], + weak_points: vec![], + statistics: ProgramStatistics { + total_lines: 1, + unsafe_blocks: 0, + panic_sites: 0, + unwrap_calls: 0, + safe_unwrap_calls: 0, + allocation_sites: 0, + io_operations: 0, + threading_constructs: 0, + }, + file_statistics: vec![], + dependency_graph: DependencyGraph { edges: vec![] }, + taint_matrix: TaintMatrix { rows: vec![] }, + recommended_attacks: vec![], + migration_metrics: None, + suppressed_count: 0, + }, + attack_results: if failed_attack { + vec![AttackResult { + program: std::path::PathBuf::from("/tmp/t"), + axis: AttackAxis::Cpu, + success: false, + skipped: false, + skip_reason: None, + exit_code: Some(1), + duration: std::time::Duration::from_millis(1), + peak_memory: 0, + crashes: vec![], + signatures_detected: vec![], + }] + } else { + vec![] + }, + total_crashes: crashes, + total_signatures: 0, + overall_assessment: OverallAssessment { + robustness_score: 50.0, + critical_issues: vec![], + recommendations: vec![], + }, + timeline: None, + }; + serde_json::to_string(&report).unwrap() +} + +#[test] +fn exit_0_on_complete_pass() { + let dir = TempDir::new().unwrap(); + let p = dir.path().join("clean.json"); + fs::write(&p, assault_json(0)).unwrap(); + let (o, j) = adjudicate(dir.path(), &[&p]); + assert_eq!(j["verdict"], "pass", "{:?}", o); + assert_eq!(o.status.code(), Some(0)); +} + +#[test] +fn exit_0_on_complete_warn() { + let dir = TempDir::new().unwrap(); + let p = dir.path().join("flaky.json"); + fs::write(&p, assault_json_with(0, true)).unwrap(); + let (o, j) = adjudicate(dir.path(), &[&p]); + assert_eq!(j["verdict"], "warn", "{:?}", o); + assert_eq!(o.status.code(), Some(0)); +} + +#[test] +fn exit_1_on_complete_fail() { + let dir = TempDir::new().unwrap(); + let p = dir.path().join("crashy.json"); + fs::write(&p, assault_json(3)).unwrap(); + let (o, j) = adjudicate(dir.path(), &[&p]); + assert_eq!(j["verdict"], "fail", "{:?}", o); + assert_eq!(j["failed_reports"], 0); + assert_eq!(o.status.code(), Some(1)); +} + +#[test] +fn exit_2_wins_over_fail_when_a_report_is_missing() { + let dir = TempDir::new().unwrap(); + let p = dir.path().join("crashy.json"); + fs::write(&p, assault_json(3)).unwrap(); + let missing = dir.path().join("gone.json"); + let (o, j) = adjudicate(dir.path(), &[&p, &missing]); + // Verdict precedence keeps the real failure visible... + assert_eq!(j["verdict"], "fail"); + // ...but incomplete input wins the exit code. + assert_eq!(o.status.code(), Some(2)); +}