diff --git a/README.md b/README.md index 09c4ff2..5623291 100644 --- a/README.md +++ b/README.md @@ -300,7 +300,14 @@ refused. The server compares the signed runtime checksum with its catalogue of official builds. An unrecognized or custom build is still accepted and shown with download guidance; only a report whose signature does not verify is rejected. -Submitting the same report again succeeds rather than failing. +Submitting a report that is still published succeeds as an existing duplicate. +If you deleted it on the website, re-submitting that saved report fails instead, +including through **Select all**. The CLI labels it **Failed: previously deleted** +and continues attempting the other valid reports. Successful uploads remain +saved; the final summary counts the failures and the command exits nonzero. +Local files are unchanged. You can rerun the same model with the same settings +and submit the newly generated report as a separate benchmark. There is no need +to choose a different configuration. The client talks to `https://computearena.ai/api/v1`. `--api-url` or `COMPUTEARENA_API_URL` point it at another deployment, such as a local diff --git a/crates/computearena-cli/src/api.rs b/crates/computearena-cli/src/api.rs index 5f94c4a..d665a2d 100644 --- a/crates/computearena-cli/src/api.rs +++ b/crates/computearena-cli/src/api.rs @@ -42,6 +42,11 @@ pub(crate) fn error_code(body: &str) -> Option { /// retire an old client by returning HTTP 426 (or the matching error code) /// without making older clients fail with an unexplained generic status. pub(crate) fn server_error(status: reqwest::StatusCode, body: &str) -> String { + if status == reqwest::StatusCode::CONFLICT + && error_code(body).as_deref() == Some("submission_deleted") + { + return "This benchmark was previously deleted from ComputeArena. This saved report cannot be submitted again, including through Select all. You can rerun the same model with the same settings and submit the newly generated report as a separate benchmark. Your local report is unchanged.".to_string(); + } let message = error_message(body).unwrap_or_else(|| format!("server returned HTTP {}", status.as_u16())); let upgrade_required = status == reqwest::StatusCode::UPGRADE_REQUIRED @@ -93,4 +98,24 @@ mod tests { "Invalid report" ); } + + #[test] + fn deleted_reports_explain_why_select_all_cannot_restore_them() { + for body in [ + r#"{"error":{"code":"submission_deleted"}}"#, + r#"{"error":{"code":"submission_deleted","message":"Report deleted"}}"#, + ] { + let message = server_error(reqwest::StatusCode::CONFLICT, body); + assert!(message.contains("previously deleted")); + assert!(message.contains("Select all")); + assert!(message.contains("rerun the same model with the same settings")); + assert!(message.contains("newly generated report as a separate benchmark")); + assert!(message.contains("local report is unchanged")); + } + let body = r#"{"error":{"code":"submission_conflict","message":"Different report"}}"#; + assert_eq!( + server_error(reqwest::StatusCode::CONFLICT, body), + "Different report" + ); + } } diff --git a/crates/computearena-cli/src/main.rs b/crates/computearena-cli/src/main.rs index 90c6374..828d59d 100644 --- a/crates/computearena-cli/src/main.rs +++ b/crates/computearena-cli/src/main.rs @@ -897,6 +897,7 @@ mod tests { #[test] fn submission_stops_only_for_systemic_http_errors() { + assert!(!should_stop_submission(reqwest::StatusCode::CONFLICT)); assert!(!should_stop_submission(reqwest::StatusCode::BAD_REQUEST)); assert!(!should_stop_submission( reqwest::StatusCode::UNPROCESSABLE_ENTITY diff --git a/crates/computearena-cli/src/submission.rs b/crates/computearena-cli/src/submission.rs index 8f247b7..d6e6f3b 100644 --- a/crates/computearena-cli/src/submission.rs +++ b/crates/computearena-cli/src/submission.rs @@ -1,4 +1,6 @@ -use crate::api::{client as api_client, server_error as api_server_error}; +use crate::api::{ + client as api_client, error_code as api_error_code, server_error as api_server_error, +}; use crate::auth::load_api_session; use crate::config::SUBMISSION_HTTP_TIMEOUT; use crate::model_identity::{verify_submission_model, SubmissionModelVerification}; @@ -16,6 +18,8 @@ use std::fs; use std::io::{self, IsTerminal}; use std::path::{Path, PathBuf}; +pub(crate) const DELETED_SUBMISSION_NOTICE: &str = "Previously deleted reports will fail to submit.\nSelect all does not restore them. Other valid reports\nwill still be attempted. Local files stay unchanged.\nYou can rerun the same model with the same settings,\nthen submit the newly generated report as a separate\nbenchmark."; + #[derive(Debug)] pub(crate) struct PreparedSubmission { pub(crate) path: PathBuf, @@ -42,6 +46,7 @@ enum SubmissionOutcomeKind { Submitted, Duplicate, Rejected, + PreviouslyDeleted, NotAttempted, } @@ -254,6 +259,7 @@ pub(crate) fn submit_reports( if prompt_yes_no("Preview the JSON data before submitting?", true)? { print_submission_preview(ui, &preflight.ready)?; } + println!("{}", ui.neutral(DELETED_SUBMISSION_NOTICE)); if !prompt_yes_no( &format!( "Submit the {} valid benchmark(s) now?", @@ -270,6 +276,8 @@ pub(crate) fn submit_reports( ..SubmissionSummary::default() }); } + } else { + println!("{}", ui.neutral(DELETED_SUBMISSION_NOTICE)); } let endpoint = format!("{api_url}/submissions"); @@ -363,7 +371,13 @@ pub(crate) fn submit_reports( eprintln!("{} {message}", ui.error("✗")); outcomes.push(SubmissionOutcome { label, - kind: SubmissionOutcomeKind::Rejected, + kind: if status == reqwest::StatusCode::CONFLICT + && api_error_code(&body).as_deref() == Some("submission_deleted") + { + SubmissionOutcomeKind::PreviouslyDeleted + } else { + SubmissionOutcomeKind::Rejected + }, detail: Some(message.clone()), }); if should_stop_submission(status) { @@ -413,13 +427,19 @@ pub(crate) fn submit_reports( .filter(|outcome| { matches!( outcome.kind, - SubmissionOutcomeKind::Rejected | SubmissionOutcomeKind::NotAttempted + SubmissionOutcomeKind::Rejected + | SubmissionOutcomeKind::PreviouslyDeleted + | SubmissionOutcomeKind::NotAttempted ) }) .count(); if failures > 0 { + let deleted = outcomes + .iter() + .filter(|outcome| outcome.kind == SubmissionOutcomeKind::PreviouslyDeleted) + .count(); bail!( - "{failures} of {report_count} eligible benchmark(s) were not submitted; successful submissions remain saved" + "{submitted} uploaded, {duplicates} already present; {failures} of {report_count} eligible benchmark(s) were not submitted ({deleted} previously deleted). Successful uploads remain saved; local files are unchanged." ); } println!( @@ -603,6 +623,9 @@ fn print_submission_results(ui: TerminalUi, outcomes: &[SubmissionOutcome]) { SubmissionOutcomeKind::Submitted => (ui.success("✓"), "Submitted"), SubmissionOutcomeKind::Duplicate => (ui.neutral("="), "Already submitted"), SubmissionOutcomeKind::Rejected => (ui.error("✗"), "Rejected"), + SubmissionOutcomeKind::PreviouslyDeleted => { + (ui.error("✗"), "Failed: previously deleted") + } SubmissionOutcomeKind::NotAttempted => (ui.warning("—"), "Not attempted"), }; println!(" {marker} {} — {status}", outcome.label); diff --git a/crates/computearena-cli/src/tui/app.rs b/crates/computearena-cli/src/tui/app.rs index f596cf8..0f2c7c4 100644 --- a/crates/computearena-cli/src/tui/app.rs +++ b/crates/computearena-cli/src/tui/app.rs @@ -544,6 +544,11 @@ impl App { "The fields above will be publicly accessible on ComputeArena. Local file paths are not sent." .to_string(), ); + lines.extend( + crate::submission::DELETED_SUBMISSION_NOTICE + .lines() + .map(str::to_string), + ); self.screens.push(Screen::Preview { lines, scroll: 0, @@ -1453,7 +1458,9 @@ mod tests { app.activate().unwrap(); assert!(matches!( app.screen(), - Screen::Preview { reports, .. } if *reports == vec![report.clone()] + Screen::Preview { reports, lines, .. } if *reports == vec![report.clone()] + && lines.iter().any(|line| line.contains("Previously deleted reports will fail")) + && lines.iter().any(|line| line.contains("Select all does not restore them")) )); // The picker that started the run is gone: Esc lands on the menu. assert!(matches!( diff --git a/crates/computearena-cli/tests/adapter_contract.rs b/crates/computearena-cli/tests/adapter_contract.rs index 997f5e9..d97a3c2 100644 --- a/crates/computearena-cli/tests/adapter_contract.rs +++ b/crates/computearena-cli/tests/adapter_contract.rs @@ -188,6 +188,65 @@ fn repeated_runs_have_unique_ids_but_keep_the_installation_identity() { } } +#[test] +fn select_all_reports_deleted_failures_and_still_uploads_other_benchmarks() { + for statuses in [vec![409, 201, 200], vec![409]] { + let f = Fixture::new("llama-cpp"); + let reports = f.dir.path().join("data/reports"); + fs::create_dir_all(&reports).unwrap(); + let originals: Vec<_> = (0..statuses.len()) + .map(|index| { + let report = f.signed(); + let path = reports.join(format!("saved-{index}.json")); + fs::rename(&f.report, &path).unwrap(); + (path, report) + }) + .collect(); + let (url, received) = server(&f, statuses); + let mut child = f + .command() + .args(["--api-url", &url, "submit", "--yes"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(b"all\n").unwrap(); + let output = child.wait_with_output().unwrap(); + failure(&output, "1 previously deleted"); + let printed = text(&output); + for hint in [ + "Failed: previously deleted", + "Select all does not restore", + "rerun the same model with the same settings", + "newly generated report as a separate benchmark", + "local files are unchanged", + ] { + assert!(printed.contains(hint), "missing {hint}: {printed}"); + } + if originals.len() == 3 { + assert!( + printed.contains("1 uploaded, 1 already present"), + "{printed}" + ); + } else { + assert!( + printed.contains("0 uploaded, 0 already present"), + "{printed}" + ); + } + let attempted = received.join().unwrap(); + assert_eq!(attempted.len(), originals.len()); + for (path, report) in originals { + assert!(attempted.contains(&report)); + assert_eq!( + serde_json::from_slice::(&fs::read(path).unwrap()).unwrap(), + report + ); + } + } +} + #[test] fn systemic_submission_failure_stops_the_queue_but_report_rejection_does_not() { for status in [422, 429, 500] { @@ -823,9 +882,14 @@ fn server(f: &Fixture, statuses: Vec) -> (String, thread::JoinHandle