Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,14 @@ refused.
The server compares the signed runtime checksum with its catalogue of official
builds. An unrecognized or custom build is still accepted and shown with
download guidance; only a report whose signature does not verify is rejected.
Submitting the same report again succeeds rather than failing.
Submitting a report that is still published succeeds as an existing duplicate.
If you deleted it on the website, re-submitting that saved report fails instead,
including through **Select all**. The CLI labels it **Failed: previously deleted**
and continues attempting the other valid reports. Successful uploads remain
saved; the final summary counts the failures and the command exits nonzero.
Local files are unchanged. You can rerun the same model with the same settings
and submit the newly generated report as a separate benchmark. There is no need
to choose a different configuration.

The client talks to `https://computearena.ai/api/v1`. `--api-url` or
`COMPUTEARENA_API_URL` point it at another deployment, such as a local
Expand Down
25 changes: 25 additions & 0 deletions crates/computearena-cli/src/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ pub(crate) fn error_code(body: &str) -> Option<String> {
/// retire an old client by returning HTTP 426 (or the matching error code)
/// without making older clients fail with an unexplained generic status.
pub(crate) fn server_error(status: reqwest::StatusCode, body: &str) -> String {
if status == reqwest::StatusCode::CONFLICT
&& error_code(body).as_deref() == Some("submission_deleted")
{
return "This benchmark was previously deleted from ComputeArena. This saved report cannot be submitted again, including through Select all. You can rerun the same model with the same settings and submit the newly generated report as a separate benchmark. Your local report is unchanged.".to_string();
}
let message =
error_message(body).unwrap_or_else(|| format!("server returned HTTP {}", status.as_u16()));
let upgrade_required = status == reqwest::StatusCode::UPGRADE_REQUIRED
Expand Down Expand Up @@ -93,4 +98,24 @@ mod tests {
"Invalid report"
);
}

#[test]
fn deleted_reports_explain_why_select_all_cannot_restore_them() {
for body in [
r#"{"error":{"code":"submission_deleted"}}"#,
r#"{"error":{"code":"submission_deleted","message":"Report deleted"}}"#,
] {
let message = server_error(reqwest::StatusCode::CONFLICT, body);
assert!(message.contains("previously deleted"));
assert!(message.contains("Select all"));
assert!(message.contains("rerun the same model with the same settings"));
assert!(message.contains("newly generated report as a separate benchmark"));
assert!(message.contains("local report is unchanged"));
}
let body = r#"{"error":{"code":"submission_conflict","message":"Different report"}}"#;
assert_eq!(
server_error(reqwest::StatusCode::CONFLICT, body),
"Different report"
);
}
}
1 change: 1 addition & 0 deletions crates/computearena-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -897,6 +897,7 @@ mod tests {

#[test]
fn submission_stops_only_for_systemic_http_errors() {
assert!(!should_stop_submission(reqwest::StatusCode::CONFLICT));
assert!(!should_stop_submission(reqwest::StatusCode::BAD_REQUEST));
assert!(!should_stop_submission(
reqwest::StatusCode::UNPROCESSABLE_ENTITY
Expand Down
31 changes: 27 additions & 4 deletions crates/computearena-cli/src/submission.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
use crate::api::{client as api_client, server_error as api_server_error};
use crate::api::{
client as api_client, error_code as api_error_code, server_error as api_server_error,
};
use crate::auth::load_api_session;
use crate::config::SUBMISSION_HTTP_TIMEOUT;
use crate::model_identity::{verify_submission_model, SubmissionModelVerification};
Expand All @@ -16,6 +18,8 @@ use std::fs;
use std::io::{self, IsTerminal};
use std::path::{Path, PathBuf};

pub(crate) const DELETED_SUBMISSION_NOTICE: &str = "Previously deleted reports will fail to submit.\nSelect all does not restore them. Other valid reports\nwill still be attempted. Local files stay unchanged.\nYou can rerun the same model with the same settings,\nthen submit the newly generated report as a separate\nbenchmark.";

#[derive(Debug)]
pub(crate) struct PreparedSubmission {
pub(crate) path: PathBuf,
Expand All @@ -42,6 +46,7 @@ enum SubmissionOutcomeKind {
Submitted,
Duplicate,
Rejected,
PreviouslyDeleted,
NotAttempted,
}

Expand Down Expand Up @@ -254,6 +259,7 @@ pub(crate) fn submit_reports(
if prompt_yes_no("Preview the JSON data before submitting?", true)? {
print_submission_preview(ui, &preflight.ready)?;
}
println!("{}", ui.neutral(DELETED_SUBMISSION_NOTICE));
if !prompt_yes_no(
&format!(
"Submit the {} valid benchmark(s) now?",
Expand All @@ -270,6 +276,8 @@ pub(crate) fn submit_reports(
..SubmissionSummary::default()
});
}
} else {
println!("{}", ui.neutral(DELETED_SUBMISSION_NOTICE));
}

let endpoint = format!("{api_url}/submissions");
Expand Down Expand Up @@ -363,7 +371,13 @@ pub(crate) fn submit_reports(
eprintln!("{} {message}", ui.error("✗"));
outcomes.push(SubmissionOutcome {
label,
kind: SubmissionOutcomeKind::Rejected,
kind: if status == reqwest::StatusCode::CONFLICT
&& api_error_code(&body).as_deref() == Some("submission_deleted")
{
SubmissionOutcomeKind::PreviouslyDeleted
} else {
SubmissionOutcomeKind::Rejected
},
detail: Some(message.clone()),
});
if should_stop_submission(status) {
Expand Down Expand Up @@ -413,13 +427,19 @@ pub(crate) fn submit_reports(
.filter(|outcome| {
matches!(
outcome.kind,
SubmissionOutcomeKind::Rejected | SubmissionOutcomeKind::NotAttempted
SubmissionOutcomeKind::Rejected
| SubmissionOutcomeKind::PreviouslyDeleted
| SubmissionOutcomeKind::NotAttempted
)
})
.count();
if failures > 0 {
let deleted = outcomes
.iter()
.filter(|outcome| outcome.kind == SubmissionOutcomeKind::PreviouslyDeleted)
.count();
bail!(
"{failures} of {report_count} eligible benchmark(s) were not submitted; successful submissions remain saved"
"{submitted} uploaded, {duplicates} already present; {failures} of {report_count} eligible benchmark(s) were not submitted ({deleted} previously deleted). Successful uploads remain saved; local files are unchanged."
);
}
println!(
Expand Down Expand Up @@ -603,6 +623,9 @@ fn print_submission_results(ui: TerminalUi, outcomes: &[SubmissionOutcome]) {
SubmissionOutcomeKind::Submitted => (ui.success("✓"), "Submitted"),
SubmissionOutcomeKind::Duplicate => (ui.neutral("="), "Already submitted"),
SubmissionOutcomeKind::Rejected => (ui.error("✗"), "Rejected"),
SubmissionOutcomeKind::PreviouslyDeleted => {
(ui.error("✗"), "Failed: previously deleted")
}
SubmissionOutcomeKind::NotAttempted => (ui.warning("—"), "Not attempted"),
};
println!(" {marker} {} — {status}", outcome.label);
Expand Down
9 changes: 8 additions & 1 deletion crates/computearena-cli/src/tui/app.rs
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,11 @@ impl App {
"The fields above will be publicly accessible on ComputeArena. Local file paths are not sent."
.to_string(),
);
lines.extend(
crate::submission::DELETED_SUBMISSION_NOTICE
.lines()
.map(str::to_string),
);
self.screens.push(Screen::Preview {
lines,
scroll: 0,
Expand Down Expand Up @@ -1453,7 +1458,9 @@ mod tests {
app.activate().unwrap();
assert!(matches!(
app.screen(),
Screen::Preview { reports, .. } if *reports == vec![report.clone()]
Screen::Preview { reports, lines, .. } if *reports == vec![report.clone()]
&& lines.iter().any(|line| line.contains("Previously deleted reports will fail"))
&& lines.iter().any(|line| line.contains("Select all does not restore them"))
));
// The picker that started the run is gone: Esc lands on the menu.
assert!(matches!(
Expand Down
68 changes: 66 additions & 2 deletions crates/computearena-cli/tests/adapter_contract.rs
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,65 @@ fn repeated_runs_have_unique_ids_but_keep_the_installation_identity() {
}
}

#[test]
fn select_all_reports_deleted_failures_and_still_uploads_other_benchmarks() {
for statuses in [vec![409, 201, 200], vec![409]] {
let f = Fixture::new("llama-cpp");
let reports = f.dir.path().join("data/reports");
fs::create_dir_all(&reports).unwrap();
let originals: Vec<_> = (0..statuses.len())
.map(|index| {
let report = f.signed();
let path = reports.join(format!("saved-{index}.json"));
fs::rename(&f.report, &path).unwrap();
(path, report)
})
.collect();
let (url, received) = server(&f, statuses);
let mut child = f
.command()
.args(["--api-url", &url, "submit", "--yes"])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap();
child.stdin.take().unwrap().write_all(b"all\n").unwrap();
let output = child.wait_with_output().unwrap();
failure(&output, "1 previously deleted");
let printed = text(&output);
for hint in [
"Failed: previously deleted",
"Select all does not restore",
"rerun the same model with the same settings",
"newly generated report as a separate benchmark",
"local files are unchanged",
] {
assert!(printed.contains(hint), "missing {hint}: {printed}");
}
if originals.len() == 3 {
assert!(
printed.contains("1 uploaded, 1 already present"),
"{printed}"
);
} else {
assert!(
printed.contains("0 uploaded, 0 already present"),
"{printed}"
);
}
let attempted = received.join().unwrap();
assert_eq!(attempted.len(), originals.len());
for (path, report) in originals {
assert!(attempted.contains(&report));
assert_eq!(
serde_json::from_slice::<Value>(&fs::read(path).unwrap()).unwrap(),
report
);
}
}
}

#[test]
fn systemic_submission_failure_stops_the_queue_but_report_rejection_does_not() {
for status in [422, 429, 500] {
Expand Down Expand Up @@ -823,9 +882,14 @@ fn server(f: &Fixture, statuses: Vec<u16>) -> (String, thread::JoinHandle<Vec<Va
data.extend_from_slice(&buf[..n]);
}
received.push(serde_json::from_slice(&data[offset..offset + length]).unwrap());
let body = json!({"id":"test-submission","runtime_provenance":{
let body = if status == 409 {
json!({"error":{"code":"submission_deleted","message":"Report deleted"}})
.to_string()
} else {
json!({"id":"test-submission","runtime_provenance":{
"status":"mismatch","message":"Benchmark accepted. This binary differs from the registered release.",
"download_url":"https://github.com/ggml-org/llama.cpp/releases"}}).to_string();
"download_url":"https://github.com/ggml-org/llama.cpp/releases"}}).to_string()
};
write!(stream,"HTTP/1.1 {status} OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",body.len()).unwrap();
}
received
Expand Down
Loading